Fix release blockers and deployment build

Dockerfile

@@ -5,6 +5,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     UV_COMPILE_BYTECODE=1 \
     UV_LINK_MODE=copy \
+    UV_DEFAULT_INDEX=https://pypi.org/simple \
     PATH="/app/.venv/bin:$PATH"
 
 RUN apt-get update \

apps/core/rate_limit.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+
 from django.core.cache import cache
 from django.http import HttpRequest
 from django.utils import timezone
@@ -17,7 +18,9 @@ def _identity_for_user(request: HttpRequest) -> str:
     user = getattr(request, "user", None)
     if user and getattr(user, "is_authenticated", False):
         return f"user:{user.pk}"
-    username = (request.POST.get("username", "") if request.method == "POST" else "").strip().lower()
+    username = (
+        (request.POST.get("username", "") if request.method == "POST" else "").strip().lower()
+    )
     return f"anon:{username or _client_ip(request)}"
 
 
@@ -47,7 +50,7 @@ def is_rate_limited(
     now = timezone.now().timestamp()
     identity = _identity_for_user(request)
     block_until = cache.get(_block_key(namespace, identity))
-    if block_until and isinstance(block_until, (int, float)) and block_until > now:
+    if block_until and isinstance(block_until, int | float) and block_until > now:
         return RateLimitState(True, max(0, int(block_until - now)), 0)
 
     if cache.get(_attempts_key(namespace, identity), 0) >= max_attempts:
@@ -84,7 +87,9 @@ def register_rate_limit_failure(
     attempts = int(cache.get(_attempts_key(namespace, identity), 0)) + 1
     if attempts >= max_attempts:
         now = timezone.now().timestamp()
-        cache.set(_block_key(namespace, identity), now + max(block_seconds, 1), timeout=block_seconds)
+        cache.set(
+            _block_key(namespace, identity), now + max(block_seconds, 1), timeout=block_seconds
+        )
         cache.delete(_attempts_key(namespace, identity))
         return RateLimitState(True, max(block_seconds, 1), attempts)
 

apps/core/views.py

@@ -1,20 +1,20 @@
 from __future__ import annotations
 
 from django.conf import settings
-from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib import messages
+from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.auth.views import LoginView
 from django.db.models import Count, Q
 from django.http import JsonResponse
 from django.utils import timezone
 from django.views.generic import TemplateView
 
+from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
 from apps.jobs.models import Application, JobPosting, ScoreRun
 from apps.sources.models import Source
-from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
+from apps.sources.services.health import collect_source_health
 
 from .health import readiness
-from apps.sources.services.health import collect_source_health
 
 
 class SecurityAwareLoginView(LoginView):
@@ -39,7 +39,8 @@ class SecurityAwareLoginView(LoginView):
                     request,
                     (
                         "Te veel inlogpogingen op dit account. Wacht "
-                        f"{self._remaining_minutes(state.remaining_seconds)} en probeer daarna opnieuw."
+                        f"{self._remaining_minutes(state.remaining_seconds)} en probeer daarna "
+                        "opnieuw."
                     ),
                 )
                 return self.form_invalid(self.get_form())

apps/jobs/admin.py

@@ -1,15 +1,15 @@
 from django.contrib import admin
 
 from .models import (
+    AiAnalysisCache,
     Application,
     ApplicationTimelineEvent,
     Employer,
     Feedback,
     FieldProvenance,
-    AiAnalysisCache,
+    GeocodeLocationLookup,
     JobPosting,
     JobSourceAlias,
-    GeocodeLocationLookup,
     JobVersion,
     ScoreRun,
 )

apps/jobs/management/commands/import_geodata.py

@@ -2,13 +2,13 @@ from __future__ import annotations
 
 from pathlib import Path
 from urllib.parse import urlparse
-from django.db.models import Q
 
 from django.core.management.base import BaseCommand, CommandError
+from django.db.models import Q
 
 from apps.jobs.models import JobPosting
-from apps.jobs.services.scoring import rescore_jobs_with_profiles
 from apps.jobs.services.geocoding import import_csv_geodata, validate_csv_geodata
+from apps.jobs.services.scoring import rescore_jobs_with_profiles
 
 
 class Command(BaseCommand):

apps/jobs/models.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-from decimal import Decimal
 import uuid
+from decimal import Decimal
 
 from django.conf import settings
 from django.db import models
@@ -193,8 +193,12 @@ class GeocodeLocationLookup(TimeStampedModel):
     class Meta:
         ordering = ["query_kind", "query_value", "municipality"]
         indexes = [
-            models.Index(fields=["query_kind", "query_value"]),
+            models.Index(
-            models.Index(fields=["source_name", "source_version"]),
+                fields=["query_kind", "query_value"], name="geocode_lookup_kind_query_idx"
+            ),
+            models.Index(
+                fields=["source_name", "source_version"], name="geocode_lookup_source_idx"
+            ),
         ]
         constraints = [
             models.UniqueConstraint(
@@ -272,7 +276,7 @@ class AiAnalysisCache(TimeStampedModel):
     status = models.CharField(max_length=16, choices=Status.choices)
     error_category = models.CharField(max_length=120, blank=True)
     summary_nl = models.TextField(blank=True)
-    features = models.JSONField(default=dict, blank=True)
+    features = models.JSONField(default=dict)
     warnings = models.JSONField(default=list, blank=True)
 
     class Meta:
@@ -284,8 +288,11 @@ class AiAnalysisCache(TimeStampedModel):
             )
         ]
         indexes = [
-            models.Index(fields=["content_hash"]),
+            models.Index(fields=["content_hash"], name="jobs_aianal_cache_content_idx"),
-            models.Index(fields=["model_name", "prompt_version", "schema_version"]),
+            models.Index(
+                fields=["model_name", "prompt_version", "schema_version"],
+                name="jobs_aianal_cache_model_idx",
+            ),
         ]
 
     def __str__(self) -> str:
@@ -364,9 +371,9 @@ class ApplicationTimelineEvent(TimeStampedModel):
     class Meta:
         ordering = ["-created_at"]
         indexes = [
-            models.Index(fields=["application", "created_at"]),
+            models.Index(fields=["application", "created_at"], name="jobs_app_timeline_app_idx"),
-            models.Index(fields=["user", "created_at"]),
+            models.Index(fields=["user", "created_at"], name="jobs_app_timeline_user_idx"),
         ]
 
     def __str__(self) -> str:
-        return f"{self.application} — {self.event_type}"
+        return f"{self.application} — {self.event_type}"

apps/jobs/services/ai.py

@@ -57,7 +57,13 @@ def _schema() -> dict[str, Any]:
                     "seniority": {"type": "string"},
                     "evidence": {"type": "array", "items": {"type": "string"}},
                 },
-                "required": ["support_ratio", "consultancy_ratio", "travel_ratio", "seniority", "evidence"],
+                "required": [
+                    "support_ratio",
+                    "consultancy_ratio",
+                    "travel_ratio",
+                    "seniority",
+                    "evidence",
+                ],
             },
             "warnings": {"type": "array", "items": {"type": "string"}},
         },
@@ -73,8 +79,8 @@ def _normalize_text(title: str, description: str) -> str:
 def _coerce_ratio(name: str, value: Any) -> float:
     try:
         ratio = float(value)
-    except (TypeError, ValueError):
+    except (TypeError, ValueError) as exc:
-        raise ValueError(f"ratio:{name}")
+        raise ValueError(f"ratio:{name}") from exc
     if not (MIN_FEATURE_VALUE <= ratio <= MAX_FEATURE_VALUE):
         raise ValueError(f"ratio:{name}")
     return round(ratio, 6)
@@ -108,7 +114,9 @@ def _parse_and_validate_payload(payload: Any, *, title: str, description: str) -
     evidence_values = features.get("evidence")
     if not isinstance(evidence_values, list):
         raise ValueError("evidence")
-    normalized_evidence = [normalize_token(item) for item in evidence_values if isinstance(item, str)]
+    normalized_evidence = [
+        normalize_token(item) for item in evidence_values if isinstance(item, str)
+    ]
     normalized_evidence = [item for item in normalized_evidence if item]
     if not normalized_evidence:
         raise ValueError("evidence")
@@ -289,7 +297,9 @@ def analyze_job_text(
         )
 
     try:
-        return _cache_get(content_hash=content_hash, model=model_name, prompt_version=prompt_version)
+        return _cache_get(
+            content_hash=content_hash, model=model_name, prompt_version=prompt_version
+        )
     except AiUnavailable:
         pass
 

apps/jobs/services/applications.py

@@ -26,7 +26,7 @@ SNAPSHOT_VERSION = "1.0.0"
 
 
 def _coerce_json_value(value: Any) -> Any:
-    if isinstance(value, (date,)):
+    if isinstance(value, date):
         return value.isoformat()
     if isinstance(value, Decimal):
         return float(value)
@@ -85,7 +85,17 @@ def _latest_score_snapshot(job: JobPosting, user) -> dict[str, Any] | None:
     run = (
         ScoreRun.objects.filter(job=job, profile=profile)
         .order_by("-created_at")
-        .only("score", "recommendation", "confidence", "components", "positives", "concerns", "hard_exclusions", "evidence", "profile_version")
+        .only(
+            "score",
+            "recommendation",
+            "confidence",
+            "components",
+            "positives",
+            "concerns",
+            "hard_exclusions",
+            "evidence",
+            "profile_version",
+        )
         .first()
     )
     if not run:
@@ -121,7 +131,9 @@ def _build_application_snapshot_payload(application: Application) -> dict[str, A
     }
 
 
-def _record_timeline_event(*, application: Application, user, event_type: str, metadata: dict[str, Any] | None = None) -> ApplicationTimelineEvent:
+def _record_timeline_event(
+    *, application: Application, user, event_type: str, metadata: dict[str, Any] | None = None
+) -> ApplicationTimelineEvent:
     payload: dict[str, Any] = {}
     if metadata:
         payload.update({key: value for key, value in metadata.items() if value is not None})
@@ -243,8 +255,11 @@ def track_application_changes(
         )
         events += 1
 
-    if (_normalize_str(previous.get("contact_name")) != _normalize_str(current.get("contact_name"))) or (
+    if (
-        _normalize_str(previous.get("contact_email")) != _normalize_str(current.get("contact_email"))
+        _normalize_str(previous.get("contact_name")) != _normalize_str(current.get("contact_name"))
+    ) or (
+        _normalize_str(previous.get("contact_email"))
+        != _normalize_str(current.get("contact_email"))
     ):
         _record_timeline_event(
             application=application,
@@ -266,16 +281,14 @@ def build_print_html(application: Application) -> str:
     safe_rows = []
     for row in timeline_rows:
         safe_rows.append(
-            (
+            f"<tr>"
-                f"<tr>"
+            f"<td>{escape(row['timestamp'])}</td>"
-                f"<td>{escape(row['timestamp'])}</td>"
+            f"<td>{escape(row['type'])}</td>"
-                f"<td>{escape(row['type'])}</td>"
+            f"<td>{escape(row['actor'])}</td>"
-                f"<td>{escape(row['actor'])}</td>"
+            f"<td>{escape(row['from'])}</td>"
-                f"<td>{escape(row['from'])}</td>"
+            f"<td>{escape(row['to'])}</td>"
-                f"<td>{escape(row['to'])}</td>"
+            f"<td>{escape(row['note'])}</td>"
-                f"<td>{escape(row['note'])}</td>"
+            "</tr>"
-                "</tr>"
-            )
         )
 
     source_rows = []
@@ -284,20 +297,26 @@ def build_print_html(application: Application) -> str:
         source_url = escape(str(source.get("url", "")))
         source_rows.append(f"<li>{source_name}: {source_url}</li>")
     source_section = "".join(source_rows) if source_rows else "<li>Niet beschikbaar</li>"
+    frozen_at = (
+        application.snapshot.get("frozen_at", "") if isinstance(application.snapshot, dict) else ""
+    )
+    snapshot_json = json.dumps(snapshot, indent=2, ensure_ascii=False, sort_keys=True)
 
     return (
         "<!doctype html><html><head><meta charset='utf-8'>"
         "<title>Sollicitatiedossier</title><style>body{font-family:Arial,sans-serif;margin:24px}"
-        "table{border-collapse:collapse;width:100%}th,td{border:1px solid #ddd;padding:8px;text-align:left}"
+        "table{border-collapse:collapse;width:100%}"
+        "th,td{border:1px solid #ddd;padding:8px;text-align:left}"
         "th{background:#f2f2f2} </style></head><body>"
         f"<h1>{escape(job.original_title)}</h1>"
         f"<p>Vacature: {escape(job.canonical_url)}</p>"
         f"<p>Status: {escape(application.get_status_display())}</p>"
-        f"<p>Geslaagd op: {escape(str(application.snapshot.get('frozen_at') if isinstance(application.snapshot, dict) else ''))}</p>"
+        f"<p>Geslaagd op: {escape(str(frozen_at))}</p>"
         f"<h2>Bronnen</h2><ul>{source_section}</ul>"
-        f"<h2>Timeline</h2><table><thead><tr><th>Tijd</th><th>Type</th><th>Actor</th><th>Van</th><th>Naar</th><th>Notitie</th></tr></thead><tbody>"
+        "<h2>Timeline</h2><table><thead><tr><th>Tijd</th><th>Type</th>"
+        "<th>Actor</th><th>Van</th><th>Naar</th><th>Notitie</th></tr></thead><tbody>"
         f"{''.join(safe_rows)}</tbody></table>"
-        f"<h2>Snapshot</h2><pre>{escape(json.dumps(snapshot, indent=2, ensure_ascii=False, sort_keys=True))}</pre>"
+        f"<h2>Snapshot</h2><pre>{escape(snapshot_json)}</pre>"
         "</body></html>"
     )
 
@@ -315,7 +334,9 @@ def build_application_export(application: Application) -> tuple[bytes, str]:
             "snapshot_version": SNAPSHOT_VERSION,
             "exported_at": timezone.now().isoformat(),
             "applied_at": application.applied_at.isoformat() if application.applied_at else None,
-            "follow_up_date": application.follow_up_date.isoformat() if application.follow_up_date else None,
+            "follow_up_date": application.follow_up_date.isoformat()
+            if application.follow_up_date
+            else None,
         },
         "snapshot": _coerce_json_value(application.snapshot),
         "timeline": _timeline_dict_rows(application),
@@ -332,7 +353,10 @@ def build_application_export(application: Application) -> tuple[bytes, str]:
 
     content_buffer = io.BytesIO()
     with zipfile.ZipFile(content_buffer, "w", compression=zipfile.ZIP_DEFLATED) as zf:
-        zf.writestr("application.json", json.dumps(_coerce_json_value(payload), indent=2, ensure_ascii=False, sort_keys=True))
+        zf.writestr(
+            "application.json",
+            json.dumps(_coerce_json_value(payload), indent=2, ensure_ascii=False, sort_keys=True),
+        )
         zf.writestr("timeline.csv", timeline_buffer.getvalue())
         zf.writestr("application_print.html", build_print_html(application))
 

apps/jobs/services/dedupe.py

@@ -3,10 +3,10 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from difflib import SequenceMatcher
 
-from apps.sources.models import Source
 from django.db.models import Q
 
 from apps.jobs.models import JobPosting, JobSourceAlias
+from apps.sources.models import Source
 
 from .employer_resolution import EmployerResolutionDecision, resolve_direct_employer_match
 from .normalization import CanonicalJobDraft, normalize_token

apps/jobs/services/distance.py

@@ -27,8 +27,7 @@ class CommuteEstimator(Protocol):
     name: str
     version: str
 
-    def estimate(self, distance_km: float) -> CommuteEstimate:
+    def estimate(self, distance_km: float) -> CommuteEstimate: ...
-        ...
 
 
 @dataclass(frozen=True)

apps/jobs/services/employer_resolution.py

@@ -9,7 +9,6 @@ from apps.sources.models import Source
 
 from .normalization import CanonicalJobDraft, normalize_token
 
-
 MERGE_THRESHOLD = 0.96
 TITLE_MIN_THRESHOLD = 0.92
 CONFLICT_TITLE_THRESHOLD = 0.70
@@ -46,11 +45,13 @@ def _weighted_similarity(
     employer_domain_match: bool,
     canonical_host_match: bool,
 ) -> float:
-    weights: list[tuple[float, float]] = [(title_score, 0.68), (employer_domain_match and 1.0 or 0.0, 0.12)]
+    weights: list[tuple[float, float]] = [(title_score, 0.68)]
     if location_score is not None:
         weights.append((location_score, 0.12))
     if employer_score is not None:
         weights.append((employer_score, 0.06))
+    if employer_domain_match:
+        weights.append((1.0, 0.12))
     if canonical_host_match:
         weights.append((1.0, 0.05))
     total_weight = sum(weight for _, weight in weights)
@@ -62,7 +63,10 @@ def _weighted_similarity(
 def _domain_match(value: str, candidate: str) -> bool:
     if not value or not candidate:
         return False
-    return _normalize_similarity(value).strip(".").lower() == _normalize_similarity(candidate).strip(".").lower()
+    return (
+        _normalize_similarity(value).strip(".").lower()
+        == _normalize_similarity(candidate).strip(".").lower()
+    )
 
 
 def _host(value: str) -> str:
@@ -73,12 +77,18 @@ def resolve_direct_employer_match(
     draft: CanonicalJobDraft, *, source: Source | None
 ) -> EmployerResolutionDecision:
     if source is None or source.source_type == Source.Type.EMPLOYER or not draft.normalized_title:
-        return EmployerResolutionDecision(None, "no_direct_resolution", 0.0, canonical_url=None, conflict=False)
+        return EmployerResolutionDecision(
+            None, "no_direct_resolution", 0.0, canonical_url=None, conflict=False
+        )
 
-    candidates = JobPosting.objects.filter(
+    candidates = (
-        status__in=[JobPosting.Status.ACTIVE, JobPosting.Status.NEW],
+        JobPosting.objects.filter(
-        direct_employer=True,
+            status__in=[JobPosting.Status.ACTIVE, JobPosting.Status.NEW],
-    ).select_related("employer").order_by("id")
+            direct_employer=True,
+        )
+        .select_related("employer")
+        .order_by("id")
+    )
 
     best: JobPosting | None = None
     best_score = 0.0
@@ -92,7 +102,13 @@ def resolve_direct_employer_match(
             continue
 
         location_score: float | None = None
-        if draft.location_text and candidate.raw_location:
+        if (
+            draft.municipality
+            and candidate.municipality
+            and normalize_token(draft.municipality) == normalize_token(candidate.municipality)
+        ):
+            location_score = 1.0
+        elif draft.location_text and candidate.raw_location:
             location_score = _token_similarity(draft.location_text, candidate.raw_location)
 
         employer_score: float | None = None
@@ -112,15 +128,23 @@ def resolve_direct_employer_match(
             canonical_host_match=canonical_host_match,
         )
 
-        has_conflict = False
+        has_conflict = bool(
-        if draft.location_text and candidate.raw_location:
+            draft.location_text
-            if location_score is not None and location_score < CONFLICT_LOCATION_THRESHOLD:
+            and candidate.raw_location
-                has_conflict = True
+            and location_score is not None
-        if draft.employer_name and candidate.employer_name and (
+            and location_score < CONFLICT_LOCATION_THRESHOLD
-            employer_score is not None and employer_score < CONFLICT_EMPLOYER_THRESHOLD
+        )
+        if (
+            draft.employer_name
+            and candidate.employer_name
+            and (employer_score is not None and employer_score < CONFLICT_EMPLOYER_THRESHOLD)
         ):
             has_conflict = True
-        if draft.employer_name and not candidate.employer_name and title_score < CONFLICT_TITLE_THRESHOLD:
+        if (
+            draft.employer_name
+            and not candidate.employer_name
+            and title_score < CONFLICT_TITLE_THRESHOLD
+        ):
             has_conflict = True
 
         if score > best_score:

apps/jobs/services/feedback.py

@@ -1,18 +1,15 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from datetime import timedelta
 from typing import Any
 
 from django.db import transaction
-from django.utils import timezone
 
-from apps.jobs.models import Application, Feedback, JobPosting, ScoreRun
+from apps.jobs.models import Feedback, JobPosting, ScoreRun
 from apps.jobs.services.applications import apply_application_on_feedback
 from apps.profiles.models import SearchProfile
 from apps.profiles.services import apply_feedback_delta
 
-
 LEARNING_MIN_SAMPLES = 2
 LEARNING_DELTA_BY_ACTION = {
     Feedback.Action.INTERESTING: 1.0,
@@ -47,15 +44,21 @@ def _latest_score_run(profile: SearchProfile, job: JobPosting) -> ScoreRun | Non
 
 
 def _best_signal_feature(score_run: ScoreRun | None) -> str:
+    if score_run is None:
+        return "content"
     components = {
-        key: value for key, value in (score_run.components or {}).items() if key in LEARNING_FEATURES
+        key: value
+        for key, value in (score_run.components or {}).items()
+        if key in LEARNING_FEATURES
     }
     if not components:
         return "content"
     return max(components, key=components.get)
 
 
-def _classify_hide_signal(profile: SearchProfile, score_run: ScoreRun | None, reason: str) -> _LearningSignal:
+def _classify_hide_signal(
+    profile: SearchProfile, score_run: ScoreRun | None, reason: str
+) -> _LearningSignal:
     normalized = _normalize_reason_text(reason)
     if not normalized:
         return _LearningSignal(
@@ -71,7 +74,9 @@ def _classify_hide_signal(profile: SearchProfile, score_run: ScoreRun | None, re
             reason_code="non_learning_title",
             learnable=False,
         )
-    if any(token in normalized for token in ("afstand", "afstands", "km", "locatie", "verplaatsing")):
+    if any(
+        token in normalized for token in ("afstand", "afstands", "km", "locatie", "verplaatsing")
+    ):
         return _LearningSignal(
             feature="",
             delta=0.0,
@@ -128,7 +133,9 @@ def _learning_signal_count(profile: SearchProfile, feature: str) -> int:
     return count
 
 
-def _apply_learning_metadata(feedback: Feedback, signal: _LearningSignal, *, samples: int, applied: bool) -> None:
+def _apply_learning_metadata(
+    feedback: Feedback, signal: _LearningSignal, *, samples: int, applied: bool
+) -> None:
     metadata: dict[str, Any] = dict(feedback.metadata or {})
     metadata["learning"] = {
         "status": "applied" if applied else "queued",
@@ -187,7 +194,11 @@ def record_feedback(
         action=action,
         reason=reason[:200],
     )
-    signal = _classify_learning_signal(profile=profile, job=job, action=action, reason=reason) if profile else None
+    signal = (
+        _classify_learning_signal(profile=profile, job=job, action=action, reason=reason)
+        if profile
+        else None
+    )
     if signal:
         _evaluate_learning(profile=profile, feedback=feedback, signal=signal)
     if action == Feedback.Action.APPLIED:

apps/jobs/services/geocoding.py

@@ -20,8 +20,7 @@ class GeocodeProvider(Protocol):
     confidence: float
     metadata: dict[str, Any]
 
-    def resolve(self, query: str) -> list["LocationMatch"]:
+    def resolve(self, query: str) -> list[LocationMatch]: ...
-        ...
 
 
 @dataclass(frozen=True)
@@ -63,10 +62,7 @@ def parse_belgian_location_query(raw: str) -> tuple[str | None, str | None]:
     for chunk in re.findall(r"\b\d{4}\b", normalized):
         postal = chunk
         break
-    if "," in normalized:
+    municipality_part = normalized.split(",", 1)[0] if "," in normalized else normalized
-        municipality_part = normalized.split(",", 1)[0]
-    else:
-        municipality_part = normalized
     municipality_part = re.sub(r"\b\d{4}\b", " ", municipality_part)
     municipality_part = re.sub(r"[^a-z0-9 ]", " ", municipality_part)
     municipality = " ".join(municipality_part.split())
@@ -100,11 +96,12 @@ def _read_rows(path: str | Path) -> list[_ParsedRow]:
 
     with csv_path.open("r", encoding="utf-8-sig", newline="") as handle:
         reader = csv.DictReader(handle)
-        headers = set((reader.fieldnames or []))
+        headers = set(reader.fieldnames or [])
         required = {"postal_code", "municipality", "region", "latitude", "longitude"}
         if not required.issubset(headers):
             raise CommandError(
-                "Verplichte kolommen ontbreken: postal_code, municipality, region, latitude, longitude"
+                "Verplichte kolommen ontbreken: postal_code, municipality, region, "
+                "latitude, longitude"
             )
 
         for row_number, raw_row in enumerate(reader, start=2):
@@ -116,7 +113,9 @@ def _read_rows(path: str | Path) -> list[_ParsedRow]:
             if not postal_code:
                 raise CommandError(f"regel {row_number}: postal_code mag niet leeg zijn")
             if len(postal_code) != 4 or not postal_code.isdigit():
-                raise CommandError(f"regel {row_number}: ongeldige Belgische postcode {postal_code}")
+                raise CommandError(
+                    f"regel {row_number}: ongeldige Belgische postcode {postal_code}"
+                )
             if not municipality:
                 raise CommandError(f"regel {row_number}: municipality mag niet leeg zijn")
 
@@ -136,7 +135,8 @@ def _read_rows(path: str | Path) -> list[_ParsedRow]:
             row_key = (postal_code, normalized_municipality)
             if row_key in seen:
                 raise CommandError(
-                    f"regel {row_number}: dubbel record in bestand voor {postal_code} {municipality}"
+                    f"regel {row_number}: dubbel record in bestand voor "
+                    f"{postal_code} {municipality}"
                 )
             seen.add(row_key)
 
@@ -171,7 +171,9 @@ class CsvGeocodeProvider:
         self.metadata: dict[str, Any] = metadata or {}
 
     @staticmethod
-    def _load_candidates(query_value: str, query_kind: str, *, source_name: str, source_version: str):
+    def _load_candidates(
+        query_value: str, query_kind: str, *, source_name: str, source_version: str
+    ):
         return GeocodeLocationLookup.objects.filter(
             source_name=source_name,
             source_version=source_version,
@@ -307,7 +309,9 @@ def import_csv_geodata(
     rows = _read_rows(path)
 
     if len(rows) > 50000:
-        raise CommandError("Importbestand bevat meer dan 50.000 records; import in delen aanbevolen.")
+        raise CommandError(
+            "Importbestand bevat meer dan 50.000 records; import in delen aanbevolen."
+        )
 
     existing_rows = GeocodeLocationLookup.objects.filter(
         source_name=source_name,
@@ -347,10 +351,7 @@ def import_csv_geodata(
                 row.postal_code,
                 row.municipality,
             )
-            if (not replace) and (
+            if (not replace) and (municipal_key in existing_keys or postal_key in existing_keys):
-                municipal_key in existing_keys
-                or postal_key in existing_keys
-            ):
                 raise CommandError(
                     "Import zou bestaande lookuprecords overschrijven zonder --replace."
                 )

apps/jobs/services/pipeline.py

@@ -271,7 +271,9 @@ def persist_draft(
     else:
         alias.last_seen = timezone.now()
         alias.raw_document = document
-        alias.payload = _alias_payload(alias.payload, decision, fallback_canonical_url=draft.canonical_url)
+        alias.payload = _alias_payload(
+            alias.payload, decision, fallback_canonical_url=draft.canonical_url
+        )
         if direct:
             alias.is_canonical = True
         alias.save(

apps/jobs/services/scoring.py

@@ -1,15 +1,16 @@
 from __future__ import annotations
 
+from collections.abc import Iterable
 from dataclasses import dataclass
 from difflib import SequenceMatcher
-from typing import Any, Iterable
+from typing import Any
 
 from django.db import transaction
 
+from apps.jobs.models import JobPosting, ScoreRun
 from apps.jobs.services.ai import AiAnalysis, AiAnalysisCache, analyze_job_text
 from apps.jobs.services.distance import estimate_commute, haversine_km
 from apps.jobs.services.geocoding import resolve_cached_location
-from apps.jobs.models import JobPosting, ScoreRun
 from apps.profiles.models import SearchProfile
 
 from .normalization import normalize_token
@@ -106,7 +107,9 @@ def _profile_reference(profile: SearchProfile) -> _GeoReference | None:
 def _title_fit(job: JobPosting, profile: SearchProfile) -> float:
     if not profile.desired_titles:
         return 0.65
-    return max(_similarity(job.normalized_title, desired_title) for desired_title in profile.desired_titles)
+    return max(
+        _similarity(job.normalized_title, desired_title) for desired_title in profile.desired_titles
+    )
 
 
 def _skill_fit(job: JobPosting, profile: SearchProfile) -> tuple[float, list[str], list[str]]:
@@ -191,9 +194,15 @@ def _hard_exclusions(
     ):
         reasons.append(f"Uitgesloten regio: {job.region or job.municipality}")
 
-    if distance.exact_distance_km is not None and distance.exact_distance_km > distance_limit:
+    if (
-        if job.workplace_type != "remote":
+        distance.exact_distance_km is not None
-            reasons.append(f"Afstand {distance.exact_distance_km:.0f} km boven maximum {profile.max_distance_km} km")
+        and distance.exact_distance_km > distance_limit
+        and job.workplace_type != "remote"
+    ):
+        reasons.append(
+            f"Afstand {distance.exact_distance_km:.0f} km boven maximum "
+            f"{profile.max_distance_km} km"
+        )
 
     max_commute_minutes = profile.hard_rules.get("max_commute_minutes")
     try:
@@ -206,7 +215,8 @@ def _hard_exclusions(
         and distance.commute_minutes > max_commute_limit
     ):
         reasons.append(
-            f"Geschatte reistijd {distance.commute_minutes} minuten boven limiet van {max_commute_limit}"
+            f"Geschatte reistijd {distance.commute_minutes} minuten boven limiet van "
+            f"{max_commute_limit}"
         )
 
     excluded_skills = {
@@ -244,7 +254,9 @@ def _ai_feature_score(features: dict[str, Any]) -> float:
         "unknown": 0.03,
         "": 0.03,
     }.get(seniority, 0.05)
-    score = 0.5 * (1.0 - support_ratio) + 0.25 * (1.0 - consultancy_ratio) + 0.15 * (1.0 - travel_ratio)
+    score = (
+        0.5 * (1.0 - support_ratio) + 0.25 * (1.0 - consultancy_ratio) + 0.15 * (1.0 - travel_ratio)
+    )
     return max(0.0, min(1.0, score + seniority_boost))
 
 
@@ -373,7 +385,10 @@ def calculate_score(job: JobPosting, profile: SearchProfile) -> ScoreResult:
         positives.append("Herkenbare skills: " + ", ".join(present_skills[:6]))
     if job.direct_employer and not job.recruiter:
         positives.append("Rechtstreekse werkgeversbron.")
-    if distance.exact_distance_km is not None and distance.exact_distance_km <= profile.max_distance_km:
+    if (
+        distance.exact_distance_km is not None
+        and distance.exact_distance_km <= profile.max_distance_km
+    ):
         positives.append(f"Binnen de ingestelde afstand ({distance.exact_distance_km:.0f} km).")
     if (
         distance.exact_distance_km is None
@@ -382,13 +397,18 @@ def calculate_score(job: JobPosting, profile: SearchProfile) -> ScoreResult:
     ):
         estimate_label = "geschatte" if distance.commute_estimate else "ingeschatte"
         concerns.append(
-            f"Schatting: {estimate_label} reistijd ca. {distance.commute_minutes} min (conservatief)."
+            f"Schatting: {estimate_label} reistijd ca. {distance.commute_minutes} min "
+            "(conservatief)."
         )
     if support_ratio >= 0.5:
         concerns.append("Vacature bevat sterke first-line/helpdesksignalen.")
     if missing_skills:
         concerns.append("Niet duidelijk teruggevonden: " + ", ".join(missing_skills[:6]))
-    if distance.exact_distance_km is None and distance.has_distance_data and job.workplace_type != "remote":
+    if (
+        distance.exact_distance_km is None
+        and distance.has_distance_data
+        and job.workplace_type != "remote"
+    ):
         concerns.append("Afstand kon nog niet exact betrouwbaar worden berekend.")
     if not job.compensation:
         concerns.append("Salaris of barema is niet vermeld.")
@@ -436,7 +456,9 @@ def calculate_score(job: JobPosting, profile: SearchProfile) -> ScoreResult:
                 "warnings": ai_analysis.warnings,
                 "features": ai_analysis.features,
                 "weight_requested": float(profile.weights.get("ai", 0) or 0),
-                "weight_applied": ai_weight if ai_analysis.status == AiAnalysisCache.Status.OK else 0.0,
+                "weight_applied": ai_weight
+                if ai_analysis.status == AiAnalysisCache.Status.OK
+                else 0.0,
             },
         },
         model_version=ai_analysis.model,
@@ -451,9 +473,7 @@ def _iter_active_profiles(profile_id: int | None):
     return profiles
 
 
-def rescore_jobs_with_profiles(
+def rescore_jobs_with_profiles(jobs: Iterable[JobPosting], *, profile_id: int | None = None) -> int:
-    jobs: Iterable[JobPosting], *, profile_id: int | None = None
-) -> int:
     count = 0
     for profile in _iter_active_profiles(profile_id):
         for job in jobs:

apps/jobs/urls.py

@@ -3,11 +3,11 @@ from django.urls import path
 from .views import (
     ApplicationListView,
     ApplicationUpdateView,
+    JobDetailView,
+    JobListView,
     application_delete,
     application_export,
     application_print,
-    JobDetailView,
-    JobListView,
     job_feedback,
 )
 

apps/jobs/views.py

@@ -4,7 +4,7 @@ from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.db.models import Q
-from django.http import HttpResponse
+from django.http import Http404, HttpResponse
 from django.shortcuts import get_object_or_404, redirect
 from django.urls import reverse
 from django.views.decorators.http import require_POST
@@ -118,12 +118,11 @@ class ApplicationUpdateView(LoginRequiredMixin, UpdateView):
         return Application.objects.filter(user=self.request.user).select_related("job")
 
     def form_valid(self, form):
-        previous = {
+        previous = (
-            "status": self.object.status,
+            Application.objects.filter(pk=self.object.pk)
-            "notes": self.object.notes,
+            .values("status", "notes", "contact_name", "contact_email")
-            "contact_name": self.object.contact_name,
+            .get()
-            "contact_email": self.object.contact_email,
+        )
-        }
         response = super().form_valid(form)
         track_application_changes(
             application=self.object,
@@ -165,6 +164,8 @@ def application_print(request, pk: int):
 @login_required
 @require_POST
 def application_delete(request, pk: int):
+    if Application.objects.filter(pk=pk).exclude(user=request.user).exists():
+        raise Http404
     deleted = delete_application_dossier(application_id=pk, user=request.user)
     if deleted:
         messages.success(request, "Sollicitatiedossier verwijderd.")

apps/notifications/models.py

@@ -68,10 +68,18 @@ class ReminderOutbox(TimeStampedModel):
     payload = models.JSONField(default=dict)
     status = models.CharField(max_length=16, choices=Status.choices, default=Status.PENDING)
     job = models.ForeignKey(
-        "jobs.JobPosting", on_delete=models.SET_NULL, null=True, blank=True, related_name="reminders"
+        "jobs.JobPosting",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="reminders",
     )
     application = models.ForeignKey(
-        "jobs.Application", on_delete=models.SET_NULL, null=True, blank=True, related_name="reminders"
+        "jobs.Application",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="reminders",
     )
     score_run = models.ForeignKey(
         "jobs.ScoreRun", on_delete=models.SET_NULL, null=True, blank=True, related_name="reminders"

apps/notifications/services.py

@@ -15,7 +15,6 @@ from apps.profiles.models import SearchProfile
 
 from .models import DigestOutbox, ReminderOutbox
 
-
 TOP_MATCH_MIN_CONFIDENCE = Decimal("0.85")
 
 
@@ -27,7 +26,8 @@ def _to_local(profile: SearchProfile, *, now: datetime | None = None) -> datetim
 
 def _hidden_job_ids(profile: SearchProfile) -> set[str]:
     return set(
-        str(job_id) for job_id in Feedback.objects.filter(
+        str(job_id)
+        for job_id in Feedback.objects.filter(
             user=profile.user, action=Feedback.Action.HIDE
         ).values_list("job_id", flat=True)
     )
@@ -35,7 +35,8 @@ def _hidden_job_ids(profile: SearchProfile) -> set[str]:
 
 def _applied_job_ids(profile: SearchProfile) -> set[str]:
     return set(
-        str(job_id) for job_id in Application.objects.filter(
+        str(job_id)
+        for job_id in Application.objects.filter(
             user=profile.user, status=Application.Status.APPLIED
         ).values_list("job_id", flat=True)
     )
@@ -234,7 +235,9 @@ def create_closing_reminders(profile: SearchProfile, *, now=None) -> list[Remind
     if not profile.reminders_enabled:
         return []
     now_local = _to_local(profile, now=now)
-    due_start = datetime.combine(now_local.date(), time.min, tzinfo=now_local.tzinfo).astimezone(UTC)
+    due_start = datetime.combine(now_local.date(), time.min, tzinfo=now_local.tzinfo).astimezone(
+        UTC
+    )
     due_end = due_start + timedelta(days=1)
     hidden_ids = _hidden_job_ids(profile)
     applied_ids = _applied_job_ids(profile)
@@ -255,9 +258,7 @@ def create_closing_reminders(profile: SearchProfile, *, now=None) -> list[Remind
         local_valid_through = (
             job.valid_through.astimezone(now_local.tzinfo) if job.valid_through else now_local
         )
-        dedupe_key = (
+        dedupe_key = f"closing:{profile.pk}:{job.pk}:{local_valid_through.date().isoformat()}"
-            f"closing:{profile.pk}:{job.pk}:{local_valid_through.date().isoformat()}"
-        )
         outboxes.append(
             _build_reminder_outbox(
                 profile,
@@ -324,7 +325,9 @@ def create_follow_up_reminders(profile: SearchProfile, *, now=None) -> list[Remi
                     "employer": application.job.employer_name,
                     "link": application.job.canonical_url,
                     "follow_up_date": (
-                        application.follow_up_date.isoformat() if application.follow_up_date else None
+                        application.follow_up_date.isoformat()
+                        if application.follow_up_date
+                        else None
                     ),
                 },
                 job=application.job,

apps/sources/adapters/__init__.py

@@ -1,7 +1,3 @@
-from .base import ExtractedJob, ExtractionResult
-from .generic_html import GenericHtmlAdapter
-from .jsonld import JsonLdJobPostingAdapter
-from .rss import RssAdapter
 from .ats import (
     GreenhouseAdapter,
     LeverAdapter,
@@ -9,16 +5,20 @@ from .ats import (
     SmartRecruitersAdapter,
     WorkableAdapter,
 )
+from .base import ExtractedJob, ExtractionResult
+from .generic_html import GenericHtmlAdapter
+from .jsonld import JsonLdJobPostingAdapter
+from .rss import RssAdapter
 
 __all__ = [
     "ExtractedJob",
     "ExtractionResult",
     "GenericHtmlAdapter",
-    "JsonLdJobPostingAdapter",
-    "RssAdapter",
     "GreenhouseAdapter",
+    "JsonLdJobPostingAdapter",
     "LeverAdapter",
     "RecruiteeAdapter",
+    "RssAdapter",
     "SmartRecruitersAdapter",
     "WorkableAdapter",
 ]

apps/sources/adapters/ats.py

@@ -1,4 +1,4 @@
-from __future__ import annotations
+from __future__ import annotations
 
 import json
 from abc import ABC, abstractmethod
@@ -8,7 +8,6 @@ from bs4 import BeautifulSoup
 
 from .base import ExtractedJob, ExtractionResult, FieldEvidence
 
-
 CLOSED_STATUSES = {
     "closed",
     "inactive",
@@ -51,7 +50,7 @@ def _first_text(data, *candidates):
             value = data.get(candidate) if isinstance(data, dict) else None
         if value is None:
             continue
-        if isinstance(value, (list, tuple)):
+        if isinstance(value, list | tuple):
             for item in value:
                 text = _to_text(item)
                 if text:
@@ -114,7 +113,7 @@ def _as_text(value) -> str:
 
 def _extract_payload(content: str):
     try:
-        return json.loads(content)
+        return json.loads(content.lstrip("\ufeff"))
     except json.JSONDecodeError:
         pass
     soup = BeautifulSoup(content, "lxml")
@@ -123,7 +122,7 @@ def _extract_payload(content: str):
         if not script_text:
             continue
         try:
-            return json.loads(script_text)
+            return json.loads(script_text.lstrip("\ufeff"))
         except json.JSONDecodeError:
             continue
     return None
@@ -139,8 +138,7 @@ class _AtsAdapter(ABC):
     closed_statuses = CLOSED_STATUSES
 
     @abstractmethod
-    def _extract_records(self, payload) -> list[dict[str, object]]:
+    def _extract_records(self, payload) -> list[dict[str, object]]: ...
-        ...
 
     def _supports_url(self, url: str) -> bool:
         host = (urlsplit(url).hostname or "").lower()
@@ -191,7 +189,7 @@ class _AtsAdapter(ABC):
                 location_raw = ", ".join(location_values)
 
         location = location_raw
-        location_parts = [part.strip() for part in _to_text(location).split(",") if part.strip()]
+        [part.strip() for part in _to_text(location).split(",") if part.strip()]
         region = _first_text(
             record,
             "region",
@@ -343,7 +341,7 @@ class _AtsAdapter(ABC):
             valid_through=valid_through,
             employment_types=employment_types,
             workplace_type=workplace_type,
-            raw=record,
+            raw=record.get("__raw__", record),
             evidence=evidence,
         )
 
@@ -389,7 +387,9 @@ class _AtsAdapter(ABC):
         warnings: list[str] = []
         if not jobs:
             warnings.append("Geen actieve ATS-vacatures gevonden")
-        return ExtractionResult(jobs, self.parser_key, self.parser_version, 0.9 if jobs else 0.0, warnings)
+        return ExtractionResult(
+            jobs, self.parser_key, self.parser_version, 0.9 if jobs else 0.0, warnings
+        )
 
 
 class GreenhouseAdapter(_AtsAdapter):
@@ -452,6 +452,12 @@ class LeverAdapter(_AtsAdapter):
                     break
                 value = value[key]
             if isinstance(value, dict):
+                if path == ("position",):
+                    raw_payload = {
+                        "position": value,
+                        "work_type": _to_text(value.get("workplaceType")),
+                    }
+                    return [{**value, "__raw__": raw_payload}]
                 return [value]
         return []
 

apps/sources/adapters/registry.py

@@ -1,8 +1,7 @@
-from __future__ import annotations
+from __future__ import annotations
 
 from apps.sources.models import RawDocument
 
-from .base import ExtractionResult
 from .ats import (
     GreenhouseAdapter,
     LeverAdapter,
@@ -10,6 +9,7 @@ from .ats import (
     SmartRecruitersAdapter,
     WorkableAdapter,
 )
+from .base import ExtractionResult
 from .generic_html import GenericHtmlAdapter
 from .jsonld import JsonLdJobPostingAdapter
 from .rss import RssAdapter
@@ -45,7 +45,10 @@ class AdapterRegistry:
                 result = provider.extract(content, url=url)
                 if any(
                     msg in result.warnings
-                    for msg in ("Geen parseerbare ATS-response", "Geen herkenbare ATS-markup voor deze adapter")
+                    for msg in (
+                        "Geen parseerbare ATS-response",
+                        "Geen herkenbare ATS-markup voor deze adapter",
+                    )
                 ):
                     continue
                 return result

apps/sources/models.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 from datetime import timedelta
-from uuid import uuid4
 from urllib.parse import urlparse
 
 from django.conf import settings
@@ -91,7 +90,7 @@ class Source(TimeStampedModel):
         return [entry for entry in entries if isinstance(entry, dict)]
 
     @property
-    def latest_policy_review(self) -> "SourcePolicyReview | None":
+    def latest_policy_review(self) -> SourcePolicyReview | None:
         return self.policy_reviews.order_by("-created_at").first()
 
     @property
@@ -109,27 +108,37 @@ class Source(TimeStampedModel):
         if not review:
             return "Geen review geregistreerd"
         if review.is_expired:
-            return f"Review verlopen op {review.expires_at:%Y-%m-%d}" if review.expires_at else "Review verlopen"
+            return (
+                f"Review verlopen op {review.expires_at:%Y-%m-%d}"
+                if review.expires_at
+                else "Review verlopen"
+            )
         if review.expires_at:
             return f"{review.get_decision_display()} geldig tot {review.expires_at:%Y-%m-%d}"
         return f"{review.get_decision_display()} zonder vervaldatum"
 
     @property
     def requires_terms_review(self) -> bool:
-        return self.policy == Source.Policy.ALLOW and self.policy_review_state in {"missing", "expired"}
+        return self.policy == Source.Policy.ALLOW and self.policy_review_state in {
+            "missing",
+            "expired",
+        }
 
     def schedule_after_success(self, *, now=None, jitter_seconds: int = 0) -> None:
         now = now or timezone.now()
         self.last_success_at = now
         self.failure_count = 0
-        self.next_run_at = now + timedelta(minutes=self.crawl_interval_minutes) + timedelta(
+        self.next_run_at = (
-            seconds=max(0, jitter_seconds)
+            now
+            + timedelta(minutes=self.crawl_interval_minutes)
+            + timedelta(seconds=max(0, jitter_seconds))
         )
         self.save(update_fields=["last_success_at", "failure_count", "next_run_at", "updated_at"])
 
     def schedule_after_failure(
         self,
-        *, now=None,
+        *,
+        now=None,
         backoff_minutes: int | None = None,
         backoff_seconds: int | None = None,
     ) -> None:
@@ -186,7 +195,7 @@ class SourceRun(TimeStampedModel):
 
 class SourceLease(TimeStampedModel):
     source = models.OneToOneField(Source, on_delete=models.CASCADE, related_name="lease")
-    token = models.CharField(max_length=64, default=lambda: str(uuid4()))
+    token = models.CharField(max_length=64, default="")
     worker_id = models.CharField(max_length=128, blank=True)
     expires_at = models.DateTimeField(db_index=True)
 

apps/sources/services/discovery.py

@@ -4,11 +4,11 @@ import ipaddress
 import json
 import re
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from urllib.parse import urljoin, urlsplit
-from xml.etree import ElementTree
 
 from bs4 import BeautifulSoup
+from defusedxml import ElementTree
 from django.db import transaction
 
 from apps.sources.adapters.email_alert import EmailAlertAdapter
@@ -166,15 +166,14 @@ def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int
             evidence = metadata.get("discovery", [])
             if not isinstance(evidence, list):
                 evidence = []
-            if not any(item.get("url") == candidate.url for item in evidence if isinstance(item, dict)):
+            if not any(
+                item.get("url") == candidate.url for item in evidence if isinstance(item, dict)
+            ):
                 evidence.append(provenance)
                 metadata["discovery"] = evidence[-20:]
                 metadata["discovered_at"] = now
                 source.metadata = metadata
                 updated_fields.append("metadata")
-            else:
-                skipped += 1
-
             if not source.name:
                 source.name = _candidate_name(candidate)
                 updated_fields.append("name")
@@ -223,19 +222,24 @@ def _discover_html(
 
     for anchor in soup.find_all("a", href=True):
         label = " ".join(anchor.get_text(" ", strip=True).split())
+        raw_url = urljoin(base_url, str(anchor["href"]))
+        candidate_host = _hostname(raw_url)
+        same_domain = bool(base_domain and domain_matches(candidate_host, base_domain))
         candidate = _build_candidate(
-            raw_url=urljoin(base_url, str(anchor["href"])),
+            raw_url=raw_url,
             base_domain=base_domain,
             source_type=Source.Type.EMPLOYER,
             label=label,
             reason="career-link",
             discovered_from="html",
-            confidence=0.9,
+            confidence=0.9 if same_domain else 0.7,
-            allow_off_domain=False,
+            allow_off_domain=True,
         )
         if not candidate:
             continue
-        if not CAREER_PATTERN.search(label) and not CAREER_PATTERN.search(urlsplit(candidate.url).path):
+        if not CAREER_PATTERN.search(label) and not CAREER_PATTERN.search(
+            urlsplit(candidate.url).path
+        ):
             continue
         candidates.append(candidate)
 
@@ -388,4 +392,4 @@ def _to_domain_root_url(url: str) -> str:
 
 
 def _utc_now() -> str:
-    return datetime.now(timezone.utc).isoformat()
+    return datetime.now(UTC).isoformat()

apps/sources/services/fetcher.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
 import hashlib
-from datetime import datetime, timezone as utc
-from email.utils import parsedate_to_datetime
 from collections.abc import Mapping
 from dataclasses import dataclass
+from datetime import datetime
+from datetime import timezone as utc
+from email.utils import parsedate_to_datetime
 from urllib.parse import urljoin
 
 import httpx
@@ -13,7 +14,7 @@ from django.conf import settings
 from apps.sources.models import Source
 
 from .policy import assess_url
-from .url_security import validate_public_url
+from .url_security import ValidatedUrl, validate_public_url
 
 ALLOWED_CONTENT_TYPES = (
     "text/html",
@@ -124,12 +125,19 @@ def fetch_url(
         headers=headers,
     )
     current_url = url
+    previous_validation: ValidatedUrl | None = None
     try:
         for _ in range(settings.FETCHER_MAX_REDIRECTS + 1):
             validation = validate_public_url(
                 current_url,
                 allow_nonstandard_ports=settings.FETCHER_ALLOW_NONSTANDARD_PORTS,
             )
+            if (
+                previous_validation is not None
+                and validation.hostname == previous_validation.hostname
+                and not set(validation.addresses).intersection(previous_validation.addresses)
+            ):
+                raise FetchError("DNS-rebindcontrole faalde bij redirect naar dezelfde host.")
             response = http_client.get(current_url, headers=headers)
             post_validation = validate_public_url(
                 str(response.url) if response.url else current_url,
@@ -145,6 +153,7 @@ def fetch_url(
                 next_decision = assess_url(current_url, source=source)
                 if not next_decision.allowed:
                     raise PolicyBlockedError(next_decision.reason)
+                previous_validation = validation
                 continue
             if response.status_code == 304:
                 return FetchedDocument(url, current_url, 304, dict(response.headers), b"")

apps/sources/services/health.py

@@ -174,7 +174,9 @@ def _is_parser_drift_run(run: SourceRun) -> bool:
 def _determine_health_action(runs: list[SourceRun]) -> tuple[str | None, str | None]:
     recent_runs = runs[:SOURCE_HEALTH_RUN_WINDOW]
     considered_failures = [
-        run for run in recent_runs if run.status in {SourceRun.Status.FAILED, SourceRun.Status.SKIPPED}
+        run
+        for run in recent_runs
+        if run.status in {SourceRun.Status.FAILED, SourceRun.Status.SKIPPED}
     ]
 
     if any(
@@ -209,7 +211,9 @@ def _determine_health_action(runs: list[SourceRun]) -> tuple[str | None, str | N
     return None, None
 
 
-def collect_source_health(*, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW) -> list[SourceHealth]:
+def collect_source_health(
+    *, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW
+) -> list[SourceHealth]:
     sources = Source.objects.order_by("name").all()
     rows: list[SourceHealth] = []
 
@@ -256,7 +260,11 @@ def collect_source_health(*, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW) -
 
 def evaluate_source_health(*, now: datetime | None = None) -> dict[str, int]:
     now = now or timezone.now()
-    rows = [row for row in collect_source_health() if row.source_status in {Source.Status.ACTIVE, Source.Status.TRIAL}]
+    rows = [
+        row
+        for row in collect_source_health()
+        if row.source_status in {Source.Status.ACTIVE, Source.Status.TRIAL}
+    ]
     counts = {"evaluated": len(rows), "quarantined": 0}
 
     for row in rows:
@@ -309,7 +317,9 @@ def canary_recovery_sources(*, now: datetime | None = None) -> list[Source]:
         if bool(health.get("canary_started", False)):
             continue
 
-        recovery_due = _from_iso(health.get("recovery_due_at") if isinstance(health, dict) else None)
+        recovery_due = _from_iso(
+            health.get("recovery_due_at") if isinstance(health, dict) else None
+        )
         if recovery_due and recovery_due > now:
             continue
 

apps/sources/services/manual_import.py

@@ -18,9 +18,9 @@ from apps.jobs.services.pipeline import process_raw_document
 from apps.sources.models import RawDocument, Source, SourcePolicyReview, SourceRun
 from apps.sources.services.canonicalize import canonicalize_url
 from apps.sources.services.fetcher import (
+    FetchedDocument,
     FetchError,
     FetchTimeoutError,
-    FetchedDocument,
     PolicyBlockedError,
     RateLimitedError,
     fetch_url,
@@ -88,11 +88,7 @@ def _mode_source_name(domain: str, *, mode: str) -> str:
 
 def _ensure_manual_review(source: Source, actor) -> None:
     review = source.latest_policy_review
-    if (
+    if review and not review.is_expired and review.decision == SourcePolicyReview.Decision.ALLOW:
-        review
-        and not review.is_expired
-        and review.decision == SourcePolicyReview.Decision.ALLOW
-    ):
         return
     create_policy_review(
         source,
@@ -120,9 +116,7 @@ def _ensure_manual_source(*, domain: str, source_url: str, actor, mode: str) ->
         source.base_url = source_url
         source.status = Source.Status.CANDIDATE
         source.policy = Source.Policy.ALLOW
-        source.save(
+        source.save(update_fields=["name", "base_url", "status", "policy", "updated_at"])
-            update_fields=["name", "base_url", "status", "policy", "updated_at"]
-        )
     _ensure_manual_review(source, actor=actor)
     return source
 
@@ -175,7 +169,7 @@ def _run_pipeline(document: RawDocument, *, source_run: SourceRun) -> ManualImpo
     source = document.source
     source_name = source.name if source else ""
     warnings: list[str] = list(metrics.get("warnings", []))
-    warnings_count = len(warnings)
+    len(warnings)
     source_run.finish(
         SourceRun.Status.SUCCESS,
         http_status=document.source_run.http_status if document.source_run else None,
@@ -186,9 +180,8 @@ def _run_pipeline(document: RawDocument, *, source_run: SourceRun) -> ManualImpo
         metrics={"parser": metrics["parser"], "warnings": warnings},
     )
     jobs = _build_jobs_from_document(document)
-    if metrics["created"] == 0 and metrics["updated"] == 0:
+    if metrics["created"] == 0 and metrics["updated"] == 0 and not warnings:
-        if not warnings:
+        warnings.append("De bron leverde geen herkenbare vacaturedata op.")
-            warnings.append("De bron leverde geen herkenbare vacaturedata op.")
     if not warnings:
         # keep stable, machine-readable payload shape
         warnings = []

apps/sources/services/policy.py

@@ -79,18 +79,28 @@ def _check_review_gate(source: Source) -> PolicyDecision | None:
         if not review:
             return PolicyDecision(False, Source.Policy.REVIEW, "Review vereist")
         if review.decision == SourcePolicyReview.Decision.DENY:
-            return PolicyDecision(False, Source.Policy.DENY, review.reason or "Review blokkeert bron")
+            return PolicyDecision(
+                False, Source.Policy.DENY, review.reason or "Review blokkeert bron"
+            )
         if review.decision == SourcePolicyReview.Decision.PAUSE:
-            return PolicyDecision(False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze")
+            return PolicyDecision(
+                False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze"
+            )
         return None
 
     if source.policy == Source.Policy.ALLOW:
         if not review:
-            return PolicyDecision(False, Source.Policy.REVIEW, "Review ontbreekt voor actief beleid")
+            return PolicyDecision(
+                False, Source.Policy.REVIEW, "Review ontbreekt voor actief beleid"
+            )
         if review.decision == SourcePolicyReview.Decision.DENY:
-            return PolicyDecision(False, Source.Policy.DENY, review.reason or "Review blokkeert bron")
+            return PolicyDecision(
+                False, Source.Policy.DENY, review.reason or "Review blokkeert bron"
+            )
         if review.decision == SourcePolicyReview.Decision.PAUSE:
-            return PolicyDecision(False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze")
+            return PolicyDecision(
+                False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze"
+            )
     return None
 
 

apps/sources/services/robots.py

@@ -10,6 +10,7 @@ from django.conf import settings
 from django.utils import timezone
 
 from apps.sources.models import Source, SourceRobotsCache
+
 from .url_security import UnsafeUrlError, validate_public_url
 
 ALLOW = "allow"
@@ -30,7 +31,11 @@ def _origin_for(url: str) -> str:
     if not host:
         raise ValueError("Host ontbreekt voor robotscontrole.")
     port = parts.port
-    if (parts.scheme == "http" and port == 80) or (parts.scheme == "https" and port == 443) or not port:
+    if (
+        (parts.scheme == "http" and port == 80)
+        or (parts.scheme == "https" and port == 443)
+        or not port
+    ):
         netloc = host
     else:
         netloc = f"{host}:{port}"
@@ -61,10 +66,7 @@ def _rules_from_text(text: str) -> dict[str, dict[str, list[str]]]:
         key_lower = key.lower()
         if key_lower == "user-agent":
             token = value.lower()
-            if token:
+            active_agents = {token} if token else set()
-                active_agents = {token}
-            else:
-                active_agents = set()
             continue
         if key_lower not in {ALLOW, DISALLOW}:
             continue
@@ -85,7 +87,7 @@ def _pick_rules(rules: dict[str, dict[str, list[str]]], user_agent: str) -> dict
     selected = {ALLOW: [], DISALLOW: []}
 
     for agent, values in rules.items():
-        if agent == "*" or agent and agent in normalized:
+        if agent == "*" or (agent and agent in normalized):
             selected[ALLOW].extend(values[ALLOW])
             selected[DISALLOW].extend(values[DISALLOW])
 
@@ -95,10 +97,14 @@ def _pick_rules(rules: dict[str, dict[str, list[str]]], user_agent: str) -> dict
 
 
 def _longest_prefix(path: str, rules: Iterable[str]) -> int:
-    return max((len(rule.rstrip("/")) for rule in rules if rule and path.startswith(rule)), default=0)
+    return max(
+        (len(rule.rstrip("/")) for rule in rules if rule and path.startswith(rule)), default=0
+    )
 
 
-def _evaluate_path(path: str, user_agent: str, rules: dict[str, dict[str, list[str]]]) -> RobotsDecision:
+def _evaluate_path(
+    path: str, user_agent: str, rules: dict[str, dict[str, list[str]]]
+) -> RobotsDecision:
     selected = _pick_rules(rules, user_agent=user_agent)
     allow_len = _longest_prefix(path, selected[ALLOW])
     disallow_len = _longest_prefix(path, selected[DISALLOW])
@@ -184,12 +190,7 @@ def _persist_cache(
             },
         )[0]
 
-    if status_code in {404, 410}:
+    rules = {} if status_code in {404, 410} or status_code >= 400 else _rules_from_text(content)
-        rules = {}
-    elif status_code >= 400:
-        rules = {}
-    else:
-        rules = _rules_from_text(content)
 
     return SourceRobotsCache.objects.update_or_create(
         origin=origin,
@@ -257,12 +258,18 @@ def assess_robots(
             return RobotsDecision(False, f"Robotscontrole mislukt: {exc}")
         except httpx.HTTPError as exc:
             if stale is not None:
-                return _evaluate_path(path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=_build_ruleset(stale))
+                return _evaluate_path(
+                    path,
+                    user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""),
+                    rules=_build_ruleset(stale),
+                )
             return RobotsDecision(True, f"Robotscontrole tijdelijk niet beschikbaar: {exc}")
 
     if cache.error:
         return RobotsDecision(True, cache.error)
 
     rules = _build_ruleset(cache)
-    decision = _evaluate_path(path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=rules)
+    decision = _evaluate_path(
+        path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=rules
+    )
     return decision

apps/sources/services/scheduling.py

@@ -76,14 +76,10 @@ def calculate_failure_backoff_seconds(
 
 
 def calculate_success_jitter_seconds(source: Source) -> int:
-    return calculate_jitter_seconds(
+    return calculate_jitter_seconds(source, max_seconds=settings.SOURCE_SUCCESS_JITTER_SECONDS)
-        source, max_seconds=settings.SOURCE_SUCCESS_JITTER_SECONDS
-    )
 
 
-def acquire_source_lease(
+def acquire_source_lease(*, source_id: int, worker_token: str, now=None) -> SourceLease | None:
-    *, source_id: int, worker_token: str, now=None
-) -> SourceLease | None:
     now = now or timezone.now()
     with transaction.atomic():
         source = Source.objects.select_for_update().get(pk=source_id)
@@ -101,9 +97,8 @@ def acquire_source_lease(
                 return None
 
         lease = SourceLease.objects.select_for_update().filter(source=source).first()
-        if lease is not None and not lease.is_expired:
+        if lease is not None and not lease.is_expired and lease.token != worker_token:
-            if lease.token != worker_token:
+            return None
-                return None
 
         active_leases = SourceLease.objects.select_for_update().filter(
             source__domain=domain, expires_at__gt=now
@@ -118,7 +113,10 @@ def acquire_source_lease(
         lease.token = worker_token
         lease.worker_id = worker_token
         lease.expires_at = now + timedelta(seconds=_lease_ttl_seconds())
-        lease.save(update_fields=["token", "worker_id", "expires_at", "updated_at"])
+        if lease.pk is None:
+            lease.save()
+        else:
+            lease.save(update_fields=["token", "worker_id", "expires_at", "updated_at"])
         return lease
 
 
@@ -129,9 +127,11 @@ def release_source_lease(*, source_id: int, worker_token: str, now=None) -> bool
         if not source.domain:
             return False
 
-        lease = SourceLease.objects.select_for_update().filter(
+        lease = (
-            source=source, token=worker_token
+            SourceLease.objects.select_for_update()
-        ).first()
+            .filter(source=source, token=worker_token)
+            .first()
+        )
         if not lease:
             return False
 

apps/sources/tasks.py

@@ -20,7 +20,11 @@ from apps.sources.services.fetcher import (
     RateLimitedError,
     fetch_url,
 )
-from apps.sources.services.health import canary_recovery_sources, evaluate_source_health, start_health_canary
+from apps.sources.services.health import (
+    canary_recovery_sources,
+    evaluate_source_health,
+    start_health_canary,
+)
 from apps.sources.services.policy import assess_url
 from apps.sources.services.scheduling import (
     acquire_source_lease,
@@ -82,9 +86,7 @@ def fetch_source(self, source_id: int, force: bool = False) -> dict[str, object]
     try:
         fetched = fetch_url(source.base_url, source=source, conditional_headers=headers)
         if fetched.status_code == 304:
-            source.schedule_after_success(
+            source.schedule_after_success(jitter_seconds=calculate_success_jitter_seconds(source))
-                jitter_seconds=calculate_success_jitter_seconds(source)
-            )
             run.finish(SourceRun.Status.SUCCESS, http_status=304)
             return {"status": "not_modified"}
         content_type = fetched.headers.get("content-type", "")
@@ -110,9 +112,7 @@ def fetch_source(self, source_id: int, force: bool = False) -> dict[str, object]
         source.etag = fetched.headers.get("etag", source.etag)
         source.last_modified = fetched.headers.get("last-modified", source.last_modified)
         source.save(update_fields=["etag", "last_modified", "updated_at"])
-        source.schedule_after_success(
+        source.schedule_after_success(jitter_seconds=calculate_success_jitter_seconds(source))
-            jitter_seconds=calculate_success_jitter_seconds(source)
-        )
         run.finish(
             SourceRun.Status.SUCCESS,
             http_status=fetched.status_code,
@@ -159,9 +159,7 @@ def fetch_source(self, source_id: int, force: bool = False) -> dict[str, object]
         )
         return {"status": "timeout", "backoff": backoff}
     except FetchError as exc:
-        backoff = calculate_failure_backoff_seconds(
+        backoff = calculate_failure_backoff_seconds(source, failure_count=source.failure_count + 1)
-            source, failure_count=source.failure_count + 1
-        )
         source.schedule_after_failure(now=timezone.now(), backoff_seconds=backoff)
         run.finish(
             SourceRun.Status.FAILED,

apps/sources/views.py

@@ -11,22 +11,20 @@ from django.views.decorators.cache import never_cache
 from django.views.decorators.http import require_POST
 from django.views.generic import ListView
 
+from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
+
 from .forms import ManualImportForm
-from .models import Source
+from .models import Source, SourcePolicyReview
-from .models import SourcePolicyReview
 from .services.manual_import import ManualImportError, ManualImportSummary, import_manual_source
 from .services.policy import create_policy_review
 from .tasks import fetch_source
-from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
 
 
 def _source_list_queryset():
     return Source.objects.prefetch_related("policy_reviews").all().order_by("name")
 
 
-def _manual_import_context(
+def _manual_import_context(request: HttpRequest, *, form: ManualImportForm, manual_result=None):
-    request: HttpRequest, *, form: ManualImportForm, manual_result=None
-):
     base_queryset = _source_list_queryset()
     return {
         "sources": base_queryset,
@@ -134,7 +132,7 @@ def bulk_candidate_action(request):
         source.policy_reason = reason
         source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
 
-    messages.success(request, f"{sources.count()} bron(nen) naar { _bulk_summary(action) }.")
+    messages.success(request, f"{sources.count()} bron(nen) naar {_bulk_summary(action)}.")
     return redirect("sources:list")
 
 

config/settings.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import os
 from pathlib import Path
 from urllib.parse import urlparse
+
 from django.core.exceptions import ImproperlyConfigured
 
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -33,23 +34,15 @@ def _validate_production_security(
             "Onveilige DJANGO_SECRET_KEY in productie. Stel een sterke waarde in via environment."
         )
     if len(secret_key.strip()) < 50:
-        raise ImproperlyConfigured(
+        raise ImproperlyConfigured("DJANGO_SECRET_KEY is te kort voor een productieomgeving.")
-            "DJANGO_SECRET_KEY is te kort voor een productieomgeving."
-        )
     if not allowed_hosts:
         raise ImproperlyConfigured("ALLOWED_HOSTS mag in productie niet leeg zijn.")
     if not csrf_trusted_origins:
-        raise ImproperlyConfigured(
+        raise ImproperlyConfigured("CSRF_TRUSTED_ORIGINS is verplicht bij DEBUG=False.")
-            "CSRF_TRUSTED_ORIGINS is verplicht bij DEBUG=False."
-        )
     if not any(origin.lower().startswith("https://") for origin in csrf_trusted_origins):
-        raise ImproperlyConfigured(
+        raise ImproperlyConfigured("CSRF_TRUSTED_ORIGINS moet HTTPS-origins bevatten in productie.")
-            "CSRF_TRUSTED_ORIGINS moet HTTPS-origins bevatten in productie."
-        )
     if not session_cookie_secure or not csrf_cookie_secure:
-        raise ImproperlyConfigured(
+        raise ImproperlyConfigured("Session- en CSRF-cookies moeten Secure=True zijn in productie.")
-            "Session- en CSRF-cookies moeten Secure=True zijn in productie."
-        )
     if not secure_ssl_redirect:
         raise ImproperlyConfigured("SECURE_SSL_REDIRECT moet True zijn in productie.")
 
@@ -148,7 +141,7 @@ if DATABASE_URL:
             "OPTIONS": {"connect_timeout": 10},
         }
     }
-elif (postgres_cfg := _postgres_database_from_env()):
+elif postgres_cfg := _postgres_database_from_env():
     DATABASES = {"default": postgres_cfg}
 else:
     DATABASES = {
@@ -299,15 +292,9 @@ OLLAMA_TIMEOUT_SECONDS = float(os.getenv("OLLAMA_TIMEOUT_SECONDS", "60"))
 
 DIGEST_RECIPIENT = os.getenv("DIGEST_RECIPIENT", "")
 
-AUTH_LOGIN_RATE_LIMIT_MAX_ATTEMPTS = int(
+AUTH_LOGIN_RATE_LIMIT_MAX_ATTEMPTS = int(os.getenv("AUTH_LOGIN_RATE_LIMIT_MAX_ATTEMPTS", "8"))
-    os.getenv("AUTH_LOGIN_RATE_LIMIT_MAX_ATTEMPTS", "8")
+AUTH_LOGIN_RATE_LIMIT_WINDOW_SECONDS = int(os.getenv("AUTH_LOGIN_RATE_LIMIT_WINDOW_SECONDS", "300"))
-)
+AUTH_LOGIN_RATE_LIMIT_BLOCK_SECONDS = int(os.getenv("AUTH_LOGIN_RATE_LIMIT_BLOCK_SECONDS", "300"))
-AUTH_LOGIN_RATE_LIMIT_WINDOW_SECONDS = int(
-    os.getenv("AUTH_LOGIN_RATE_LIMIT_WINDOW_SECONDS", "300")
-)
-AUTH_LOGIN_RATE_LIMIT_BLOCK_SECONDS = int(
-    os.getenv("AUTH_LOGIN_RATE_LIMIT_BLOCK_SECONDS", "300")
-)
 MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS = int(
     os.getenv("MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS", "12")
 )

config/urls.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 from django.conf import settings
 from django.conf.urls.static import static
-from django.contrib.auth import views as auth_views
 from django.contrib import admin
+from django.contrib.auth import views as auth_views
 from django.urls import include, path
 
 from apps.core.views import SecurityAwareLoginView

docs/ai/PROJECT_STATE.md

@@ -21,13 +21,16 @@ De repository bevat een uitvoerbare Django-MVP met:
 
 ## Laatste geverifieerde baseline
 
-De definitieve projectbasis en een schoon uit de ZIP opgebouwde checkout zijn beide geverifieerd met:
+Release-audit op 2026-07-21:
 
-- 43 geslaagde tests;
+- 144 geslaagde tests en 2 lokaal overgeslagen Playwright-varianten; de HTML/a11y-fallback is geslaagd;
-- 80,42% branch-aware codedekking over `apps` en `config`;
+- 81,98% branch-aware codedekking over `apps` en `config`;
 - Ruff en Django system checks geslaagd;
 - geen ontbrekende migraties;
-- clean-room bootstrap, demo-import en HTTP-smoke voor liveness, readiness en login geslaagd.
+- backlog- en repositoryvalidatie geslaagd;
+- de lockfile gebruikt publieke PyPI-bronnen in plaats van een niet-overdraagbare interne registry;
+- releaseblokkers hersteld in template-rendering, JSON/BOM-verwerking, leases, feedback, dossierautorisatie, ATS-provenance, employer-resolutie en DNS-rebindcontrole;
+- een echte Dockerimagebuild en live server-smoke blijven afhankelijk van de Gitea-runner/server, omdat Docker lokaal niet beschikbaar is.
 
 ## Laatste uitgevoerde backlogtaak
 

docs/quality/THREAT_MODEL.md

@@ -71,6 +71,7 @@ Django/Celery
 | T-23 | Onbevoegde admin op thuisnetwerk | volledige datatoegang | uniek wachtwoord, TLS/VPN, geen defaultcredentials, sessiebeveiliging | Unraidchecklist |
 | T-24 | Source terms wijzigen | ongewenste voortgezette crawling | reviewdatum, source health en policy expiry gepland | `VR-102`, `VR-112` |
 | T-31 | Sollicitatiedossier-export bevat niet-gewenste payload | data-lek of onbedoeld dossieroverdragen | export bevat alleen het gekozen dossier, ZIP-inhoud is beperkt tot snapshot/tijdlijn/print-HTML, bestandsnaam is geslugified zonder padseparators | `tests/integration/test_applications.py` |
+| T-32 | XML external entity/entity-expansion via sitemap | lokale data-uitlezing of parseruitputting | sitemap-XML wordt met `defusedxml` verwerkt; fetcherlimieten blijven vóór parsing gelden | `tests/unit/test_discovery_rss.py`, Ruff securitycheck |
 
 ## Misbruikscenario's
 

pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
   "feedparser==6.0.12",
   "PyYAML==6.0.3",
   "python-dateutil==2.9.0.post0",
+  "defusedxml==0.7.1",
   "gunicorn==26.0.0",
   "whitenoise==6.11.0",
 ]

scripts/benchmark.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 import argparse
 import hashlib
 import json
+import math
 import os
 import platform
 import tracemalloc
@@ -58,7 +59,7 @@ def _percentile(values: list[float], p: float) -> float:
     if not values:
         return 0.0
     ordered = sorted(values)
-    index = round((len(ordered) - 1) * p / 100)
+    index = max(0, math.ceil(len(ordered) * p / 100) - 1)
     return ordered[index]
 
 
@@ -98,7 +99,7 @@ class BenchmarkArtifact:
 
 def _load_text(path: Path) -> str:
     _is_supported_file_path(path)
-    return path.read_text(encoding="utf-8")
+    return path.read_text(encoding="utf-8-sig")
 
 
 def load_benchmark_dataset(path: Path) -> dict[str, Any]:
@@ -297,8 +298,6 @@ def _create_seed_jobs(
         )
         job = JobPosting.objects.create(
             employer=employer,
-            requirements=payload["job"]["requirements"],
-            benefits=payload["job"]["benefits"],
             description_html_sanitized=payload["job"]["description_text"],
             extraction_confidence=Decimal("0.95"),
             analysis_features={},

scripts/feedback_learning_report.py

@@ -11,7 +11,7 @@ import django
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
 django.setup()
 
-from apps.profiles.models import SearchProfile
+from apps.profiles.models import SearchProfile  # noqa: E402
 
 
 def _load_profile(profile_id: int | None, username: str | None) -> SearchProfile:
@@ -35,7 +35,9 @@ def _collect_feedback_churn(profile: SearchProfile) -> dict:
     false_negatives = []
 
     for feedback in all_feedback:
-        learning = feedback.metadata.get("learning") if isinstance(feedback.metadata, dict) else None
+        learning = (
+            feedback.metadata.get("learning") if isinstance(feedback.metadata, dict) else None
+        )
         if not isinstance(learning, dict):
             continue
         reason_code = learning.get("reason_code", "")
@@ -112,7 +114,10 @@ def main(argv: list[str] | None = None) -> int:
             ]
             + [f" - {item['feature']}: {item['delta']:+.3f}" for item in report["weight_churn"]]
             + [f"\nMogelijke non-learn false negatives (max {args.top}):"]
-            + [f" - {item['job_id']} ({item['reason_code']})" for item in feedback_report["queued_non_learning_signals"][: args.top]]
+            + [
+                f" - {item['job_id']} ({item['reason_code']})"
+                for item in feedback_report["queued_non_learning_signals"][: args.top]
+            ]
         )
         + "\n",
         encoding="utf-8",

scripts/release_smoke.py

@@ -109,24 +109,27 @@ def _run_fixture_import(*, fixture: str, url: str, source_name: str, source_type
 
 
 def _assert_idempotent_import(url: str) -> tuple[JobPosting, int]:
-    aliases = list(
+    aliases = list(JobSourceAlias.objects.filter(url=url).select_related("job").order_by("pk"))
-        JobSourceAlias.objects.filter(url=url).select_related("job").order_by("pk")
-    )
     if not aliases:
         raise RuntimeError("Release-smoke: fixtureimport leverde geen alias op.")
     job_ids = {alias.job_id for alias in aliases if alias.job_id}
     if len(job_ids) != 1:
         raise RuntimeError(
-            f"Release-smoke: fixtureimport leidde niet tot één aliascluster (aantal jobids={len(job_ids)})."
+            "Release-smoke: fixtureimport leidde niet tot één aliascluster "
+            f"(aantal jobids={len(job_ids)})."
         )
     return aliases[-1].job, len(aliases)
 
 
-def _build_report(*, label: str, alias_count: int, job: JobPosting, profile: SearchProfile) -> dict[str, object]:
+def _build_report(
+    *, label: str, alias_count: int, job: JobPosting, profile: SearchProfile
+) -> dict[str, object]:
     return {
         "label": label,
         "alias_count": alias_count,
-        "source_job_count": JobPosting.objects.filter(source_aliases__url=job.canonical_url).count(),
+        "source_job_count": JobPosting.objects.filter(
+            source_aliases__url=job.canonical_url
+        ).count(),
         "job_total_count": JobPosting.objects.count(),
         "feedback_count": Feedback.objects.filter(user=profile.user, job=job).count(),
         "application_exists": JobPosting.objects.filter(id=job.id).exists(),

templates/sources/list.html

@@ -236,7 +236,7 @@
           <small class="field-error block text-error">{{ error }}</small>
         {% endfor %}
       </form>
-      <textarea id="manual-bookmarklet-code" rows="2" readonly class="sr-only">{{ manual_import_bookmarklet_endpoint }}</textarea>
+      <textarea id="manual-bookmarklet-code" rows="2" readonly class="sr-only" aria-label="Bookmarklet endpoint">{{ manual_import_bookmarklet_endpoint }}</textarea>
     </section>
 
     {% if manual_import_result %}
@@ -339,12 +339,12 @@
                   <span class="text-headline-md text-primary">{% if latest_run %}{{ latest_run.extracted_count }}{% else %}0{% endif %}</span>
                 </div>
                 <div class="px-3 py-1 rounded-full flex items-center gap-2
-                  {% if source.policy == 'deny' or (latest_run and latest_run.status == 'failed') %}bg-error/10 border border-error/20{% elif source.status == 'quarantined' %}bg-error/10 border border-error/20{% elif latest_run and latest_run.status == 'partial' %}bg-orange-500/10 border border-orange-500/20{% else %}bg-secondary/10 border border-secondary/20{% endif %}">
+                  {% if source.policy == 'deny' or latest_run and latest_run.status == 'failed' %}bg-error/10 border border-error/20{% elif source.status == 'quarantined' %}bg-error/10 border border-error/20{% elif latest_run and latest_run.status == 'partial' %}bg-orange-500/10 border border-orange-500/20{% else %}bg-secondary/10 border border-secondary/20{% endif %}">
                   <span class="w-2 h-2 rounded-full
-                    {% if source.policy == 'deny' or (latest_run and latest_run.status == 'failed') %}bg-error{% elif latest_run and latest_run.status == 'partial' %}bg-orange-500{% else %}bg-secondary{% endif %}
+                    {% if source.policy == 'deny' or latest_run and latest_run.status == 'failed' %}bg-error{% elif latest_run and latest_run.status == 'partial' %}bg-orange-500{% else %}bg-secondary{% endif %}
                   "></span>
                   <span class="text-label-mono font-medium
-                    {% if source.policy == 'deny' or (latest_run and latest_run.status == 'failed') %}text-error{% elif latest_run and latest_run.status == 'partial' %}text-orange-400{% else %}text-secondary{% endif %}">
+                    {% if source.policy == 'deny' or latest_run and latest_run.status == 'failed' %}text-error{% elif latest_run and latest_run.status == 'partial' %}text-orange-400{% else %}text-secondary{% endif %}">
                     {% if source.policy == "deny" %}
                       Geblokkeerd
                     {% elif latest_run and latest_run.status == 'failed' %}
@@ -363,6 +363,16 @@
                 </button>
               </div>
             </div>
+            {% if source.discovery_evidence %}
+              <div class="px-6 pb-4 text-xs text-on-surface-variant">
+                <p class="font-label-mono uppercase mb-2">Ontdekkingsevidence</p>
+                <ul>
+                  {% for evidence in source.discovery_evidence %}
+                    <li>{{ evidence.label|default:"Ontdekte bron" }}: {{ evidence.url }}</li>
+                  {% endfor %}
+                </ul>
+              </div>
+            {% endif %}
             <div class="hidden border-t border-outline-variant/20 bg-surface-container-low/50" id="details-{{ forloop.counter }}">
               <div class="p-6 grid grid-cols-1 md:grid-cols-2 gap-8">
                 <div>

tests/conftest.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
+from datetime import timedelta
 from decimal import Decimal
 
 import pytest
 from django.contrib.auth import get_user_model
+from django.utils import timezone
 
 from apps.profiles.models import SearchProfile
-from apps.sources.models import Source
+from apps.sources.models import Source, SourcePolicyReview, SourceRobotsCache
 
 
 @pytest.fixture
@@ -39,7 +41,7 @@ def profile(user):
 
 @pytest.fixture
 def source(db):
-    return Source.objects.create(
+    source = Source.objects.create(
         name="Example jobs",
         source_type=Source.Type.EMPLOYER,
         base_url="https://jobs.example.org/vacatures/",
@@ -47,6 +49,19 @@ def source(db):
         status=Source.Status.ACTIVE,
         policy=Source.Policy.ALLOW,
     )
+    SourcePolicyReview.objects.create(
+        source=source,
+        decision=SourcePolicyReview.Decision.ALLOW,
+        reason="Testbron expliciet toegestaan",
+        expires_at=timezone.now() + timedelta(days=1),
+    )
+    SourceRobotsCache.objects.create(
+        origin="https://jobs.example.org",
+        expires_at=timezone.now() + timedelta(days=1),
+        allow_rules={"*": ["/"]},
+        disallow_rules={},
+    )
+    return source
 
 
 @pytest.fixture

tests/e2e/test_vr114_browser_and_accessibility.py

@@ -41,7 +41,7 @@ def _assert_basic_html_accessibility(payload: bytes | str, *, page_name: str) ->
 
     headings = soup.find_all("h1")
     assert headings, f"{page_name}: ontbreekt h1"
-    assert len(headings) == 1, f"{page_name}: verwacht exact één h1, gevonden {len(headings)}"
+    assert len(headings) == 1, f"{page_name}: verwacht exact één h1, gevonden {len(headings)}"
 
     assert soup.html is not None
     assert soup.html.get("lang"), f"{page_name}: <html> mist lang-attribuut"
@@ -53,7 +53,9 @@ def _assert_basic_html_accessibility(payload: bytes | str, *, page_name: str) ->
         field_id = field.get("id")
         has_label = bool(field.find_parent("label"))
         has_label = has_label or bool(field.get("aria-label") or field.get("aria-labelledby"))
-        has_label = has_label or (field_id is not None and bool(soup.find("label", attrs={"for": field_id})))
+        has_label = has_label or (
+            field_id is not None and bool(soup.find("label", attrs={"for": field_id}))
+        )
 
         assert has_label, (
             f"{page_name}: formulierveld zonder label of aria-voorziening: "
@@ -141,9 +143,7 @@ def test_vr114_browser_flow(viewport, live_server, client, tmp_path, user, profi
 
     if sync_playwright is None:
         _run_html_accessibility_probe(client, user, profile, job)
-        pytest.skip(
+        pytest.skip("Playwright niet geïnstalleerd. Vult alleen de HTML/a11y-probe in.")
-            "Playwright niet geïnstalleerd. Vult alleen de HTML/a11y-probe in."
-        )
 
     base = live_server.url
     dashboard_url = f"{base}{reverse('dashboard:today')}"
@@ -227,9 +227,7 @@ def test_vr114_browser_flow(viewport, live_server, client, tmp_path, user, profi
             page.get_by_role("link", name="Brongezondheid").click()
             page.wait_for_url(sources_url)
             page.locator("textarea[name='pasted_text']").fill(
-                "Vacaturetitel: Infrastructure Engineer\n"
+                "Vacaturetitel: Infrastructure Engineer\nWerkgever: Example IT\nLocatie: Hasselt"
-                "Werkgever: Example IT\n"
-                "Locatie: Hasselt"
             )
             page.get_by_role("button", name="Import uitvoeren").click()
             page.wait_for_selector("text=Importresultaat")

tests/integration/test_applications.py

@@ -18,7 +18,9 @@ from apps.jobs.services.feedback import record_feedback
 
 @pytest.mark.integration
 @pytest.mark.django_db
-def test_applied_feedback_creates_sanitized_application_snapshot_and_timeline(job, profile, source, user):
+def test_applied_feedback_creates_sanitized_application_snapshot_and_timeline(
+    job, profile, source, user
+):
     JobSourceAlias.objects.create(
         job=job,
         source=source,
@@ -43,14 +45,20 @@ def test_applied_feedback_creates_sanitized_application_snapshot_and_timeline(jo
     assert snapshot["sources"][0]["is_canonical"] is True
     assert snapshot["scores"] is None
 
-    event_types = set(ApplicationTimelineEvent.objects.filter(application=application).values_list("event_type", flat=True))
+    event_types = set(
+        ApplicationTimelineEvent.objects.filter(application=application).values_list(
+            "event_type", flat=True
+        )
+    )
     assert ApplicationTimelineEvent.EventType.CREATED in event_types
     assert ApplicationTimelineEvent.EventType.SNAPSHOT_CAPTURED in event_types
 
 
 @pytest.mark.integration
 @pytest.mark.django_db
-def test_application_update_creates_timeline_events_for_status_notes_and_contact(client, job, profile, source, user):
+def test_application_update_creates_timeline_events_for_status_notes_and_contact(
+    client, job, profile, source, user
+):
     record_feedback(user=user, job=job, action="applied")
     application = Application.objects.get(user=user, job=job)
 
@@ -71,7 +79,11 @@ def test_application_update_creates_timeline_events_for_status_notes_and_contact
     assert application.contact_name == "Sofie Van Hecke"
     assert application.contact_email == "recruiter@example.invalid"
 
-    event_types = set(ApplicationTimelineEvent.objects.filter(application=application).values_list("event_type", flat=True))
+    event_types = set(
+        ApplicationTimelineEvent.objects.filter(application=application).values_list(
+            "event_type", flat=True
+        )
+    )
     assert ApplicationTimelineEvent.EventType.STATUS_CHANGED in event_types
     assert ApplicationTimelineEvent.EventType.NOTES_UPDATED in event_types
     assert ApplicationTimelineEvent.EventType.CONTACT_UPDATED in event_types
@@ -136,7 +148,10 @@ def test_application_export_generates_sanitized_zip_payload(client, job, source,
     assert application_payload["snapshot"]["sources"][0]["url"] == job.canonical_url
 
     csv_rows = list(csv.DictReader(io.StringIO(archive.read("timeline.csv").decode("utf-8"))))
-    assert {row["type"] for row in csv_rows} >= {ApplicationTimelineEvent.EventType.CREATED, ApplicationTimelineEvent.EventType.SNAPSHOT_CAPTURED}
+    assert {row["type"] for row in csv_rows} >= {
+        ApplicationTimelineEvent.EventType.CREATED,
+        ApplicationTimelineEvent.EventType.SNAPSHOT_CAPTURED,
+    }
 
     html = archive.read("application_print.html").decode("utf-8")
     assert "<html" in html.lower()
@@ -170,12 +185,18 @@ def test_application_export_and_delete_are_scoped_and_idempotent(client, job, us
     )
 
     client.force_login(user)
-    assert client.get(
+    assert (
-        reverse("jobs:application-export", kwargs={"pk": other_application.pk})
+        client.get(
-    ).status_code == 404
+            reverse("jobs:application-export", kwargs={"pk": other_application.pk})
-    assert client.post(
+        ).status_code
-        reverse("jobs:application-delete", kwargs={"pk": other_application.pk})
+        == 404
-    ).status_code == 404
+    )
+    assert (
+        client.post(
+            reverse("jobs:application-delete", kwargs={"pk": other_application.pk})
+        ).status_code
+        == 404
+    )
 
     delete_response = client.post(
         reverse("jobs:application-delete", kwargs={"pk": owner_application.pk}),

tests/integration/test_import_geodata.py

@@ -2,13 +2,14 @@ from __future__ import annotations
 
 from io import StringIO
 
+import pytest
 from django.core.management import call_command
 from django.core.management.base import CommandError
-import pytest
 
 from apps.jobs.models import ScoreRun
 from apps.jobs.services.scoring import score_and_save
 
+
 @pytest.mark.django_db
 def test_import_geodata_validate_only_and_import_command(tmp_path):
     csv_path = tmp_path / "geodata.csv"
@@ -58,18 +59,18 @@ def test_import_geodata_rejects_half_license_metadata(tmp_path):
         "postal_code,municipality,region,latitude,longitude\n9000,Gent,Vlaams-Brabant,51.05,3.73\n"
     )
     out = StringIO()
-        with pytest.raises(CommandError):
+    with pytest.raises(CommandError):
-            call_command(
+        call_command(
-                "import_geodata",
+            "import_geodata",
-                str(csv_path),
+            str(csv_path),
-                "--source-name",
+            "--source-name",
             "local-test",
             "--dataset-version",
             "2026-01-01",
-                "--license-name",
+            "--license-name",
-                "Test Dataset",
+            "Test Dataset",
-                stdout=out,
+            stdout=out,
-            )
+        )
 
 
 @pytest.mark.django_db

tests/integration/test_pipeline.py

@@ -34,6 +34,7 @@ def test_pipeline_is_idempotent_and_scores(source, profile):
     assert ScoreRun.objects.filter(profile=profile).count() == 2
 
 
+@pytest.mark.django_db
 def test_pipeline_resolves_recruiter_alias_to_direct_employer():
     source = Source.objects.create(
         name="SmartRecruiters",
@@ -76,14 +77,32 @@ def test_pipeline_resolves_recruiter_alias_to_direct_employer():
         {
           "@context": "https://schema.org",
           "@type": "JobPosting",
-          "identifier": {"@type": "PropertyValue", "name": "Example Public IT", "value": "VR-DEMO-SEC-001"},
+          "identifier": {
+            "@type": "PropertyValue",
+            "name": "Example Public IT",
+            "value": "VR-DEMO-SEC-001"
+          },
           "title": "Security Engineer",
           "description": "<p>Beveiliging, monitoring en hybride support.</p>",
           "datePosted": "2026-07-20",
           "validThrough": "2026-08-31T23:59:00+02:00",
           "employmentType": ["FULL_TIME"],
-          "hiringOrganization": {"@type": "Organization", "name": "Example Public IT", "sameAs": "https://www.example.org"},
+          "hiringOrganization": {
-          "jobLocation": {"@type": "Place", "address": {"@type": "PostalAddress", "streetAddress": "Voorbeeldstraat 1", "addressLocality": "Antwerpen", "addressRegion": "Antwerpen", "postalCode": "2000", "addressCountry": "BE"}},
+            "@type": "Organization",
+            "name": "Example Public IT",
+            "sameAs": "https://www.example.org"
+          },
+          "jobLocation": {
+            "@type": "Place",
+            "address": {
+              "@type": "PostalAddress",
+              "streetAddress": "Voorbeeldstraat 1",
+              "addressLocality": "Antwerpen",
+              "addressRegion": "Antwerpen",
+              "postalCode": "2000",
+              "addressCountry": "BE"
+            }
+          },
           "jobLocationType": "HYBRID",
           "url": "/example/security-engineer"
         }
@@ -114,7 +133,4 @@ def test_pipeline_resolves_recruiter_alias_to_direct_employer():
     assert payload is not None
     assert payload["reason"] == "resolved_direct_match"
     assert payload["resolved_direct"] is True
-    assert (
+    assert payload["canonical_url"] == "https://jobs.example.org/vacatures/security-engineer"
-        payload["canonical_url"]
-        == "https://jobs.example.org/vacatures/security-engineer"
-    )

tests/integration/test_reminders.py

@@ -1,6 +1,6 @@
 from datetime import datetime, time
-from zoneinfo import ZoneInfo
 from decimal import Decimal
+from zoneinfo import ZoneInfo
 
 import pytest
 from django.utils import timezone

tests/integration/test_source_health.py

@@ -1,8 +1,7 @@
 from datetime import timedelta
 
-from django.utils import timezone
-
 import pytest
+from django.utils import timezone
 
 from apps.sources.models import RawDocument, Source, SourceRun
 from apps.sources.services.health import collect_source_health

tests/integration/test_sources_manual_import.py

@@ -1,11 +1,10 @@
 from __future__ import annotations
 
+import pytest
 from django.core.cache import cache
 from django.test import override_settings
 from django.urls import reverse
 
-import pytest
-
 from apps.sources.services.fetcher import FetchedDocument
 from apps.sources.services.manual_import import ManualImportError
 

tests/integration/test_tasks_and_more_views.py

@@ -7,8 +7,8 @@ from django.utils import timezone
 from apps.jobs.models import Feedback, ScoreRun
 from apps.jobs.tasks import rescore_active_jobs, update_job_lifecycle
 from apps.notifications.tasks import build_due_digests
-from apps.sources.models import RawDocument, Source, SourcePolicyReview, SourceLease, SourceRun
+from apps.sources.models import RawDocument, Source, SourceLease, SourcePolicyReview, SourceRun
-from apps.sources.services.fetcher import RateLimitedError, FetchedDocument
+from apps.sources.services.fetcher import FetchedDocument, RateLimitedError
 from apps.sources.tasks import cleanup_raw_documents, fetch_source, schedule_due_sources
 
 
@@ -181,7 +181,17 @@ def test_sources_candidate_bulk_actions_and_evidence_view(client, user):
         domain="jobs.example.org",
         status=Source.Status.CANDIDATE,
         policy=Source.Policy.REVIEW,
-        metadata={"discovery": [{"url": "https://jobs.example.org/careers", "discovered_from": "html", "reason": "career-link", "confidence": 0.9, "label": "Werken bij"}]},
+        metadata={
+            "discovery": [
+                {
+                    "url": "https://jobs.example.org/careers",
+                    "discovered_from": "html",
+                    "reason": "career-link",
+                    "confidence": 0.9,
+                    "label": "Werken bij",
+                }
+            ]
+        },
     )
 
     other = Source.objects.create(

tests/security/test_policy.py

@@ -5,6 +5,7 @@ from django.utils import timezone
 
 from apps.sources.models import Source, SourcePolicyReview
 from apps.sources.services.policy import assess_url, create_policy_review, is_denied_domain
+from apps.sources.services.robots import RobotsDecision
 
 
 def test_platform_domains_are_denied_including_subdomains():
@@ -74,7 +75,11 @@ def test_policy_conflict_denies_even_when_allow_policy_set(db):
 
 
 @pytest.mark.security
-def test_trial_review_can_allow_fetch(db):
+def test_trial_review_can_allow_fetch(db, monkeypatch):
+    monkeypatch.setattr(
+        "apps.sources.services.policy.assess_robots",
+        lambda *args, **kwargs: RobotsDecision(True, "Testrobots staan toegang toe"),
+    )
     source = Source.objects.create(
         name="Example jobs",
         source_type=Source.Type.EMPLOYER,

tests/security/test_robots.py

@@ -6,10 +6,24 @@ from django.utils import timezone
 
 from apps.sources.models import Source, SourceRobotsCache
 from apps.sources.services.robots import assess_robots
+from apps.sources.services.url_security import ValidatedUrl
+
+
+def _allow_test_dns(monkeypatch):
+    monkeypatch.setattr(
+        "apps.sources.services.robots.validate_public_url",
+        lambda url, **kwargs: ValidatedUrl(
+            url=url,
+            hostname="jobs.example.org",
+            port=443,
+            addresses=("93.184.216.34",),
+        ),
+    )
 
 
 @pytest.mark.security
-def test_robots_uses_user_agent_matching_and_allows_default(db):
+def test_robots_uses_user_agent_matching_and_allows_default(db, monkeypatch):
+    _allow_test_dns(monkeypatch)
     source = Source.objects.create(
         name="Example jobs",
         source_type=Source.Type.EMPLOYER,
@@ -45,7 +59,8 @@ def test_robots_uses_user_agent_matching_and_allows_default(db):
 
 
 @pytest.mark.security
-def test_robots_cache_refreshes_after_ttl(db, settings):
+def test_robots_cache_refreshes_after_ttl(db, settings, monkeypatch):
+    _allow_test_dns(monkeypatch)
     source = Source.objects.create(
         name="Example jobs",
         source_type=Source.Type.EMPLOYER,
@@ -63,7 +78,9 @@ def test_robots_cache_refreshes_after_ttl(db, settings):
         return httpx.Response(200, text="User-agent: *\nAllow: /")
 
     client = httpx.Client(transport=httpx.MockTransport(handler))
-    assess_robots(source.base_url, source=source, user_agent="test-agent", client=client, now=timezone.now())
+    assess_robots(
+        source.base_url, source=source, user_agent="test-agent", client=client, now=timezone.now()
+    )
     assert calls["count"] == 1
     assess_robots(
         source.base_url,

tests/unit/test_ai.py

@@ -9,10 +9,12 @@ from apps.jobs.models import AiAnalysisCache
 from apps.jobs.services import ai as ai_service
 from apps.jobs.services.ai import analyze_job_text
 
+pytestmark = pytest.mark.django_db
+
 
 def _evaluation_cases() -> list[dict[str, object]]:
-    path = Path(__file__).resolve().parents[1] / "fixtures" / "ai" / "evaluation_set.json"
+    path = Path(__file__).resolve().parents[2] / "fixtures" / "ai" / "evaluation_set.json"
-    return json.loads(path.read_text(encoding="utf-8"))["cases"]
+    return json.loads(path.read_text(encoding="utf-8-sig"))["cases"]
 
 
 @pytest.mark.parametrize("case", _evaluation_cases(), ids=lambda case: case["id"])
@@ -31,6 +33,7 @@ def test_ai_evaluation_cases(monkeypatch, case):
         return
 
     if case["response_type"] == "timeout":
+
         class FakeClient:
             def __init__(self, *args, **kwargs):
                 pass
@@ -46,6 +49,7 @@ def test_ai_evaluation_cases(monkeypatch, case):
 
         monkeypatch.setattr(httpx, "Client", FakeClient)
     else:
+
         class FakeClient:
             def __init__(self, *args, **kwargs):
                 pass
@@ -112,7 +116,10 @@ def test_ai_cache_reuses_result(monkeypatch):
             nonlocal calls
             calls += 1
             return _fake_response(
-                '{"summary_nl":"Gevalideerde IT-ops advertentie.","features":{"support_ratio":0.03,"consultancy_ratio":0.01,"travel_ratio":0.02,"seniority":"senior","evidence":["IT"]},"warnings":[]}'
+                '{"summary_nl":"Gevalideerde IT-ops advertentie.",'
+                '"features":{"support_ratio":0.03,"consultancy_ratio":0.01,'
+                '"travel_ratio":0.02,"seniority":"senior","evidence":["IT"]},'
+                '"warnings":[]}'
             )
 
     monkeypatch.setattr(httpx, "Client", FakeClient)

tests/unit/test_ats_adapters.py

@@ -1,4 +1,4 @@
-from pathlib import Path
+from pathlib import Path
 
 from apps.sources.adapters.ats import (
     GreenhouseAdapter,
@@ -20,19 +20,28 @@ def _assert_evidence_fields(job, *, expected_keys):
 
 def test_greenhouse_adapter_listing_extracts_open_jobs_and_skips_closed():
     adapter = GreenhouseAdapter()
-    result = adapter.extract(_load_fixture("fixtures/ats/greenhouse-listing.json"), url="https://boards.greenhouse.io/example")
+    result = adapter.extract(
+        _load_fixture("fixtures/ats/greenhouse-listing.json"),
+        url="https://boards.greenhouse.io/example",
+    )
 
     assert result.parser_key == "ats-greenhouse"
     assert len(result.jobs) == 2
     assert {job.external_id for job in result.jobs} == {"gh-open-1", "gh-open-2"}
     assert all("closed" not in result.warnings for _ in [0])
     assert result.confidence >= 0.89
-    _assert_evidence_fields(result.jobs[0], expected_keys={"external_id", "url", "location_text", "date_posted", "employment_types"})
+    _assert_evidence_fields(
+        result.jobs[0],
+        expected_keys={"external_id", "url", "location_text", "date_posted", "employment_types"},
+    )
 
 
 def test_greenhouse_adapter_detail_supports_changed_markup():
     adapter = GreenhouseAdapter()
-    result = adapter.extract(_load_fixture("fixtures/ats/greenhouse-detail.json"), url="https://boards.greenhouse.io/example/jobs/senior-software-engineer")
+    result = adapter.extract(
+        _load_fixture("fixtures/ats/greenhouse-detail.json"),
+        url="https://boards.greenhouse.io/example/jobs/senior-software-engineer",
+    )
 
     assert len(result.jobs) == 1
     job = result.jobs[0]
@@ -40,12 +49,24 @@ def test_greenhouse_adapter_detail_supports_changed_markup():
     assert job.external_id == "gh-open-1"
     assert job.url == "https://boards.greenhouse.io/example/jobs/senior-software-engineer"
     assert job.description_html
-    _assert_evidence_fields(job, expected_keys={"external_id", "url", "location_text", "date_posted", "valid_through", "employment_types"})
+    _assert_evidence_fields(
+        job,
+        expected_keys={
+            "external_id",
+            "url",
+            "location_text",
+            "date_posted",
+            "valid_through",
+            "employment_types",
+        },
+    )
 
 
 def test_lever_adapter_listing_extracts_open_jobs_and_skips_closed():
     adapter = LeverAdapter()
-    result = adapter.extract(_load_fixture("fixtures/ats/lever-listing.json"), url="https://jobs.lever.co/example")
+    result = adapter.extract(
+        _load_fixture("fixtures/ats/lever-listing.json"), url="https://jobs.lever.co/example"
+    )
 
     assert result.parser_key == "ats-lever"
     assert len(result.jobs) == 2
@@ -55,7 +76,10 @@ def test_lever_adapter_listing_extracts_open_jobs_and_skips_closed():
 
 def test_lever_adapter_detail_supports_changed_markup():
     adapter = LeverAdapter()
-    result = adapter.extract(_load_fixture("fixtures/ats/lever-detail.json"), url="https://jobs.lever.co/example/platform-engineer")
+    result = adapter.extract(
+        _load_fixture("fixtures/ats/lever-detail.json"),
+        url="https://jobs.lever.co/example/platform-engineer",
+    )
 
     assert len(result.jobs) == 1
     job = result.jobs[0]
@@ -67,7 +91,9 @@ def test_lever_adapter_detail_supports_changed_markup():
 
 def test_recruitee_adapter_listing_extracts_open_jobs_and_skips_closed():
     adapter = RecruiteeAdapter()
-    result = adapter.extract(_load_fixture("fixtures/ats/recruitee-listing.json"), url="https://acme.recruitee.com/o/" )
+    result = adapter.extract(
+        _load_fixture("fixtures/ats/recruitee-listing.json"), url="https://acme.recruitee.com/o/"
+    )
 
     assert result.parser_key == "ats-recruitee"
     assert len(result.jobs) == 2
@@ -76,7 +102,10 @@ def test_recruitee_adapter_listing_extracts_open_jobs_and_skips_closed():
 
 def test_recruitee_adapter_detail_supports_changed_markup():
     adapter = RecruiteeAdapter()
-    result = adapter.extract(_load_fixture("fixtures/ats/recruitee-detail.json"), url="https://acme.recruitee.com/o/system-engineer")
+    result = adapter.extract(
+        _load_fixture("fixtures/ats/recruitee-detail.json"),
+        url="https://acme.recruitee.com/o/system-engineer",
+    )
 
     assert len(result.jobs) == 1
     job = result.jobs[0]
@@ -87,7 +116,10 @@ def test_recruitee_adapter_detail_supports_changed_markup():
 
 def test_smartrecruiters_adapter_listing_extracts_open_jobs_and_skips_closed():
     adapter = SmartRecruitersAdapter()
-    result = adapter.extract(_load_fixture("fixtures/ats/smartrecruiters-listing.json"), url="https://jobs.smartrecruiters.com/example")
+    result = adapter.extract(
+        _load_fixture("fixtures/ats/smartrecruiters-listing.json"),
+        url="https://jobs.smartrecruiters.com/example",
+    )
 
     assert result.parser_key == "ats-smartrecruiters"
     assert len(result.jobs) == 2
@@ -96,7 +128,10 @@ def test_smartrecruiters_adapter_listing_extracts_open_jobs_and_skips_closed():
 
 def test_smartrecruiters_adapter_detail_supports_changed_markup():
     adapter = SmartRecruitersAdapter()
-    result = adapter.extract(_load_fixture("fixtures/ats/smartrecruiters-detail.json"), url="https://jobs.smartrecruiters.com/example/security-engineer")
+    result = adapter.extract(
+        _load_fixture("fixtures/ats/smartrecruiters-detail.json"),
+        url="https://jobs.smartrecruiters.com/example/security-engineer",
+    )
 
     assert len(result.jobs) == 1
     job = result.jobs[0]
@@ -106,7 +141,10 @@ def test_smartrecruiters_adapter_detail_supports_changed_markup():
 
 def test_workable_adapter_listing_extracts_open_jobs_and_skips_closed():
     adapter = WorkableAdapter()
-    result = adapter.extract(_load_fixture("fixtures/ats/workable-listing.json"), url="https://apply.workable.com/example")
+    result = adapter.extract(
+        _load_fixture("fixtures/ats/workable-listing.json"),
+        url="https://apply.workable.com/example",
+    )
 
     assert result.parser_key == "ats-workable"
     assert len(result.jobs) == 2
@@ -115,7 +153,10 @@ def test_workable_adapter_listing_extracts_open_jobs_and_skips_closed():
 
 def test_workable_adapter_detail_supports_changed_markup():
     adapter = WorkableAdapter()
-    result = adapter.extract(_load_fixture("fixtures/ats/workable-detail.json"), url="https://apply.workable.com/example/data-engineer/")
+    result = adapter.extract(
+        _load_fixture("fixtures/ats/workable-detail.json"),
+        url="https://apply.workable.com/example/data-engineer/",
+    )
 
     assert len(result.jobs) == 1
     job = result.jobs[0]

tests/unit/test_discovery_rss.py

@@ -38,12 +38,15 @@ def test_discovery_from_html_includes_jsonld_and_feed_links():
     assert candidate_urls["https://example.org/werken-bij"] == "html"
     assert candidate_urls["https://jobs.example.org/vacatures/"] == "html"
     assert candidate_urls["https://jobs.example.org/vacatures/data-engineer"] == "html-jsonld"
-    assert candidate_urls["https://jobs.example.org/vacatures/data-engineer/solliciteer"] == "html-jsonld"
+    assert (
-    assert candidate_urls["https://example.org/feed/jobs.xml"] == "html"
+        candidate_urls["https://jobs.example.org/vacatures/data-engineer/solliciteer"]
-    assert candidate_urls["https://jobs.example.org/vacatures/data-engineer"].startswith("html-jsonld")
+        == "html-jsonld"
-    assert any(
-        c.source_type == Source.Type.RSS and c.reason == "feed" for c in result
     )
+    assert candidate_urls["https://example.org/feed/jobs.xml"] == "html"
+    assert candidate_urls["https://jobs.example.org/vacatures/data-engineer"].startswith(
+        "html-jsonld"
+    )
+    assert any(c.source_type == Source.Type.RSS and c.reason == "feed" for c in result)
     assert any(
         c.source_type == Source.Type.EMPLOYER and c.url == "https://example.org/werken-bij"
         for c in result
@@ -69,14 +72,16 @@ def test_discovery_from_sitemap_prefers_candidates_and_filters_private_or_denied
     assert len(index_candidates) == 1
     assert index_candidates[0].url == "https://jobs.example.org/sitemap-jobs.xml"
     assert len(urlset_candidates) >= 2
-    assert any(item.url == "https://jobs.example.org/vacatures/data-engineer" for item in urlset_candidates)
+    assert any(
+        item.url == "https://jobs.example.org/vacatures/data-engineer" for item in urlset_candidates
+    )
     assert all("linkedin.com" not in item.url for item in index_candidates + urlset_candidates)
 
 
 def test_discovery_from_email_extracts_employer_domain_root():
     raw = Path("fixtures/emails/sample_alert.eml").read_bytes()
     result = discover_from_email(raw)
-    assert [item.url for item in result] == ["https://jobs.example.org"]
+    assert [item.url for item in result] == ["https://jobs.example.org/"]
     assert result[0].source_type == Source.Type.EMPLOYER
 
 

tests/unit/test_employer_resolution.py

@@ -10,7 +10,7 @@ from apps.jobs.services.normalization import CanonicalJobDraft
 from apps.sources.models import Source
 
 
-def _draft_for_test(job: JobPosting, *, **overrides) -> CanonicalJobDraft:
+def _draft_for_test(job: JobPosting, **overrides) -> CanonicalJobDraft:
     values = {
         "source_url": "https://jobs.smartrecruiters.com/example/security-engineer",
         "canonical_url": "https://jobs.smartrecruiters.com/example/security-engineer",

tests/unit/test_feedback_learning.py

@@ -27,7 +27,9 @@ def test_learning_skip_for_implicit_hide(profile, job, user):
 
 
 def test_learning_marks_explicit_title_hide_as_nonlearning(profile, job, user):
-    feedback = record_feedback(user=user, job=job, action=Feedback.Action.HIDE, reason="Titel past niet bij mij")
+    feedback = record_feedback(
+        user=user, job=job, action=Feedback.Action.HIDE, reason="Titel past niet bij mij"
+    )
     metadata = feedback.metadata.get("learning", {})
     assert metadata["status"] == "queued"
     assert metadata["reason_code"] == "non_learning_title"

tests/unit/test_fetcher.py

@@ -6,8 +6,8 @@ from apps.sources.services.fetcher import (
     ContentRejectedError,
     FetchError,
     FetchTimeoutError,
-    RateLimitedError,
     PolicyBlockedError,
+    RateLimitedError,
     fetch_url,
 )
 from apps.sources.services.url_security import ValidatedUrl
@@ -79,7 +79,9 @@ def test_fetcher_rejects_dns_rebinding(monkeypatch):
 
     def validate(url: str, **kwargs):
         if "/final" in url:
-            return ValidatedUrl(url=url, hostname="example.org", port=443, addresses=("198.51.100.12",))
+            return ValidatedUrl(
+                url=url, hostname="example.org", port=443, addresses=("198.51.100.12",)
+            )
         return ValidatedUrl(url=url, hostname="example.org", port=443, addresses=("93.184.216.34",))
 
     monkeypatch.setattr("apps.sources.services.fetcher.validate_public_url", validate)
@@ -133,7 +135,9 @@ def test_fetcher_rejects_policy_content_size_and_http(monkeypatch):
     client.close()
 
     client = httpx.Client(
-        transport=httpx.MockTransport(lambda request: (_ for _ in ()).throw(httpx.TimeoutException("timeout")))
+        transport=httpx.MockTransport(
+            lambda request: (_ for _ in ()).throw(httpx.TimeoutException("timeout"))
+        )
     )
     with pytest.raises(FetchTimeoutError):
         fetch_url("https://example.org/slow", client=client)

tests/unit/test_geocoding.py

@@ -127,8 +127,7 @@ def test_validate_csv_geodata_flags_duplicates_and_invalid_coordinates(tmp_path:
         validate_csv_geodata(source)
 
     source.write_text(
-        "postal_code,municipality,region,latitude,longitude\n"
+        "postal_code,municipality,region,latitude,longitude\n9000,Gent,Vlaams-Brabant,99,3.73\n"
-        "9000,Gent,Vlaams-Brabant,99,3.73\n"
     )
     with pytest.raises(CommandError, match="latitude buiten bereik"):
         validate_csv_geodata(source)

tests/unit/test_manual_import_service.py

@@ -34,9 +34,7 @@ def test_manual_import_url_uses_fetch_and_succeeds(monkeypatch, user):
     monkeypatch.setattr("apps.sources.services.manual_import.fetch_url", fake_fetch)
     monkeypatch.setattr("apps.sources.services.manual_import.process_raw_document", fake_process)
 
-    summary = import_manual_source(
+    summary = import_manual_source(actor=user, source_url="https://jobs.example.org/jobs")
-        actor=user, source_url="https://jobs.example.org/jobs"
-    )
     source = Source.objects.get(pk=summary.source_id)
 
     assert summary.mode == "url"

tests/unit/test_scheduling.py

@@ -19,7 +19,11 @@ def test_source_lease_blocks_concurrent_workers_and_recovers_when_expired(source
     assert acquire_source_lease(source_id=source.pk, worker_token="worker-b", now=now) is None
 
     assert release_source_lease(source_id=source.pk, worker_token="worker-a", now=now)
-    lease_b = acquire_source_lease(source_id=source.pk, worker_token="worker-b", now=timezone.now())
+    lease_b = acquire_source_lease(
+        source_id=source.pk,
+        worker_token="worker-b",
+        now=now + timedelta(seconds=source.minimum_interval_seconds),
+    )
     assert lease_b is not None
 
 
@@ -51,7 +55,7 @@ def test_source_lease_allows_origin_concurrency(db):
     )
     source_b = Source.objects.create(
         name="Source B",
-        source_type=Source.Type.EMPLOYER,
+        source_type=Source.Type.RSS,
         base_url="https://shared.example.org/careers/",
         domain=shared_domain,
         status=Source.Status.ACTIVE,
@@ -76,8 +80,6 @@ def test_failure_backoff_uses_retry_after_and_bounds():
         policy=Source.Policy.ALLOW,
     )
     source.save()
-    backoff = calculate_failure_backoff_seconds(
+    backoff = calculate_failure_backoff_seconds(source, failure_count=2, retry_after_seconds=120)
-        source, failure_count=2, retry_after_seconds=120
-    )
     assert backoff >= 120
     assert backoff <= 3600 + 45

tests/unit/test_scoring.py

@@ -4,10 +4,9 @@ from decimal import Decimal
 
 import pytest
 
-from apps.jobs.models import AiAnalysisCache
+from apps.jobs.models import AiAnalysisCache, Employer, JobPosting, ScoreRun
-from apps.jobs.services.ai import AiAnalysis
-from apps.jobs.models import Employer, JobPosting, ScoreRun
 from apps.jobs.services import scoring
+from apps.jobs.services.ai import AiAnalysis
 from apps.jobs.services.scoring import calculate_score
 
 
@@ -78,10 +77,12 @@ def test_distance_boundary_inclusief(matching_job, profile, monkeypatch):
 
 def test_remote_job_heeft_geen_afstandsexclusie(matching_job, profile):
     matching_job.workplace_type = JobPosting.Workplace.REMOTE
-    matching_job.postal_code = None
+    matching_job.postal_code = ""
-    matching_job.municipality = None
+    matching_job.municipality = ""
-    matching_job.raw_location = None
+    matching_job.raw_location = ""
-    matching_job.save(update_fields=["workplace_type", "postal_code", "municipality", "raw_location"])
+    matching_job.save(
+        update_fields=["workplace_type", "postal_code", "municipality", "raw_location"]
+    )
     matching_job.latitude = None
     matching_job.longitude = None
     result = calculate_score(matching_job, profile)
@@ -90,12 +91,14 @@ def test_remote_job_heeft_geen_afstandsexclusie(matching_job, profile):
 
 
 def test_onbekende_locatie_leidt_niet_tot_automatische_uitsluiting(matching_job, profile):
-    matching_job.postal_code = None
+    matching_job.postal_code = ""
-    matching_job.municipality = None
+    matching_job.municipality = ""
     matching_job.raw_location = "onbekend"
     matching_job.latitude = None
     matching_job.longitude = None
-    matching_job.save(update_fields=["postal_code", "municipality", "raw_location", "latitude", "longitude"])
+    matching_job.save(
+        update_fields=["postal_code", "municipality", "raw_location", "latitude", "longitude"]
+    )
     result = calculate_score(matching_job, profile)
     assert result.evidence["distance_km"] is None
     assert result.recommendation != ScoreRun.Recommendation.HIDDEN
@@ -162,7 +165,10 @@ def test_ai_score_influence_is_opt_in_and_bounded(matching_job, profile, monkeyp
     )
 
     result = calculate_score(matching_job, profile)
-    assert result.recommendation in {ScoreRun.Recommendation.STRONG, ScoreRun.Recommendation.POSSIBLE}
+    assert result.recommendation in {
+        ScoreRun.Recommendation.STRONG,
+        ScoreRun.Recommendation.POSSIBLE,
+    }
     assert result.model_version == "local-model"
     assert result.evidence["ai"]["status"] == AiAnalysisCache.Status.OK
     assert result.evidence["ai"]["weight_applied"] == 20.0

tests/unit/test_settings_security.py

@@ -19,7 +19,14 @@ def test_validate_production_security_skips_when_debug_mode_is_enabled():
 
 
 @pytest.mark.parametrize(
-    "secret_key, allowed_hosts, csrf_trusted_origins, session_cookie_secure, csrf_cookie_secure, secure_ssl_redirect",
+    (
+        "secret_key",
+        "allowed_hosts",
+        "csrf_trusted_origins",
+        "session_cookie_secure",
+        "csrf_cookie_secure",
+        "secure_ssl_redirect",
+    ),
     [
         ("", ["example.org"], ["https://example.org"], True, True, True),
         ("change-me", ["example.org"], ["https://example.org"], True, True, True),