Initial deploy setup

apps/sources/services/canonicalize.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import posixpath
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+TRACKING_PARAMETERS = {
+    "fbclid",
+    "gclid",
+    "mc_cid",
+    "mc_eid",
+    "ref",
+    "referrer",
+    "source",
+    "trk",
+    "trackingid",
+}
+TRACKING_PREFIXES = ("utm_", "pk_")
+
+
+def canonicalize_url(url: str) -> str:
+    value = (url or "").strip()
+    if not value:
+        return ""
+    parts = urlsplit(value)
+    scheme = parts.scheme.lower()
+    host = (parts.hostname or "").lower().rstrip(".")
+    if not scheme or not host:
+        return value
+    port = parts.port
+    if port and not ((scheme == "http" and port == 80) or (scheme == "https" and port == 443)):
+        netloc = f"{host}:{port}"
+    else:
+        netloc = host
+    path = parts.path or "/"
+    normalized_path = posixpath.normpath(path)
+    if path.endswith("/") and not normalized_path.endswith("/"):
+        normalized_path += "/"
+    if not normalized_path.startswith("/"):
+        normalized_path = "/" + normalized_path
+    query_pairs = []
+    for key, value in parse_qsl(parts.query, keep_blank_values=True):
+        lower = key.lower()
+        if lower in TRACKING_PARAMETERS or any(
+            lower.startswith(prefix) for prefix in TRACKING_PREFIXES
+        ):
+            continue
+        query_pairs.append((key, value))
+    query_pairs.sort()
+    return urlunsplit((scheme, netloc, normalized_path, urlencode(query_pairs, doseq=True), ""))
+
+
+def domain_matches(hostname: str, domain: str) -> bool:
+    host = hostname.lower().rstrip(".")
+    target = domain.lower().rstrip(".")
+    return host == target or host.endswith("." + target)

apps/sources/services/discovery.py

@@ -0,0 +1,391 @@
+from __future__ import annotations
+
+import ipaddress
+import json
+import re
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from urllib.parse import urljoin, urlsplit
+from xml.etree import ElementTree
+
+from bs4 import BeautifulSoup
+from django.db import transaction
+
+from apps.sources.adapters.email_alert import EmailAlertAdapter
+from apps.sources.adapters.rss import RssAdapter
+from apps.sources.models import Source
+
+from .canonicalize import canonicalize_url, domain_matches
+from .policy import is_denied_domain
+
+CAREER_PATTERN = re.compile(
+    r"\b(career|careers|jobs|job|vacature|vacatures|werken-bij|werken bij|emploi|emplois|offres)\b",
+    re.I,
+)
+FEED_TYPE_PATTERN = re.compile(r"application/(?:atom|rss)\+xml|text/xml|application/xml", re.I)
+
+
+@dataclass(frozen=True)
+class SourceCandidate:
+    url: str
+    domain: str
+    source_type: str
+    label: str
+    confidence: float
+    reason: str
+    discovered_from: str
+
+
+def discover_career_links(html: str, *, base_url: str) -> list[SourceCandidate]:
+    return _discover_html(
+        html,
+        base_url=base_url,
+        include_jsonld=False,
+        include_feed_links=False,
+    )
+
+
+def discover_from_html(html: str, *, base_url: str) -> list[SourceCandidate]:
+    return _discover_html(
+        html,
+        base_url=base_url,
+        include_jsonld=True,
+        include_feed_links=True,
+    )
+
+
+def discover_from_feed(feed_content: str, *, base_url: str) -> list[SourceCandidate]:
+    adapter = RssAdapter()
+    result = adapter.extract(feed_content, url=base_url)
+    base_domain = _hostname(base_url)
+    candidates: list[SourceCandidate] = []
+    for extracted in result.jobs:
+        candidate = _build_candidate(
+            raw_url=extracted.url,
+            base_domain=base_domain,
+            source_type=Source.Type.RSS,
+            label=extracted.title[:120],
+            reason="rss",
+            discovered_from="feed",
+            confidence=0.82,
+            allow_off_domain=False,
+        )
+        if candidate:
+            candidates.append(candidate)
+    return _dedupe(candidates)
+
+
+def discover_from_sitemap(xml_content: str, *, base_url: str) -> list[SourceCandidate]:
+    try:
+        root = ElementTree.fromstring(xml_content)
+    except ElementTree.ParseError:
+        return []
+
+    root_name = _local_tag(root.tag)
+    if root_name == "sitemapindex":
+        discovered_from = "sitemap-index"
+    elif root_name == "urlset":
+        discovered_from = "sitemap-urlset"
+    else:
+        return []
+
+    base_domain = _hostname(base_url)
+    candidates: list[SourceCandidate] = []
+    for location in root.findall(".//{*}loc"):
+        raw = (location.text or "").strip()
+        if not raw:
+            continue
+        candidate = _build_candidate(
+            raw_url=urljoin(base_url, raw),
+            base_domain=base_domain,
+            source_type=Source.Type.SITEMAP,
+            label="Sitemaplocatie",
+            reason=discovered_from,
+            discovered_from=discovered_from,
+            confidence=0.86 if discovered_from == "sitemap-urlset" else 0.75,
+            allow_off_domain=False,
+        )
+        if candidate:
+            candidates.append(candidate)
+
+    return _dedupe(candidates)
+
+
+def discover_from_email(raw_message: bytes) -> list[SourceCandidate]:
+    adapter = EmailAlertAdapter()
+    result = adapter.extract_message(raw_message)
+    candidates: list[SourceCandidate] = []
+    for extracted in result.jobs:
+        candidate_url = _to_domain_root_url(extracted.url)
+        candidate = _build_candidate(
+            raw_url=candidate_url,
+            base_domain=_hostname(extracted.url),
+            source_type=Source.Type.EMPLOYER,
+            label=extracted.title[:120],
+            reason="email",
+            discovered_from="email",
+            confidence=0.62,
+            allow_off_domain=True,
+        )
+        if candidate:
+            candidates.append(candidate)
+
+    return _dedupe(candidates)
+
+
+def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int, int, int]:
+    created = updated = skipped = 0
+    ordered = _dedupe(candidates)
+    with transaction.atomic():
+        for candidate in ordered:
+            provenance = _provenance_entry(candidate)
+            now = _utc_now()
+            source, was_created = Source.objects.get_or_create(
+                domain=candidate.domain,
+                source_type=candidate.source_type,
+                defaults={
+                    "name": _candidate_name(candidate),
+                    "base_url": candidate.url,
+                    "status": Source.Status.CANDIDATE,
+                    "policy": Source.Policy.REVIEW,
+                    "policy_reason": "Automatisch ontdekt",
+                    "parser_key": "auto",
+                    "strict_mode": True,
+                    "metadata": {
+                        "discovery": [provenance],
+                        "discovered_at": now,
+                    },
+                },
+            )
+            if was_created:
+                created += 1
+                continue
+
+            updated_fields: list[str] = ["updated_at"]
+            metadata = source.metadata if isinstance(source.metadata, dict) else {}
+            evidence = metadata.get("discovery", [])
+            if not isinstance(evidence, list):
+                evidence = []
+            if not any(item.get("url") == candidate.url for item in evidence if isinstance(item, dict)):
+                evidence.append(provenance)
+                metadata["discovery"] = evidence[-20:]
+                metadata["discovered_at"] = now
+                source.metadata = metadata
+                updated_fields.append("metadata")
+            else:
+                skipped += 1
+
+            if not source.name:
+                source.name = _candidate_name(candidate)
+                updated_fields.append("name")
+            if not source.base_url:
+                source.base_url = candidate.url
+                updated_fields.append("base_url")
+
+            if len(updated_fields) > 1:
+                source.save(update_fields=sorted(set(updated_fields)))
+                updated += 1
+            else:
+                skipped += 1
+
+    return (created, updated, skipped)
+
+
+def _candidate_name(candidate: SourceCandidate) -> str:
+    label = candidate.label.strip()
+    if label:
+        return label
+    return candidate.domain
+
+
+def _provenance_entry(candidate: SourceCandidate) -> dict[str, str | float]:
+    return {
+        "url": candidate.url,
+        "source_type": candidate.source_type,
+        "discovered_from": candidate.discovered_from,
+        "confidence": candidate.confidence,
+        "reason": candidate.reason,
+        "label": candidate.label[:240],
+        "seen_at": _utc_now(),
+    }
+
+
+def _discover_html(
+    html: str,
+    *,
+    base_url: str,
+    include_jsonld: bool,
+    include_feed_links: bool,
+) -> list[SourceCandidate]:
+    soup = BeautifulSoup(html, "lxml")
+    base_domain = _hostname(base_url)
+    candidates: list[SourceCandidate] = []
+
+    for anchor in soup.find_all("a", href=True):
+        label = " ".join(anchor.get_text(" ", strip=True).split())
+        candidate = _build_candidate(
+            raw_url=urljoin(base_url, str(anchor["href"])),
+            base_domain=base_domain,
+            source_type=Source.Type.EMPLOYER,
+            label=label,
+            reason="career-link",
+            discovered_from="html",
+            confidence=0.9,
+            allow_off_domain=False,
+        )
+        if not candidate:
+            continue
+        if not CAREER_PATTERN.search(label) and not CAREER_PATTERN.search(urlsplit(candidate.url).path):
+            continue
+        candidates.append(candidate)
+
+    if include_jsonld:
+        for js in soup.find_all("script", type=re.compile(r"application/ld\+json", re.I)):
+            raw = js.get_text("", strip=True)
+            if not raw:
+                continue
+            try:
+                parsed = json.loads(raw)
+            except json.JSONDecodeError:
+                continue
+            for url in _collect_jsonld_urls(parsed):
+                candidate = _build_candidate(
+                    raw_url=urljoin(base_url, url),
+                    base_domain=base_domain,
+                    source_type=Source.Type.EMPLOYER,
+                    label="JSON-LD job",
+                    reason="jsonld",
+                    discovered_from="html-jsonld",
+                    confidence=0.95,
+                    allow_off_domain=False,
+                )
+                if candidate:
+                    candidates.append(candidate)
+
+    if include_feed_links:
+        for link in soup.find_all("link", href=True):
+            rel = {str(item).lower() for item in (link.get("rel") or [])}
+            if "alternate" not in rel:
+                continue
+            link_type = str(link.get("type") or "")
+            if not FEED_TYPE_PATTERN.search(link_type):
+                continue
+            candidate = _build_candidate(
+                raw_url=urljoin(base_url, str(link["href"])),
+                base_domain=base_domain,
+                source_type=Source.Type.RSS,
+                label=(str(link.get("title") or "Feedlink")).strip()[:120],
+                reason="feed",
+                discovered_from="html",
+                confidence=0.86,
+                allow_off_domain=False,
+            )
+            if candidate:
+                candidates.append(candidate)
+
+    return _dedupe(candidates)
+
+
+def _collect_jsonld_urls(payload) -> list[str]:
+    urls: list[str] = []
+
+    def walk(node):
+        if isinstance(node, dict):
+            candidate_type = str(node.get("@type", "")).lower()
+            if candidate_type == "jobposting":
+                for key in ("url", "applyUrl", "application", "applicationurl"):
+                    value = node.get(key)
+                    if isinstance(value, str) and value:
+                        urls.append(value)
+            for key in ("@graph", "itemListElement", "item", "jobs", "jobPosting"):
+                nested = node.get(key)
+                if nested is not None:
+                    walk(nested)
+        elif isinstance(node, list):
+            for item in node:
+                walk(item)
+
+    walk(payload)
+    return urls
+
+
+def _build_candidate(
+    *,
+    raw_url: str,
+    base_domain: str,
+    source_type: str,
+    label: str,
+    reason: str,
+    discovered_from: str,
+    confidence: float,
+    allow_off_domain: bool,
+) -> SourceCandidate | None:
+    canonical = canonicalize_url(raw_url)
+    if not canonical:
+        return None
+    if not _url_is_http(canonical):
+        return None
+    hostname = _hostname(canonical)
+    if not hostname:
+        return None
+    if is_denied_domain(hostname):
+        return None
+    if _is_private_host(hostname):
+        return None
+    if (not allow_off_domain) and base_domain and not domain_matches(hostname, base_domain):
+        return None
+    return SourceCandidate(
+        url=canonical,
+        domain=hostname,
+        source_type=source_type,
+        label=label,
+        confidence=confidence,
+        reason=reason,
+        discovered_from=discovered_from,
+    )
+
+
+def _url_is_http(url: str) -> bool:
+    return urlsplit(url).scheme.lower() in {"http", "https"}
+
+
+def _hostname(value: str) -> str:
+    return (urlsplit(value).hostname or "").lower()
+
+
+def _is_private_host(hostname: str) -> bool:
+    host = hostname.lower().rstrip(".")
+    if host in {"localhost", "127.0.0.1", "::1"} or host.endswith((".local", ".localhost")):
+        return True
+    try:
+        ip = ipaddress.ip_address(host)
+    except ValueError:
+        return False
+    return not ip.is_global
+
+
+def _dedupe(candidates: list[SourceCandidate]) -> list[SourceCandidate]:
+    seen: set[tuple[str, str]] = set()
+    result: list[SourceCandidate] = []
+    for candidate in sorted(candidates, key=lambda item: item.confidence, reverse=True):
+        key = (candidate.url, candidate.source_type)
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(candidate)
+    return result
+
+
+def _local_tag(tag_name: str) -> str:
+    return tag_name.rsplit("}", 1)[-1].lower()
+
+
+def _to_domain_root_url(url: str) -> str:
+    parsed = urlsplit(url)
+    if not parsed.scheme or not parsed.hostname:
+        return url
+    return f"{parsed.scheme}://{parsed.hostname}"
+
+
+def _utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()

apps/sources/services/email_import.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import timedelta
+from email import policy
+from email.parser import BytesParser
+from email.utils import parsedate_to_datetime
+
+from django.conf import settings
+from django.db import transaction
+from django.utils import timezone
+
+from apps.jobs.services.normalization import normalize_extracted_job
+from apps.jobs.services.pipeline import persist_draft
+from apps.sources.adapters.email_alert import EmailAlertAdapter
+from apps.sources.models import EmailMessageRecord, RawDocument, Source
+
+
+def message_identity(raw_message: bytes) -> str:
+    """Return the RFC Message-ID or a deterministic hash when it is absent."""
+    parsed = BytesParser(policy=policy.default).parsebytes(raw_message, headersonly=True)
+    fallback_id = hashlib.sha256(raw_message).hexdigest()
+    return str(parsed.get("message-id") or f"sha256:{fallback_id}").strip()
+
+
+@transaction.atomic
+def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageRecord:
+    parsed = BytesParser(policy=policy.default).parsebytes(raw_message)
+    message_id = message_identity(raw_message)
+    existing = EmailMessageRecord.objects.filter(message_id=message_id).first()
+    if existing:
+        return existing
+
+    source, _ = Source.objects.get_or_create(
+        domain="mailbox.local",
+        source_type=Source.Type.EMAIL,
+        defaults={
+            "name": "Vacaturemailbox",
+            "status": Source.Status.ACTIVE,
+            "policy": Source.Policy.ALLOW,
+            "parser_key": "email-alert",
+            "crawl_interval_minutes": 10,
+        },
+    )
+    content_hash = hashlib.sha256(raw_message).hexdigest()
+    retain_until = timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS)
+    document = RawDocument.objects.create(
+        source=source,
+        kind=RawDocument.Kind.EMAIL,
+        content_type="message/rfc822",
+        content_hash=content_hash,
+        body_text=raw_message.decode("utf-8", errors="replace"),
+        byte_length=len(raw_message),
+        retain_until=retain_until,
+        metadata={"mailbox": mailbox},
+    )
+    received_at = None
+    if parsed.get("date"):
+        try:
+            received_at = parsedate_to_datetime(str(parsed.get("date")))
+            if timezone.is_naive(received_at):
+                received_at = timezone.make_aware(received_at, timezone.get_current_timezone())
+        except (TypeError, ValueError, OverflowError):
+            received_at = None
+
+    adapter = EmailAlertAdapter()
+    result = adapter.extract_message(raw_message)
+    document.parser_key = result.parser_key
+    document.parser_version = result.parser_version
+    document.extraction_confidence = result.confidence
+    document.save(
+        update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"]
+    )
+    links: list[str] = []
+    errors: list[str] = []
+    for extracted in result.jobs:
+        links.append(extracted.url)
+        try:
+            draft = normalize_extracted_job(extracted)
+            persist_draft(
+                draft,
+                document=document,
+                parser_key=result.parser_key,
+                parser_version=result.parser_version,
+                extraction_confidence=result.confidence,
+            )
+        except Exception as exc:
+            errors.append(f"{exc.__class__.__name__}: {exc}")
+    return EmailMessageRecord.objects.create(
+        message_id=message_id,
+        mailbox=mailbox,
+        sender=str(parsed.get("from") or "")[:500],
+        subject=str(parsed.get("subject") or "")[:998],
+        received_at=received_at,
+        raw_document=document,
+        links=links,
+        processed=not errors,
+        error_message="; ".join(errors)[:1000],
+    )

apps/sources/services/fetcher.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import datetime, timezone as utc
+from email.utils import parsedate_to_datetime
+from collections.abc import Mapping
+from dataclasses import dataclass
+from urllib.parse import urljoin
+
+import httpx
+from django.conf import settings
+
+from apps.sources.models import Source
+
+from .policy import assess_url
+from .url_security import validate_public_url
+
+ALLOWED_CONTENT_TYPES = (
+    "text/html",
+    "application/xhtml+xml",
+    "application/xml",
+    "text/xml",
+    "application/json",
+    "application/ld+json",
+    "text/plain",
+    "application/rss+xml",
+    "application/atom+xml",
+)
+
+
+class FetchError(RuntimeError):
+    pass
+
+
+class PolicyBlockedError(FetchError):
+    pass
+
+
+class ContentRejectedError(FetchError):
+    pass
+
+
+class RateLimitedError(FetchError):
+    def __init__(self, retry_after_seconds: int | None, message: str = "HTTP 429") -> None:
+        self.retry_after_seconds = retry_after_seconds
+        suffix = f"; Retry-After={retry_after_seconds}s" if retry_after_seconds else ""
+        super().__init__(f"{message}{suffix}".strip())
+
+
+class FetchTimeoutError(FetchError):
+    pass
+
+
+def parse_retry_after(value: str | None) -> int | None:
+    if not value:
+        return None
+    value = value.strip()
+    if not value:
+        return None
+    try:
+        return max(0, int(value))
+    except ValueError:
+        pass
+    try:
+        retry_datetime = parsedate_to_datetime(value)
+    except (TypeError, ValueError):
+        return None
+    if retry_datetime is None:
+        return None
+    if retry_datetime.tzinfo is None:
+        retry_datetime = retry_datetime.replace(tzinfo=utc)
+    retry_aware = retry_datetime.astimezone(utc)
+    now = datetime.now(utc)
+    delta = (retry_aware - now).total_seconds()
+    return max(0, int(delta))
+
+
+@dataclass(slots=True)
+class FetchedDocument:
+    requested_url: str
+    final_url: str
+    status_code: int
+    headers: dict[str, str]
+    content: bytes
+
+    @property
+    def text(self) -> str:
+        encoding = "utf-8"
+        content_type = self.headers.get("content-type", "")
+        if "charset=" in content_type:
+            encoding = content_type.split("charset=", 1)[1].split(";", 1)[0].strip()
+        return self.content.decode(encoding, errors="replace")
+
+    @property
+    def sha256(self) -> str:
+        return hashlib.sha256(self.content).hexdigest()
+
+
+def fetch_url(
+    url: str,
+    *,
+    source: Source | None = None,
+    conditional_headers: Mapping[str, str] | None = None,
+    client: httpx.Client | None = None,
+) -> FetchedDocument:
+    decision = assess_url(url, source=source)
+    if not decision.allowed:
+        raise PolicyBlockedError(decision.reason)
+
+    headers = {
+        "User-Agent": settings.FETCHER_USER_AGENT,
+        "Accept": (
+            "text/html,application/xhtml+xml,application/xml,application/json,"
+            "text/plain;q=0.8,*/*;q=0.1"
+        ),
+    }
+    if conditional_headers:
+        headers.update(conditional_headers)
+
+    own_client = client is None
+    http_client = client or httpx.Client(
+        timeout=httpx.Timeout(settings.FETCHER_TIMEOUT_SECONDS),
+        follow_redirects=False,
+        headers=headers,
+    )
+    current_url = url
+    try:
+        for _ in range(settings.FETCHER_MAX_REDIRECTS + 1):
+            validation = validate_public_url(
+                current_url,
+                allow_nonstandard_ports=settings.FETCHER_ALLOW_NONSTANDARD_PORTS,
+            )
+            response = http_client.get(current_url, headers=headers)
+            post_validation = validate_public_url(
+                str(response.url) if response.url else current_url,
+                allow_nonstandard_ports=settings.FETCHER_ALLOW_NONSTANDARD_PORTS,
+            )
+            if not set(validation.addresses).intersection(set(post_validation.addresses)):
+                raise FetchError("DNS-rebindcontrole faalde bij het benaderen van bron.")
+            if response.status_code in {301, 302, 303, 307, 308}:
+                location = response.headers.get("location")
+                if not location:
+                    raise FetchError("Redirect zonder Location-header.")
+                current_url = urljoin(current_url, location)
+                next_decision = assess_url(current_url, source=source)
+                if not next_decision.allowed:
+                    raise PolicyBlockedError(next_decision.reason)
+                continue
+            if response.status_code == 304:
+                return FetchedDocument(url, current_url, 304, dict(response.headers), b"")
+            if response.status_code == 429:
+                retry_after = parse_retry_after(response.headers.get("retry-after"))
+                raise RateLimitedError(retry_after)
+            if response.status_code >= 400:
+                raise FetchError(f"HTTP {response.status_code}")
+            content_type = response.headers.get("content-type", "").lower()
+            if content_type and not any(
+                allowed in content_type for allowed in ALLOWED_CONTENT_TYPES
+            ):
+                raise ContentRejectedError(f"Content-Type niet toegestaan: {content_type}")
+            content = response.content
+            if len(content) > settings.FETCHER_MAX_BYTES:
+                raise ContentRejectedError("Document overschrijdt de ingestelde groottebeperking.")
+            return FetchedDocument(
+                url, current_url, response.status_code, dict(response.headers), content
+            )
+        raise FetchError("Te veel redirects.")
+    except httpx.TimeoutException as exc:
+        raise FetchTimeoutError("Timeout tijdens HTTP-opvraag") from exc
+    except httpx.RequestError as exc:
+        raise FetchError(f"Netwerkfout tijdens HTTP-opvraag: {exc}") from exc
+    finally:
+        if own_client:
+            http_client.close()

apps/sources/services/health.py

@@ -0,0 +1,349 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+from django.utils import timezone
+
+from apps.sources.models import Source, SourcePolicyReview, SourceRun
+
+from .policy import create_policy_review
+
+
+@dataclass(frozen=True)
+class SourceHealth:
+    source_id: int
+    source_name: str
+    source_type: str
+    source_status: str
+    monitored_runs: int
+    success_ratio: float
+    avg_latency_ms: float | None
+    error_counts: dict[str, int]
+    http_status_counts: dict[str, int]
+    extracted_count: int
+    updated_count: int
+    duplicate_count: int
+    last_parser: str | None
+    last_parser_warnings: int
+    last_health_action: str | None
+    last_health_reason: str | None
+
+
+SOURCE_HEALTH_RUN_WINDOW = 30
+SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE = 6
+SOURCE_HEALTH_TEMPORARY_ERROR_RATIO = 0.7
+SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK = 3
+SOURCE_HEALTH_PARSER_MIN_WARNINGS = 2
+SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX = 0
+SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS = 12
+
+TEMPORARY_ERROR_CATEGORIES = {
+    "timeout",
+    "rate_limited",
+    "FetchTimeoutError",
+    "FetchError",
+    "unexpected",
+    "NetworkError",
+}
+POLICY_ERROR_CATEGORIES = {"policy"}
+
+
+def _normalize_metrics(raw: object) -> dict[str, object]:
+    if isinstance(raw, dict):
+        return raw
+    return {}
+
+
+def _warnings_from_run(run: SourceRun) -> list[str]:
+    metrics = _normalize_metrics(run.metrics)
+    warnings = metrics.get("warnings", [])
+    if not isinstance(warnings, list):
+        return []
+    return [str(item) for item in warnings if isinstance(item, str)]
+
+
+def _parser_from_run(run: SourceRun) -> str | None:
+    metrics = _normalize_metrics(run.metrics)
+    parser = metrics.get("parser")
+    if isinstance(parser, str) and parser:
+        return parser
+    return None
+
+
+def _int(value: object) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
+def _float(value: object) -> float:
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def _to_iso(dt: datetime | None) -> str | None:
+    if dt is None:
+        return None
+    return dt.isoformat()
+
+
+def _from_iso(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        dt = datetime.fromisoformat(value)
+    except ValueError:
+        return None
+    if timezone.is_naive(dt):
+        return timezone.make_aware(dt)
+    return dt
+
+
+def _health_metadata(source: Source) -> dict[str, object]:
+    metadata = source.metadata
+    if not isinstance(metadata, dict):
+        return {}
+    health = metadata.get("source_health")
+    return health if isinstance(health, dict) else {}
+
+
+def _set_health_metadata(source: Source, health_data: dict[str, object]) -> None:
+    metadata = source.metadata
+    if not isinstance(metadata, dict):
+        metadata = {}
+    metadata["source_health"] = health_data
+    source.metadata = metadata
+    source.save(update_fields=["metadata", "updated_at"])
+
+
+def _run_counts(runs: Iterable[SourceRun]) -> tuple[dict[str, int], dict[str, int]]:
+    error_counts: dict[str, int] = {}
+    http_status_counts: dict[str, int] = {}
+    for run in runs:
+        if run.status != SourceRun.Status.SUCCESS:
+            category = run.error_category or "unknown"
+            error_counts[category] = error_counts.get(category, 0) + 1
+        if run.http_status:
+            status = str(run.http_status)
+            http_status_counts[status] = http_status_counts.get(status, 0) + 1
+    return error_counts, http_status_counts
+
+
+def _last_parser_output(runs: list[SourceRun]) -> tuple[str | None, int]:
+    for run in runs:
+        if run.status != SourceRun.Status.SUCCESS:
+            continue
+        parser = _parser_from_run(run)
+        if parser:
+            warnings = _warnings_from_run(run)
+            return parser, len(warnings)
+    return None, 0
+
+
+def _latency_ms(runs: list[SourceRun]) -> float | None:
+    latencies = []
+    for run in runs:
+        if run.finished_at is None or run.started_at is None:
+            continue
+        latencies.append(max(0.0, (run.finished_at - run.started_at).total_seconds() * 1000))
+    if not latencies:
+        return None
+    return sum(latencies) / len(latencies)
+
+
+def _is_parser_drift_run(run: SourceRun) -> bool:
+    if run.status != SourceRun.Status.SUCCESS:
+        return False
+    if _int(run.extracted_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
+        return False
+    if _int(run.created_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
+        return False
+    if _int(run.updated_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
+        return False
+    if _parser_from_run(run) in {None, "not-modified"}:
+        return False
+    warnings = _warnings_from_run(run)
+    return len(warnings) >= SOURCE_HEALTH_PARSER_MIN_WARNINGS
+
+
+def _determine_health_action(runs: list[SourceRun]) -> tuple[str | None, str | None]:
+    recent_runs = runs[:SOURCE_HEALTH_RUN_WINDOW]
+    considered_failures = [
+        run for run in recent_runs if run.status in {SourceRun.Status.FAILED, SourceRun.Status.SKIPPED}
+    ]
+
+    if any(
+        run.error_category in POLICY_ERROR_CATEGORIES
+        for run in considered_failures[:3]
+        if run.error_category
+    ):
+        return "quarantine", "Herhaald beleid-/securityprobleem in bronruns."
+
+    if len(considered_failures) >= SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE:
+        temporary_count = sum(
+            1
+            for run in considered_failures
+            if (run.error_category or "") in TEMPORARY_ERROR_CATEGORIES
+        )
+        ratio = _float(temporary_count) / _float(len(considered_failures))
+        if ratio >= SOURCE_HEALTH_TEMPORARY_ERROR_RATIO:
+            return "quarantine", "Herhaald tijdelijk foutgedrag tijdens bronruns."
+
+    streak = 0
+    for run in recent_runs:
+        if _is_parser_drift_run(run):
+            streak += 1
+            if streak >= SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK:
+                return (
+                    "quarantine",
+                    "Parserdrift vermoed: opeenvolgende succesvolle runs met minimale output.",
+                )
+            continue
+        streak = 0
+
+    return None, None
+
+
+def collect_source_health(*, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW) -> list[SourceHealth]:
+    sources = Source.objects.order_by("name").all()
+    rows: list[SourceHealth] = []
+
+    for source in sources:
+        run_queryset = SourceRun.objects.filter(source=source).order_by("-started_at")
+        runs = list(run_queryset[:runs_to_consider])
+        monitored_runs = len(runs)
+        success_runs = [run for run in runs if run.status == SourceRun.Status.SUCCESS]
+
+        success_ratio = _float(len(success_runs) / monitored_runs) if monitored_runs else 0.0
+        avg_latency_ms = _latency_ms(runs)
+        error_counts, http_status_counts = _run_counts(runs)
+
+        extracted_count = sum(_int(run.extracted_count) for run in runs)
+        updated_count = sum(_int(run.updated_count) for run in runs)
+        duplicate_count = sum(_int(run.duplicate_count) for run in runs)
+
+        last_parser, last_warnings = _last_parser_output(runs)
+        action, reason = _determine_health_action(runs)
+
+        rows.append(
+            SourceHealth(
+                source_id=source.pk,
+                source_name=source.name,
+                source_type=source.source_type,
+                source_status=source.status,
+                monitored_runs=monitored_runs,
+                success_ratio=success_ratio,
+                avg_latency_ms=avg_latency_ms,
+                error_counts=error_counts,
+                http_status_counts=http_status_counts,
+                extracted_count=extracted_count,
+                updated_count=updated_count,
+                duplicate_count=duplicate_count,
+                last_parser=last_parser,
+                last_parser_warnings=last_warnings,
+                last_health_action=action,
+                last_health_reason=reason,
+            )
+        )
+
+    return rows
+
+
+def evaluate_source_health(*, now: datetime | None = None) -> dict[str, int]:
+    now = now or timezone.now()
+    rows = [row for row in collect_source_health() if row.source_status in {Source.Status.ACTIVE, Source.Status.TRIAL}]
+    counts = {"evaluated": len(rows), "quarantined": 0}
+
+    for row in rows:
+        if row.last_health_action != "quarantine":
+            continue
+        counts["evaluated"] += 1
+
+        source = Source.objects.get(pk=row.source_id)
+        source.status = Source.Status.QUARANTINED
+        source.policy = Source.Policy.DENY
+        source.policy_reason = row.last_health_reason or "Bronhealth detecteert instabiele bron"
+        _set_health_metadata(
+            source,
+            {
+                "state": "quarantined",
+                "quarantine_reason": source.policy_reason,
+                "quarantined_at": _to_iso(now),
+                "recovery_due_at": _to_iso(
+                    now + timedelta(hours=SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS)
+                ),
+                "canary_started": False,
+            },
+        )
+        source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
+
+        latest_review = source.policy_reviews.order_by("-created_at").first()
+        if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.DENY:
+            create_policy_review(
+                source,
+                actor=None,
+                decision=SourcePolicyReview.Decision.DENY,
+                reason=source.policy_reason,
+                scope=SourcePolicyReview.Scope.SOURCE,
+                notes="Automatische bronhealth",
+            )
+
+        counts["quarantined"] += 1
+
+    return counts
+
+
+def canary_recovery_sources(*, now: datetime | None = None) -> list[Source]:
+    now = now or timezone.now()
+    sources: list[Source] = []
+    for source in Source.objects.filter(status=Source.Status.QUARANTINED):
+        health = _health_metadata(source)
+        if health.get("state") != "quarantined":
+            continue
+
+        if bool(health.get("canary_started", False)):
+            continue
+
+        recovery_due = _from_iso(health.get("recovery_due_at") if isinstance(health, dict) else None)
+        if recovery_due and recovery_due > now:
+            continue
+
+        sources.append(source)
+
+    return sources
+
+
+def start_health_canary(source: Source, *, now: datetime | None = None) -> None:
+    now = now or timezone.now()
+    health = _health_metadata(source)
+
+    latest_review = source.policy_reviews.order_by("-created_at").first()
+    if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.TRIAL:
+        create_policy_review(
+            source,
+            actor=None,
+            decision=SourcePolicyReview.Decision.TRIAL,
+            reason="Automatische bronrecovery via canary",
+            scope=SourcePolicyReview.Scope.SOURCE,
+            notes="Canaryherstel",
+        )
+
+    source.status = Source.Status.TRIAL
+    source.policy = Source.Policy.REVIEW
+    source.policy_reason = "Bronherstel via geautomatiseerde canary"
+    source.next_run_at = now
+    _set_health_metadata(
+        source,
+        {
+            **health,
+            "state": "canary_in_progress",
+            "canary_started": True,
+            "canary_started_at": _to_iso(now),
+        },
+    )
+    source.save(update_fields=["status", "policy", "policy_reason", "next_run_at", "updated_at"])

apps/sources/services/manual_import.py

@@ -0,0 +1,355 @@
+from __future__ import annotations
+
+import hashlib
+import html
+from dataclasses import dataclass
+from datetime import timedelta
+from urllib.parse import urlsplit
+from uuid import uuid4
+
+import bleach
+from django.conf import settings
+from django.db import transaction
+from django.db.models import QuerySet
+from django.utils import timezone
+
+from apps.jobs.models import JobSourceAlias
+from apps.jobs.services.pipeline import process_raw_document
+from apps.sources.models import RawDocument, Source, SourcePolicyReview, SourceRun
+from apps.sources.services.canonicalize import canonicalize_url
+from apps.sources.services.fetcher import (
+    FetchError,
+    FetchTimeoutError,
+    FetchedDocument,
+    PolicyBlockedError,
+    RateLimitedError,
+    fetch_url,
+)
+from apps.sources.services.policy import assess_url, create_policy_review
+
+
+class ManualImportError(RuntimeError):
+    pass
+
+
+@dataclass(frozen=True)
+class ManualImportSummary:
+    source_id: int
+    source_name: str
+    source_url: str
+    mode: str
+    extracted_count: int
+    created_count: int
+    duplicate_count: int
+    warnings: list[str]
+    jobs: list[dict[str, str]]
+
+    def to_session_payload(self) -> dict[str, object]:
+        return {
+            "source_id": self.source_id,
+            "source_name": self.source_name,
+            "source_url": self.source_url,
+            "mode": self.mode,
+            "extracted_count": self.extracted_count,
+            "created_count": self.created_count,
+            "duplicate_count": self.duplicate_count,
+            "warnings": self.warnings,
+            "jobs": self.jobs,
+        }
+
+
+def _sanitize_pasted_text(value: str) -> str:
+    text = bleach.clean(value or "", tags=[], attributes={}, strip=True).strip()
+    if not text:
+        raise ManualImportError("Het geplakte tekstveld bevat geen bruikbare inhoud.")
+    if len(text.encode("utf-8")) > settings.MANUAL_IMPORT_PASTE_MAX_BYTES:
+        raise ManualImportError(
+            "Het tekstveld is te groot voor veilige import. Verwijder overtollige tekst."
+        )
+    return text
+
+
+def _kind_for(content_type: str) -> str:
+    content_type = (content_type or "").lower()
+    if "html" in content_type:
+        return RawDocument.Kind.HTML
+    if "xml" in content_type or "rss" in content_type or "atom" in content_type:
+        return RawDocument.Kind.XML
+    if "json" in content_type:
+        return RawDocument.Kind.JSON
+    return RawDocument.Kind.TEXT
+
+
+def _mode_source_name(domain: str, *, mode: str) -> str:
+    if mode == "paste":
+        return f"Handmatige tekstimport {domain}"
+    return f"Handmatige URL-import {domain}"
+
+
+def _ensure_manual_review(source: Source, actor) -> None:
+    review = source.latest_policy_review
+    if (
+        review
+        and not review.is_expired
+        and review.decision == SourcePolicyReview.Decision.ALLOW
+    ):
+        return
+    create_policy_review(
+        source,
+        actor=actor if actor and getattr(actor, "pk", None) else None,
+        decision=SourcePolicyReview.Decision.ALLOW,
+        reason="Handmatige import uitgevoerd.",
+        scope=SourcePolicyReview.Scope.SOURCE,
+    )
+
+
+def _ensure_manual_source(*, domain: str, source_url: str, actor, mode: str) -> Source:
+    defaults = {
+        "name": _mode_source_name(domain, mode=mode),
+        "base_url": source_url,
+        "status": Source.Status.CANDIDATE,
+        "policy": Source.Policy.ALLOW,
+    }
+    source, created = Source.objects.get_or_create(
+        domain=domain,
+        source_type=Source.Type.MANUAL,
+        defaults=defaults,
+    )
+    if not created:
+        source.name = _mode_source_name(domain, mode=mode)
+        source.base_url = source_url
+        source.status = Source.Status.CANDIDATE
+        source.policy = Source.Policy.ALLOW
+        source.save(
+            update_fields=["name", "base_url", "status", "policy", "updated_at"]
+        )
+    _ensure_manual_review(source, actor=actor)
+    return source
+
+
+def _create_raw_document(
+    source: Source,
+    *,
+    source_run: SourceRun,
+    requested_url: str,
+    final_url: str,
+    content_type: str,
+    content: bytes,
+) -> RawDocument:
+    return RawDocument.objects.create(
+        source=source,
+        source_run=source_run,
+        url=requested_url,
+        final_url=final_url,
+        kind=_kind_for(content_type),
+        content_type=content_type[:200],
+        http_status=None,
+        response_headers={},
+        content_hash=hashlib.sha256(content).hexdigest(),
+        body_text=content.decode("utf-8", errors="replace"),
+        byte_length=len(content),
+        retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
+    )
+
+
+def _build_jobs_from_document(document: RawDocument) -> list[dict[str, str]]:
+    alias_qs: QuerySet[JobSourceAlias] = JobSourceAlias.objects.select_related("job").filter(
+        raw_document=document
+    )
+    jobs: list[dict[str, str]] = []
+    for alias in alias_qs:
+        title = alias.source_title or (alias.job.original_title if alias.job else "Vacature")
+        employer = alias.source_employer or "Onbekende werkgever"
+        jobs.append(
+            {
+                "id": str(alias.job_id),
+                "title": title,
+                "employer": employer,
+            }
+        )
+    return jobs
+
+
+def _run_pipeline(document: RawDocument, *, source_run: SourceRun) -> ManualImportSummary:
+    metrics = process_raw_document(document)
+    source = document.source
+    source_name = source.name if source else ""
+    warnings: list[str] = list(metrics.get("warnings", []))
+    warnings_count = len(warnings)
+    source_run.finish(
+        SourceRun.Status.SUCCESS,
+        http_status=document.source_run.http_status if document.source_run else None,
+        extracted_count=int(metrics["extracted"]),
+        created_count=int(metrics["created"]),
+        updated_count=int(metrics["updated"]),
+        duplicate_count=int(metrics["duplicates"]),
+        metrics={"parser": metrics["parser"], "warnings": warnings},
+    )
+    jobs = _build_jobs_from_document(document)
+    if metrics["created"] == 0 and metrics["updated"] == 0:
+        if not warnings:
+            warnings.append("De bron leverde geen herkenbare vacaturedata op.")
+    if not warnings:
+        # keep stable, machine-readable payload shape
+        warnings = []
+    return ManualImportSummary(
+        source_id=source.pk,
+        source_name=source_name,
+        source_url=document.final_url,
+        mode="url",
+        extracted_count=int(metrics["extracted"]),
+        created_count=int(metrics["created"]),
+        duplicate_count=int(metrics["duplicates"]),
+        warnings=warnings,
+        jobs=jobs,
+    )
+
+
+def _build_synthetic_paste_payload(raw_text: str) -> tuple[str, bytes, str]:
+    lines = [html.escape(line.strip()) for line in raw_text.splitlines() if line.strip()]
+    title = lines[0] if lines else "Handmatige vacature"
+    body = "".join(f"<p>{line}</p>" for line in lines)
+    source_domain = f"manual-{uuid4().hex[:16]}"
+    synthetic_url = f"https://{source_domain}.vacature.local/"
+    html_content = (
+        f"<html><body><main><h1>{title}</h1>{body}</main>"
+        "<footer>Handmatige import; geen externe fetch</footer></body></html>"
+    )
+    return synthetic_url, html_content.encode("utf-8"), source_domain
+
+
+def import_manual_source(
+    *, actor, source_url: str | None = None, pasted_text: str | None = None
+) -> ManualImportSummary:
+    source_url = (source_url or "").strip()
+    pasted_text = (pasted_text or "").strip()
+    if not source_url and not pasted_text:
+        raise ManualImportError("Vul een URL of tekst in.")
+
+    with transaction.atomic():
+        if source_url:
+            normalized = canonicalize_url(source_url)
+            parsed = urlsplit(normalized)
+            domain = (parsed.hostname or "").lower()
+            if not domain:
+                raise ManualImportError("De bron-URL bevat geen geldig domein.")
+            decision = assess_url(normalized)
+            if not decision.allowed:
+                raise ManualImportError(f"Import geblokkeerd: {decision.reason}")
+            source = _ensure_manual_source(
+                domain=domain,
+                source_url=normalized,
+                actor=actor,
+                mode="url",
+            )
+            source_run = SourceRun.objects.create(source=source)
+            try:
+                fetched: FetchedDocument = fetch_url(normalized, source=source)
+            except PolicyBlockedError as exc:
+                source.status = Source.Status.QUARANTINED
+                source.policy = Source.Policy.DENY
+                source.policy_reason = str(exc)[:1000]
+                source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
+                source_run.finish(
+                    SourceRun.Status.SKIPPED,
+                    error_category="policy",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Import geblokkeerd: {exc}") from exc
+            except RateLimitedError as exc:
+                source_run.finish(
+                    SourceRun.Status.FAILED,
+                    error_category="rate_limited",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Rate limiting tijdens import: {exc}") from exc
+            except FetchTimeoutError as exc:
+                source_run.finish(
+                    SourceRun.Status.FAILED,
+                    error_category="timeout",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Time-out tijdens import: {exc}") from exc
+            except FetchError as exc:
+                source_run.finish(
+                    SourceRun.Status.FAILED,
+                    error_category="fetch",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Fetch mislukt: {exc}") from exc
+
+            if fetched.status_code == 304:
+                source_run.finish(
+                    SourceRun.Status.SUCCESS,
+                    http_status=304,
+                    extracted_count=0,
+                    created_count=0,
+                    updated_count=0,
+                    duplicate_count=0,
+                    metrics={"parser": "not-modified", "warnings": []},
+                )
+                return ManualImportSummary(
+                    source_id=source.pk,
+                    source_name=source.name,
+                    source_url=source.base_url,
+                    mode="url",
+                    extracted_count=0,
+                    created_count=0,
+                    duplicate_count=0,
+                    warnings=["Geen inhoudsverandering (304)."],
+                    jobs=[],
+                )
+
+            document = _create_raw_document(
+                source,
+                source_run=source_run,
+                requested_url=fetched.requested_url,
+                final_url=fetched.final_url,
+                content_type=fetched.headers.get("content-type", ""),
+                content=fetched.content,
+            )
+            source.etag = fetched.headers.get("etag", source.etag)
+            source.last_modified = fetched.headers.get("last-modified", source.last_modified)
+            source.save(update_fields=["etag", "last_modified", "updated_at"])
+            summary = _run_pipeline(document, source_run=source_run)
+            return ManualImportSummary(
+                source_id=summary.source_id,
+                source_name=summary.source_name,
+                source_url=summary.source_url,
+                mode="url",
+                extracted_count=summary.extracted_count,
+                created_count=summary.created_count,
+                duplicate_count=summary.duplicate_count,
+                warnings=summary.warnings,
+                jobs=summary.jobs,
+            )
+
+        text = _sanitize_pasted_text(pasted_text)
+        synthetic_url, payload, synthetic_domain = _build_synthetic_paste_payload(text)
+        source = _ensure_manual_source(
+            domain=synthetic_domain,
+            source_url=synthetic_url,
+            actor=actor,
+            mode="paste",
+        )
+        source_run = SourceRun.objects.create(source=source)
+        document = _create_raw_document(
+            source,
+            source_run=source_run,
+            requested_url=synthetic_url,
+            final_url=synthetic_url,
+            content_type="text/html; charset=utf-8",
+            content=payload,
+        )
+        summary = _run_pipeline(document, source_run=source_run)
+        return ManualImportSummary(
+            source_id=summary.source_id,
+            source_name=source.name,
+            source_url=synthetic_url,
+            mode="paste",
+            extracted_count=summary.extracted_count,
+            created_count=summary.created_count,
+            duplicate_count=summary.duplicate_count,
+            warnings=summary.warnings,
+            jobs=summary.jobs,
+        )

apps/sources/services/policy.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from urllib.parse import urlsplit
+
+from django.conf import settings
+from django.utils import timezone
+
+from apps.sources.models import Source, SourcePolicyReview
+
+from .canonicalize import domain_matches
+from .robots import assess_robots
+
+DEFAULT_DENYLIST = {
+    "linkedin.com",
+    "indeed.com",
+    "indeed.be",
+    "stepstone.be",
+    "jobat.be",
+    "vdab.be",
+}
+
+
+@dataclass(frozen=True)
+class PolicyDecision:
+    allowed: bool
+    status: str
+    reason: str
+
+
+def _review_expiry(now: datetime | None = None) -> datetime:
+    return (now or timezone.now()) + timedelta(days=getattr(settings, "SOURCE_REVIEW_TTL_DAYS", 90))
+
+
+def is_denied_domain(hostname: str, denylist: set[str] | None = None) -> bool:
+    denylist = denylist or DEFAULT_DENYLIST
+    return any(domain_matches(hostname, domain) for domain in denylist)
+
+
+def create_policy_review(
+    source: Source,
+    *,
+    actor,
+    decision: SourcePolicyReview.Decision,
+    reason: str,
+    scope: SourcePolicyReview.Scope = SourcePolicyReview.Scope.SOURCE,
+    notes: str = "",
+    evidence_link: str = "",
+    expires_at: datetime | None = None,
+    metadata: dict | None = None,
+) -> SourcePolicyReview:
+    return SourcePolicyReview.objects.create(
+        source=source,
+        actor=actor if actor and getattr(actor, "pk", None) else None,
+        decision=decision,
+        scope=scope,
+        reason=reason,
+        notes=notes,
+        evidence_link=evidence_link,
+        expires_at=expires_at or _review_expiry(),
+        metadata=metadata or {},
+    )
+
+
+def _active_review(source: Source) -> SourcePolicyReview | None:
+    review = source.latest_policy_review
+    if review and not review.is_expired:
+        return review
+    return None
+
+
+def _check_review_gate(source: Source) -> PolicyDecision | None:
+    review = _active_review(source)
+    if source.policy == Source.Policy.REVIEW and source.status in {
+        Source.Status.CANDIDATE,
+        Source.Status.TRIAL,
+    }:
+        if not review:
+            return PolicyDecision(False, Source.Policy.REVIEW, "Review vereist")
+        if review.decision == SourcePolicyReview.Decision.DENY:
+            return PolicyDecision(False, Source.Policy.DENY, review.reason or "Review blokkeert bron")
+        if review.decision == SourcePolicyReview.Decision.PAUSE:
+            return PolicyDecision(False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze")
+        return None
+
+    if source.policy == Source.Policy.ALLOW:
+        if not review:
+            return PolicyDecision(False, Source.Policy.REVIEW, "Review ontbreekt voor actief beleid")
+        if review.decision == SourcePolicyReview.Decision.DENY:
+            return PolicyDecision(False, Source.Policy.DENY, review.reason or "Review blokkeert bron")
+        if review.decision == SourcePolicyReview.Decision.PAUSE:
+            return PolicyDecision(False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze")
+    return None
+
+
+def assess_url(url: str, *, source: Source | None = None) -> PolicyDecision:
+    hostname = (urlsplit(url).hostname or "").lower()
+    if not hostname:
+        return PolicyDecision(False, Source.Policy.DENY, "URL zonder hostname")
+    if is_denied_domain(hostname):
+        return PolicyDecision(False, Source.Policy.DENY, "Platformdomein staat op de denylist")
+    if source:
+        if source.status in {Source.Status.DISABLED, Source.Status.PAUSED}:
+            return PolicyDecision(False, Source.Policy.REVIEW, f"Bronstatus: {source.status}")
+        if source.policy == Source.Policy.DENY:
+            return PolicyDecision(
+                False, Source.Policy.DENY, source.policy_reason or "Bron geblokkeerd"
+            )
+        review_decision = _check_review_gate(source)
+        if review_decision is not None:
+            return review_decision
+
+    robots = assess_robots(url, source=source)
+    if not robots.allowed:
+        return PolicyDecision(False, Source.Policy.REVIEW, robots.reason)
+
+    return PolicyDecision(True, Source.Policy.ALLOW, "Toegestane publieke bron")

apps/sources/services/robots.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from urllib.parse import urlsplit, urlunsplit
+
+import httpx
+from django.conf import settings
+from django.utils import timezone
+
+from apps.sources.models import Source, SourceRobotsCache
+from .url_security import UnsafeUrlError, validate_public_url
+
+ALLOW = "allow"
+DISALLOW = "disallow"
+
+
+@dataclass(frozen=True)
+class RobotsDecision:
+    allowed: bool
+    reason: str
+
+
+def _origin_for(url: str) -> str:
+    parts = urlsplit(url)
+    if not parts.scheme:
+        raise ValueError("Ongeldige URL voor robotscontrole.")
+    host = (parts.hostname or "").lower().rstrip(".")
+    if not host:
+        raise ValueError("Host ontbreekt voor robotscontrole.")
+    port = parts.port
+    if (parts.scheme == "http" and port == 80) or (parts.scheme == "https" and port == 443) or not port:
+        netloc = host
+    else:
+        netloc = f"{host}:{port}"
+    return f"{parts.scheme}://{netloc}"
+
+
+def _path_for(url: str) -> str:
+    path = urlsplit(url).path or "/"
+    return path if path.startswith("/") else f"/{path}"
+
+
+def _robots_url(origin: str) -> str:
+    parts = urlsplit(origin)
+    return urlunsplit((parts.scheme, parts.netloc, "/robots.txt", "", ""))
+
+
+def _rules_from_text(text: str) -> dict[str, dict[str, list[str]]]:
+    bucket: dict[str, dict[str, list[str]]] = {}
+    active_agents: set[str] = set()
+
+    for raw_line in text.splitlines():
+        line = raw_line.split("#", 1)[0].strip()
+        if ":" not in line:
+            continue
+        key, value = (part.strip() for part in line.split(":", 1))
+        if not key:
+            continue
+        key_lower = key.lower()
+        if key_lower == "user-agent":
+            token = value.lower()
+            if token:
+                active_agents = {token}
+            else:
+                active_agents = set()
+            continue
+        if key_lower not in {ALLOW, DISALLOW}:
+            continue
+        if not active_agents:
+            continue
+        for agent in active_agents:
+            section = bucket.setdefault(agent, {ALLOW: [], DISALLOW: []})
+            section[key_lower].append(value.strip() or "/")
+
+    for entry in bucket.values():
+        entry[ALLOW] = list(dict.fromkeys(entry[ALLOW]))
+        entry[DISALLOW] = list(dict.fromkeys(entry[DISALLOW]))
+    return bucket
+
+
+def _pick_rules(rules: dict[str, dict[str, list[str]]], user_agent: str) -> dict[str, list[str]]:
+    normalized = user_agent.lower()
+    selected = {ALLOW: [], DISALLOW: []}
+
+    for agent, values in rules.items():
+        if agent == "*" or agent and agent in normalized:
+            selected[ALLOW].extend(values[ALLOW])
+            selected[DISALLOW].extend(values[DISALLOW])
+
+    selected[ALLOW] = list(dict.fromkeys(selected[ALLOW]))
+    selected[DISALLOW] = list(dict.fromkeys(selected[DISALLOW]))
+    return selected
+
+
+def _longest_prefix(path: str, rules: Iterable[str]) -> int:
+    return max((len(rule.rstrip("/")) for rule in rules if rule and path.startswith(rule)), default=0)
+
+
+def _evaluate_path(path: str, user_agent: str, rules: dict[str, dict[str, list[str]]]) -> RobotsDecision:
+    selected = _pick_rules(rules, user_agent=user_agent)
+    allow_len = _longest_prefix(path, selected[ALLOW])
+    disallow_len = _longest_prefix(path, selected[DISALLOW])
+    if disallow_len > allow_len:
+        return RobotsDecision(False, "Toegang geblokkeerd door robotsregels")
+    return RobotsDecision(True, "Robotsregels staan toegang toe")
+
+
+def _cache_ttl_seconds() -> float:
+    return float(getattr(settings, "ROBOTS_CACHE_TTL_SECONDS", 3600))
+
+
+def _max_bytes() -> int:
+    return int(getattr(settings, "ROBOTS_MAX_BYTES", 131072))
+
+
+def _fetch_robots(origin: str, *, client: httpx.Client | None = None) -> tuple[str, int, str, str]:
+    robots_url = _robots_url(origin)
+    validate_public_url(
+        robots_url,
+        allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
+    )
+    own_client = client is None
+    http_client = client or httpx.Client(
+        timeout=httpx.Timeout(getattr(settings, "ROBOTS_FETCH_TIMEOUT_SECONDS", 5)),
+        follow_redirects=False,
+    )
+    try:
+        response = http_client.get(
+            robots_url,
+            headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
+        )
+    finally:
+        if own_client:
+            http_client.close()
+    return (
+        response.text,
+        response.status_code,
+        response.headers.get("etag", ""),
+        response.headers.get("last-modified", ""),
+    )
+
+
+def _load_cached(origin: str, now: datetime) -> SourceRobotsCache | None:
+    try:
+        cache = SourceRobotsCache.objects.get(origin=origin)
+    except SourceRobotsCache.DoesNotExist:
+        return None
+    if cache.expires_at <= now:
+        return None
+    return cache
+
+
+def _load_stale_cache(origin: str) -> SourceRobotsCache | None:
+    try:
+        return SourceRobotsCache.objects.get(origin=origin)
+    except SourceRobotsCache.DoesNotExist:
+        return None
+
+
+def _persist_cache(
+    origin: str,
+    *,
+    status_code: int,
+    content: str,
+    etag: str = "",
+    last_modified: str = "",
+    now: datetime,
+) -> SourceRobotsCache:
+    max_bytes = _max_bytes()
+    byte_length = len(content.encode("utf-8"))
+    if byte_length > max_bytes:
+        return SourceRobotsCache.objects.update_or_create(
+            origin=origin,
+            defaults={
+                "expires_at": now + timedelta(seconds=_cache_ttl_seconds()),
+                "allow_rules": {},
+                "disallow_rules": {},
+                "error": f"robots.txt te groot ({byte_length} bytes)",
+                "etag": etag,
+                "last_modified": last_modified,
+                "byte_length": byte_length,
+            },
+        )[0]
+
+    if status_code in {404, 410}:
+        rules = {}
+    elif status_code >= 400:
+        rules = {}
+    else:
+        rules = _rules_from_text(content)
+
+    return SourceRobotsCache.objects.update_or_create(
+        origin=origin,
+        defaults={
+            "expires_at": now + timedelta(seconds=_cache_ttl_seconds()),
+            "allow_rules": {agent: value[ALLOW] for agent, value in rules.items()},
+            "disallow_rules": {agent: value[DISALLOW] for agent, value in rules.items()},
+            "error": "",
+            "etag": etag,
+            "last_modified": last_modified,
+            "byte_length": byte_length,
+        },
+    )[0]
+
+
+def _build_ruleset(cache: SourceRobotsCache) -> dict[str, dict[str, list[str]]]:
+    rules = {"allow": {}, "disallow": {}}
+    for agent, entries in cache.allow_rules.items():
+        rules["allow"][agent] = list(entries)
+    for agent, entries in cache.disallow_rules.items():
+        rules["disallow"][agent] = list(entries)
+
+    normalized_rules: dict[str, dict[str, list[str]]] = {}
+    all_agents = set(rules["allow"].keys()) | set(rules["disallow"].keys())
+    for agent in all_agents:
+        normalized_rules[agent] = {
+            ALLOW: rules["allow"].get(agent, []),
+            DISALLOW: rules["disallow"].get(agent, []),
+        }
+    return normalized_rules
+
+
+def assess_robots(
+    url: str,
+    *,
+    source: Source | None = None,
+    user_agent: str | None = None,
+    client: httpx.Client | None = None,
+    now: datetime | None = None,
+) -> RobotsDecision:
+    if source is None or not source.honor_robots:
+        return RobotsDecision(True, "Robotscontrole niet vereist")
+
+    now = now or timezone.now()
+    try:
+        origin = _origin_for(url)
+        path = _path_for(url)
+    except ValueError as exc:
+        return RobotsDecision(False, str(exc))
+
+    cache = _load_cached(origin, now=now)
+    stale = _load_stale_cache(origin)
+    if cache is None:
+        try:
+            content, status_code, etag, last_modified = _fetch_robots(origin, client=client)
+            cache = _persist_cache(
+                origin,
+                status_code=status_code,
+                content=content,
+                etag=etag,
+                last_modified=last_modified,
+                now=now,
+            )
+        except UnsafeUrlError as exc:
+            return RobotsDecision(False, f"Robotscontrole mislukt: {exc}")
+        except httpx.HTTPError as exc:
+            if stale is not None:
+                return _evaluate_path(path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=_build_ruleset(stale))
+            return RobotsDecision(True, f"Robotscontrole tijdelijk niet beschikbaar: {exc}")
+
+    if cache.error:
+        return RobotsDecision(True, cache.error)
+
+    rules = _build_ruleset(cache)
+    decision = _evaluate_path(path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=rules)
+    return decision

apps/sources/services/scheduling.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import timedelta
+from urllib.parse import urlparse
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models import Max
+from django.utils import timezone
+
+from apps.sources.models import Source, SourceLease, SourceOriginState
+
+
+def origin_key(source: Source) -> str:
+    if source.domain:
+        return source.domain.lower()
+    hostname = (urlparse(source.base_url).hostname or "").lower()
+    return hostname.rstrip(".")
+
+
+def max_origin_minimum_interval_seconds(source: Source) -> int:
+    domain = origin_key(source)
+    aggregate = Source.objects.filter(domain=domain).aggregate(
+        maximum_interval=Max("minimum_interval_seconds")
+    )
+    max_interval = aggregate["maximum_interval"]
+    if max_interval is None:
+        return int(source.minimum_interval_seconds)
+    return int(max_interval)
+
+
+def _lease_ttl_seconds() -> int:
+    ttl = settings.SOURCE_LEASE_TTL_SECONDS
+    if ttl <= 0:
+        return 180
+    return int(ttl)
+
+
+def _max_origin_interval_seconds(source: Source) -> int:
+    explicit = settings.SOURCE_ORIGIN_MIN_INTERVAL_SECONDS
+    if explicit > 0:
+        return int(explicit)
+    return max_origin_minimum_interval_seconds(source)
+
+
+def calculate_jitter_seconds(source: Source, *, max_seconds: int) -> int:
+    max_seconds = int(max_seconds)
+    if max_seconds <= 0:
+        return 0
+    jitter_seed = f"{source.pk}:{source.domain}:{source.minimum_interval_seconds}"
+    digest = hashlib.sha256(jitter_seed.encode("utf-8")).digest()
+    jitter_span = max_seconds + 1
+    return int(int.from_bytes(digest[:8], "big") % jitter_span)
+
+
+def calculate_failure_backoff_seconds(
+    source: Source,
+    *,
+    failure_count: int,
+    retry_after_seconds: int | None = None,
+    timeout: bool = False,
+) -> int:
+    failure_count = max(1, failure_count)
+    base_seconds = settings.SOURCE_FAILURE_BACKOFF_BASE_SECONDS
+    if timeout:
+        base_seconds = max(base_seconds, settings.SOURCE_FAILURE_TIMEOUT_BASE_SECONDS)
+    factor = 2 ** min(failure_count - 1, 10)
+    backoff_seconds = base_seconds * factor
+    if retry_after_seconds:
+        backoff_seconds = max(backoff_seconds, retry_after_seconds)
+    max_seconds = settings.SOURCE_FAILURE_BACKOFF_MAX_SECONDS
+    return min(max_seconds, backoff_seconds) + calculate_jitter_seconds(
+        source, max_seconds=settings.SOURCE_FAILURE_JITTER_SECONDS
+    )
+
+
+def calculate_success_jitter_seconds(source: Source) -> int:
+    return calculate_jitter_seconds(
+        source, max_seconds=settings.SOURCE_SUCCESS_JITTER_SECONDS
+    )
+
+
+def acquire_source_lease(
+    *, source_id: int, worker_token: str, now=None
+) -> SourceLease | None:
+    now = now or timezone.now()
+    with transaction.atomic():
+        source = Source.objects.select_for_update().get(pk=source_id)
+        if not source.domain:
+            return None
+
+        domain = origin_key(source)
+        origin_state = SourceOriginState.objects.select_for_update().get_or_create(
+            domain=domain, defaults={"next_allowed_at": now - timedelta(seconds=1)}
+        )[0]
+
+        if origin_state.next_allowed_at and origin_state.next_allowed_at > now:
+            existing_lease = SourceLease.objects.select_for_update().filter(source=source).first()
+            if not existing_lease or existing_lease.token != worker_token:
+                return None
+
+        lease = SourceLease.objects.select_for_update().filter(source=source).first()
+        if lease is not None and not lease.is_expired:
+            if lease.token != worker_token:
+                return None
+
+        active_leases = SourceLease.objects.select_for_update().filter(
+            source__domain=domain, expires_at__gt=now
+        )
+        active_count = active_leases.exclude(source=source).count()
+        max_concurrency = max(1, int(source.max_concurrency))
+        if active_count >= max_concurrency:
+            return None
+
+        if lease is None:
+            lease = SourceLease(source=source)
+        lease.token = worker_token
+        lease.worker_id = worker_token
+        lease.expires_at = now + timedelta(seconds=_lease_ttl_seconds())
+        lease.save(update_fields=["token", "worker_id", "expires_at", "updated_at"])
+        return lease
+
+
+def release_source_lease(*, source_id: int, worker_token: str, now=None) -> bool:
+    now = now or timezone.now()
+    with transaction.atomic():
+        source = Source.objects.select_for_update().get(pk=source_id)
+        if not source.domain:
+            return False
+
+        lease = SourceLease.objects.select_for_update().filter(
+            source=source, token=worker_token
+        ).first()
+        if not lease:
+            return False
+
+        lease.expires_at = now
+        lease.save(update_fields=["expires_at", "updated_at"])
+
+        interval_seconds = _max_origin_interval_seconds(source)
+        domain = origin_key(source)
+        origin_state = SourceOriginState.objects.select_for_update().get_or_create(
+            domain=domain, defaults={"next_allowed_at": now}
+        )[0]
+        origin_state.next_allowed_at = now + timedelta(seconds=max(0, interval_seconds))
+        origin_state.save(update_fields=["next_allowed_at", "updated_at"])
+        return True

apps/sources/services/url_security.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import ipaddress
+import socket
+from collections.abc import Callable
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
+Resolver = Callable[..., list[tuple]]
+
+
+class UnsafeUrlError(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class ValidatedUrl:
+    url: str
+    hostname: str
+    port: int
+    addresses: tuple[str, ...]
+
+
+def _extract_addresses(results: list[tuple]) -> tuple[str, ...]:
+    addresses: list[str] = []
+    for result in results:
+        sockaddr = result[4]
+        if sockaddr:
+            addresses.append(str(sockaddr[0]))
+    return tuple(dict.fromkeys(addresses))
+
+
+def validate_public_url(
+    url: str,
+    *,
+    resolver: Resolver = socket.getaddrinfo,
+    allow_nonstandard_ports: bool = False,
+) -> ValidatedUrl:
+    parts = urlsplit(url)
+    if parts.scheme.lower() not in {"http", "https"}:
+        raise UnsafeUrlError("Alleen http en https zijn toegestaan.")
+    if parts.username or parts.password:
+        raise UnsafeUrlError("URLs met ingebedde credentials zijn niet toegestaan.")
+    hostname = (parts.hostname or "").lower().rstrip(".")
+    if not hostname:
+        raise UnsafeUrlError("URL bevat geen hostname.")
+    if hostname == "localhost" or hostname.endswith(".localhost"):
+        raise UnsafeUrlError("Lokale hostnames zijn niet toegestaan.")
+    port = parts.port or (443 if parts.scheme.lower() == "https" else 80)
+    if not allow_nonstandard_ports and port not in {80, 443}:
+        raise UnsafeUrlError("Niet-standaard poorten zijn niet toegestaan.")
+    try:
+        direct_ip = ipaddress.ip_address(hostname.strip("[]"))
+        addresses = (str(direct_ip),)
+    except ValueError:
+        try:
+            results = resolver(hostname, port, type=socket.SOCK_STREAM)
+        except OSError as exc:
+            raise UnsafeUrlError("Hostname kan niet veilig worden opgelost.") from exc
+        addresses = _extract_addresses(results)
+    if not addresses:
+        raise UnsafeUrlError("Hostname leverde geen IP-adressen op.")
+    for raw in addresses:
+        try:
+            ip = ipaddress.ip_address(raw)
+        except ValueError as exc:
+            raise UnsafeUrlError("Ongeldig IP-adres na DNS-resolutie.") from exc
+        if not ip.is_global:
+            raise UnsafeUrlError(f"Niet-publiek IP-adres geblokkeerd: {ip}")
+    return ValidatedUrl(url=url, hostname=hostname, port=port, addresses=addresses)