Initial deploy setup

apps/sources/adapters/__init__.py

@@ -0,0 +1,24 @@
+from .base import ExtractedJob, ExtractionResult
+from .generic_html import GenericHtmlAdapter
+from .jsonld import JsonLdJobPostingAdapter
+from .rss import RssAdapter
+from .ats import (
+    GreenhouseAdapter,
+    LeverAdapter,
+    RecruiteeAdapter,
+    SmartRecruitersAdapter,
+    WorkableAdapter,
+)
+
+__all__ = [
+    "ExtractedJob",
+    "ExtractionResult",
+    "GenericHtmlAdapter",
+    "JsonLdJobPostingAdapter",
+    "RssAdapter",
+    "GreenhouseAdapter",
+    "LeverAdapter",
+    "RecruiteeAdapter",
+    "SmartRecruitersAdapter",
+    "WorkableAdapter",
+]

apps/sources/adapters/ats.py

@@ -0,0 +1,546 @@
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from urllib.parse import urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+CLOSED_STATUSES = {
+    "closed",
+    "inactive",
+    "withdrawn",
+    "removed",
+    "filled",
+    "expired",
+    "archived",
+}
+
+
+def _to_text(value):
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value.strip()
+    if isinstance(value, bool):
+        return str(value).lower()
+    if isinstance(value, int | float):
+        return str(value)
+    return ""
+
+
+def _find_nested(data, path: str):
+    if data is None:
+        return None
+    node = data
+    for part in path.split("."):
+        if not isinstance(node, dict) or part not in node:
+            return None
+        node = node[part]
+    return node
+
+
+def _first_text(data, *candidates):
+    for candidate in candidates:
+        if "." in candidate:
+            value = _find_nested(data, candidate)
+        else:
+            value = data.get(candidate) if isinstance(data, dict) else None
+        if value is None:
+            continue
+        if isinstance(value, (list, tuple)):
+            for item in value:
+                text = _to_text(item)
+                if text:
+                    return text
+            continue
+        text = _to_text(value)
+        if text:
+            return text
+        if isinstance(value, dict):
+            nested = value.values()
+            for nested_value in nested:
+                nested_text = _to_text(nested_value)
+                if nested_text:
+                    return nested_text
+    return ""
+
+
+def _collect_texts(value) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value.strip()] if value.strip() else []
+    if isinstance(value, list):
+        out: list[str] = []
+        for item in value:
+            for inner in _collect_texts(item):
+                if inner:
+                    out.append(inner)
+        return out
+    if isinstance(value, dict):
+        out: list[str] = []
+        for key in ("name", "city", "location", "address", "raw", "region", "country"):
+            if key in value:
+                for inner in _collect_texts(value[key]):
+                    if inner:
+                        out.append(inner)
+        return out
+    if isinstance(value, bool | int | float):
+        return [_to_text(value)]
+    return []
+
+
+def _to_list(value) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return [item.strip() for item in (_to_text(v) for v in value) if item.strip()]
+    text = _to_text(value)
+    if not text:
+        return []
+    return [part.strip() for part in text.replace(";", ",").split(",") if part.strip()]
+
+
+def _as_text(value) -> str:
+    text = _to_text(value)
+    if not text:
+        return ""
+    return BeautifulSoup(text, "lxml").get_text(" ", strip=True)
+
+
+def _extract_payload(content: str):
+    try:
+        return json.loads(content)
+    except json.JSONDecodeError:
+        pass
+    soup = BeautifulSoup(content, "lxml")
+    for script in soup.find_all("script", attrs={"type": "application/json"}):
+        script_text = script.string or script.get_text("", strip=True)
+        if not script_text:
+            continue
+        try:
+            return json.loads(script_text)
+        except json.JSONDecodeError:
+            continue
+    return None
+
+
+class _AtsAdapter(ABC):
+    parser_key = "ats-provider"
+    parser_version = "1.0.0"
+    source_hosts: tuple[str, ...] = ()
+    support_markers: tuple[str, ...] = ()
+    listing_paths: tuple[tuple[str, ...], ...] = ()
+    detail_paths: tuple[tuple[str, ...], ...] = ()
+    closed_statuses = CLOSED_STATUSES
+
+    @abstractmethod
+    def _extract_records(self, payload) -> list[dict[str, object]]:
+        ...
+
+    def _supports_url(self, url: str) -> bool:
+        host = (urlsplit(url).hostname or "").lower()
+        return any(host.endswith(suffix) for suffix in self.source_hosts)
+
+    def _supports_payload(self, payload, content: str) -> bool:
+        if not payload:
+            return False
+        payload_text = str(payload).lower()
+        return any(
+            marker in payload_text or marker in content.lower() for marker in self.support_markers
+        )
+
+    def _job_is_closed(self, record: dict[str, object]) -> bool:
+        status = _first_text(record, "status", "state", "job_status", "data.status").lower()
+        if status in self.closed_statuses:
+            return True
+        active = record.get("active")
+        if isinstance(active, bool):
+            return not active
+        return False
+
+    def _coerce_url(self, raw_url: str, base_url: str) -> str:
+        if raw_url:
+            return urljoin(base_url, raw_url)
+        return base_url
+
+    def _extract_location(self, record: dict[str, object]) -> tuple[str, str, str, str]:
+        location_raw = _first_text(
+            record,
+            "location",
+            "city",
+            "cityName",
+            "place",
+            "office",
+            "location.address",
+            "address",
+            "data.location",
+            "officeLocation",
+            "locationName",
+        )
+        if not location_raw:
+            location_raw = _first_text(record, "data.city", "data.location")
+            if not location_raw:
+                location_values = []
+                for key in ("city", "address", "location", "region", "office"):
+                    location_values.extend(_collect_texts(record.get(key, "")))
+                location_raw = ", ".join(location_values)
+
+        location = location_raw
+        location_parts = [part.strip() for part in _to_text(location).split(",") if part.strip()]
+        region = _first_text(
+            record,
+            "region",
+            "data.region",
+            "location.region",
+            "address.region",
+        )
+        postal_code = _first_text(
+            record,
+            "postal_code",
+            "postalCode",
+            "location.postalCode",
+            "zipCode",
+        )
+        country = _first_text(
+            record,
+            "country",
+            "countryName",
+            "location.country",
+            "address.country",
+            "data.country",
+        )
+        return location, region, postal_code, country
+
+    def _extract_workplace(self, record: dict[str, object]) -> str:
+        workplace_raw = _first_text(
+            record,
+            "workplaceType",
+            "workplace",
+            "remoteType",
+            "remote_type",
+            "job_type",
+        ).lower()
+        if "remote" in workplace_raw:
+            return "remote" if "hybrid" not in workplace_raw else "hybrid"
+        if "hybrid" in workplace_raw:
+            return "hybrid"
+        if "onsite" in workplace_raw or "on site" in workplace_raw or "on-site" in workplace_raw:
+            return "on_site"
+        return ""
+
+    def _to_job(self, record: dict[str, object], base_url: str) -> ExtractedJob:
+        external_id = _first_text(
+            record,
+            "id",
+            "jobId",
+            "requisitionId",
+            "postingId",
+            "referenceId",
+            "positionId",
+        )
+        title = _first_text(
+            record,
+            "title",
+            "name",
+            "position",
+            "positionName",
+            "jobTitle",
+            "text",
+            "title.value",
+        )
+        raw_url = _first_text(
+            record,
+            "url",
+            "jobUrl",
+            "job_url",
+            "applyUrl",
+            "link",
+            "absoluteUrl",
+            "data.url",
+        )
+        employer_name = _first_text(
+            record,
+            "company",
+            "employer",
+            "organization",
+            "organizationName",
+            "company.name",
+            "department",
+            "hiringOrganization",
+        )
+        location_text, region, postal_code, country = self._extract_location(record)
+        description_raw = _first_text(
+            record,
+            "description",
+            "jobDescription",
+            "content",
+            "descriptionHtml",
+            "descriptionText",
+        )
+        description_text = _as_text(description_raw)
+        date_posted = _first_text(
+            record,
+            "createdAt",
+            "created",
+            "datePosted",
+            "publishedAt",
+            "published_at",
+            "created_at",
+            "jobCreated",
+        )
+        valid_through = _first_text(
+            record,
+            "closeDate",
+            "expiresAt",
+            "validThrough",
+            "valid_until",
+            "expirationDate",
+            "expiryDate",
+        )
+        employment_types = _to_list(
+            _first_text(
+                record,
+                "employmentType",
+                "employment_types",
+                "jobType",
+                "type",
+                "data.employmentType",
+            )
+        )
+        workplace_type = self._extract_workplace(record)
+
+        evidence = [
+            FieldEvidence("external_id", "ats-id", 0.95, external_id[:240]),
+            FieldEvidence("url", "ats-url", 0.9, raw_url[:240]),
+            FieldEvidence("location_text", "ats-location", 0.88, location_text[:240]),
+            FieldEvidence("date_posted", "ats-date", 0.8, date_posted[:240]),
+            FieldEvidence("valid_through", "ats-date", 0.8, valid_through[:240]),
+            FieldEvidence(
+                "employment_types",
+                "ats-employment",
+                0.9,
+                ", ".join(employment_types)[:240],
+            ),
+        ]
+
+        return ExtractedJob(
+            url=self._coerce_url(raw_url, base_url),
+            title=title,
+            employer_name=employer_name,
+            external_id=external_id,
+            location_text=location_text,
+            region=region,
+            postal_code=postal_code,
+            country=country,
+            description_html=description_raw,
+            description_text=description_text,
+            date_posted=date_posted,
+            valid_through=valid_through,
+            employment_types=employment_types,
+            workplace_type=workplace_type,
+            raw=record,
+            evidence=evidence,
+        )
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        if not self._supports_url(url):
+            return ExtractionResult(
+                [],
+                self.parser_key,
+                self.parser_version,
+                0.0,
+                ["Onherkenbare ATS-host"],
+            )
+
+        payload = _extract_payload(content)
+        if not payload:
+            return ExtractionResult(
+                [],
+                self.parser_key,
+                self.parser_version,
+                0.0,
+                ["Geen parseerbare ATS-response"],
+            )
+
+        if not self._supports_payload(payload, content):
+            return ExtractionResult(
+                [],
+                self.parser_key,
+                self.parser_version,
+                0.0,
+                ["Geen herkenbare ATS-markup voor deze adapter"],
+            )
+
+        jobs: list[ExtractedJob] = []
+        for record in self._extract_records(payload):
+            if not isinstance(record, dict):
+                continue
+            if self._job_is_closed(record):
+                continue
+            job = self._to_job(record, base_url=url)
+            if job.title:
+                jobs.append(job)
+
+        warnings: list[str] = []
+        if not jobs:
+            warnings.append("Geen actieve ATS-vacatures gevonden")
+        return ExtractionResult(jobs, self.parser_key, self.parser_version, 0.9 if jobs else 0.0, warnings)
+
+
+class GreenhouseAdapter(_AtsAdapter):
+    parser_key = "ats-greenhouse"
+    source_hosts = ("greenhouse.io", "boards.greenhouse.io")
+    support_markers = ("greenhouse", "job board", "jobboard")
+    listing_paths = (("jobs",), ("data", "jobs"), ("data", "results"))
+    detail_paths = (("job",), ("data", "job"), ("result", "job"), ("result", "position"))
+    closed_statuses = CLOSED_STATUSES | {"published", "draft", "deleted"}
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+            if isinstance(value, dict) and "results" in value:
+                possible = value.get("results")
+                if isinstance(possible, list):
+                    return [item for item in possible if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []
+
+
+class LeverAdapter(_AtsAdapter):
+    parser_key = "ats-lever"
+    source_hosts = ("jobs.lever.co",)
+    support_markers = ("lever", "requisition", "posting")
+    listing_paths = (("data",), ("jobs",), ("results",))
+    detail_paths = (("data",), ("job",), ("position",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"archived", "deleted"}
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []
+
+
+class RecruiteeAdapter(_AtsAdapter):
+    parser_key = "ats-recruitee"
+    source_hosts = ("recruitee.com",)
+    support_markers = ("recruitee", "career", "vacancy")
+    listing_paths = (("jobs",), ("data", "jobs"), ("vacancies",))
+    detail_paths = (("job",), ("data", "job"), ("vacancy",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"hidden", "paused"}
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []
+
+
+class SmartRecruitersAdapter(_AtsAdapter):
+    parser_key = "ats-smartrecruiters"
+    source_hosts = ("smartrecruiters.com",)
+    support_markers = ("smartrecruiters", "smart recruiter")
+    listing_paths = (("jobs",), ("data", "jobs"), ("results",))
+    detail_paths = (("job",), ("data", "job"), ("posting",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"unpublished", "expired"}
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []
+
+
+class WorkableAdapter(_AtsAdapter):
+    parser_key = "ats-workable"
+    source_hosts = ("apply.workable.com",)
+    support_markers = ("workable", "workable job")
+    listing_paths = (("jobs",), ("data", "jobs"), ("results",))
+    detail_paths = (("job",), ("data", "job"), ("position",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"draft", "inactive"}
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []

apps/sources/adapters/base.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+
+@dataclass(slots=True)
+class FieldEvidence:
+    field_name: str
+    method: str
+    confidence: float
+    evidence: str = ""
+
+
+@dataclass(slots=True)
+class ExtractedJob:
+    url: str
+    title: str
+    employer_name: str = ""
+    external_id: str = ""
+    location_text: str = ""
+    region: str = ""
+    postal_code: str = ""
+    country: str = ""
+    description_html: str = ""
+    description_text: str = ""
+    language: str = ""
+    date_posted: str = ""
+    valid_through: str = ""
+    employment_types: list[str] = field(default_factory=list)
+    workplace_type: str = ""
+    compensation: dict[str, Any] = field(default_factory=dict)
+    skills_required: list[str] = field(default_factory=list)
+    skills_preferred: list[str] = field(default_factory=list)
+    raw: dict[str, Any] = field(default_factory=dict)
+    evidence: list[FieldEvidence] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class ExtractionResult:
+    jobs: list[ExtractedJob]
+    parser_key: str
+    parser_version: str
+    confidence: float
+    warnings: list[str] = field(default_factory=list)
+
+
+class SourceAdapter(Protocol):
+    parser_key: str
+    parser_version: str
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult: ...

apps/sources/adapters/email_alert.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import html
+import re
+from email import policy
+from email.message import Message
+from email.parser import BytesParser
+from urllib.parse import urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+
+from apps.sources.services.canonicalize import canonicalize_url
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I)
+SKIP_TEXT = re.compile(r"unsubscribe|afmelden|uitschrijven|privacy|view in browser", re.I)
+
+
+class EmailAlertAdapter:
+    parser_key = "email-alert"
+    parser_version = "1.0.0"
+
+    @staticmethod
+    def _decode_parts(message: Message) -> tuple[str, str]:
+        plain_parts: list[str] = []
+        html_parts: list[str] = []
+        for part in message.walk() if message.is_multipart() else [message]:
+            disposition = str(part.get("Content-Disposition") or "")
+            if "attachment" in disposition.lower():
+                continue
+            content_type = part.get_content_type()
+            try:
+                payload = part.get_content()
+            except Exception:
+                raw = part.get_payload(decode=True) or b""
+                payload = raw.decode(part.get_content_charset() or "utf-8", errors="replace")
+            if content_type == "text/plain":
+                plain_parts.append(str(payload))
+            elif content_type == "text/html":
+                html_parts.append(str(payload))
+        return "\n".join(plain_parts), "\n".join(html_parts)
+
+    def extract_message(self, raw_message: bytes) -> ExtractionResult:
+        message = BytesParser(policy=policy.default).parsebytes(raw_message)
+        plain, html_body = self._decode_parts(message)
+        candidates: list[tuple[str, str]] = []
+        if html_body:
+            soup = BeautifulSoup(html_body, "lxml")
+            for anchor in soup.find_all("a", href=True):
+                label = " ".join(anchor.get_text(" ", strip=True).split())
+                href = html.unescape(str(anchor["href"]).strip())
+                if not href.lower().startswith(("http://", "https://")):
+                    continue
+                if SKIP_TEXT.search(label) or SKIP_TEXT.search(href):
+                    continue
+                candidates.append((label, href))
+        for href in URL_RE.findall(plain):
+            clean_href = href.rstrip(".,);]")
+            if SKIP_TEXT.search(clean_href):
+                continue
+            candidates.append(("", clean_href))
+
+        jobs: list[ExtractedJob] = []
+        seen: set[str] = set()
+        subject = str(message.get("subject") or "Vacature uit e-mail").strip()
+        for label, href in candidates:
+            canonical = canonicalize_url(urljoin("https://invalid.local/", href))
+            if not canonical or canonical in seen:
+                continue
+            seen.add(canonical)
+            hostname = urlsplit(canonical).hostname or ""
+            title = label if len(label) >= 4 else subject
+            if len(title) > 300:
+                title = title[:300]
+            jobs.append(
+                ExtractedJob(
+                    url=canonical,
+                    title=title,
+                    employer_name="",
+                    description_text=plain[:5000]
+                    or BeautifulSoup(html_body, "lxml").get_text("\n", strip=True)[:5000],
+                    raw={
+                        "email_subject": subject,
+                        "email_sender": str(message.get("from") or ""),
+                        "target_domain": hostname,
+                    },
+                    evidence=[FieldEvidence("url", "email-anchor", 0.75, label[:240])],
+                )
+            )
+        return ExtractionResult(
+            jobs,
+            self.parser_key,
+            self.parser_version,
+            0.65 if jobs else 0.0,
+            [] if jobs else ["Geen vacaturelinks in e-mail gevonden"],
+        )
+
+    def extract(self, content: str, *, url: str = "") -> ExtractionResult:
+        return self.extract_message(content.encode("utf-8", errors="replace"))

apps/sources/adapters/generic_html.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+LABEL_PATTERNS = {
+    "location": re.compile(r"^(locatie|location|lieu|plaats|standplaats)\s*:?$", re.I),
+    "employer": re.compile(r"^(werkgever|employer|company|organisatie|société)\s*:?$", re.I),
+}
+
+
+class GenericHtmlAdapter:
+    parser_key = "generic-html"
+    parser_version = "1.0.0"
+
+    @staticmethod
+    def _meta(soup: BeautifulSoup, *names: str) -> str:
+        for name in names:
+            tag = soup.find("meta", attrs={"property": name}) or soup.find(
+                "meta", attrs={"name": name}
+            )
+            if tag and tag.get("content"):
+                return str(tag["content"]).strip()
+        return ""
+
+    @staticmethod
+    def _label_value(soup: BeautifulSoup, pattern: re.Pattern[str]) -> str:
+        label = soup.find(string=lambda value: bool(value and pattern.match(value.strip())))
+        if not label:
+            return ""
+        parent = label.parent
+        if not parent:
+            return ""
+        sibling = parent.find_next_sibling()
+        if sibling:
+            return sibling.get_text(" ", strip=True)
+        text = parent.get_text(" ", strip=True)
+        return pattern.sub("", text).strip(" :-")
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        soup = BeautifulSoup(content, "lxml")
+        for element in soup(["script", "style", "noscript", "template"]):
+            element.decompose()
+        h1 = soup.find("h1")
+        title = (h1.get_text(" ", strip=True) if h1 else "") or self._meta(
+            soup, "og:title", "twitter:title"
+        )
+        if not title and soup.title:
+            title = soup.title.get_text(" ", strip=True)
+        title = re.sub(r"\s+[|\\u2013\\u2014-]\s+.*$", "", title).strip()
+        if not title:
+            return ExtractionResult([], self.parser_key, self.parser_version, 0.0, ["Geen titel"])
+
+        employer = self._meta(soup, "og:site_name", "application-name") or self._label_value(
+            soup, LABEL_PATTERNS["employer"]
+        )
+        location = self._label_value(soup, LABEL_PATTERNS["location"])
+        main = soup.find("main") or soup.find("article") or soup.body
+        description_html = str(main) if main else ""
+        description_text = (
+            main.get_text("\n", strip=True) if main else soup.get_text("\n", strip=True)
+        )
+        canonical = soup.find("link", rel=lambda value: value and "canonical" in value)
+        job_url = (
+            urljoin(url, canonical.get("href")) if canonical and canonical.get("href") else url
+        )
+        evidence = [
+            FieldEvidence("title", "html-heading", 0.78, title[:240]),
+            FieldEvidence("employer_name", "html-meta-or-label", 0.65, employer[:240]),
+            FieldEvidence("location_text", "html-label", 0.62, location[:240]),
+            FieldEvidence("description", "html-main", 0.70, description_text[:300]),
+        ]
+        job = ExtractedJob(
+            url=job_url,
+            title=title,
+            employer_name=employer,
+            location_text=location,
+            description_html=description_html,
+            description_text=description_text,
+            raw={"generic_html": True},
+            evidence=evidence,
+        )
+        return ExtractionResult([job], self.parser_key, self.parser_version, 0.70)

apps/sources/adapters/jsonld.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from typing import Any
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class JsonLdJobPostingAdapter:
+    parser_key = "jsonld-jobposting"
+    parser_version = "1.0.0"
+
+    @staticmethod
+    def _is_jobposting(value: Any) -> bool:
+        types = value if isinstance(value, list) else [value]
+        return any(str(item).lower() == "jobposting" for item in types)
+
+    def _walk(self, value: Any) -> Iterable[dict[str, Any]]:
+        if isinstance(value, dict):
+            if self._is_jobposting(value.get("@type")):
+                yield value
+            graph = value.get("@graph")
+            if graph is not None:
+                yield from self._walk(graph)
+            for child in value.values():
+                if isinstance(child, dict | list):
+                    yield from self._walk(child)
+        elif isinstance(value, list):
+            for child in value:
+                yield from self._walk(child)
+
+    @staticmethod
+    def _name(value: Any) -> str:
+        if isinstance(value, str):
+            return value.strip()
+        if isinstance(value, dict):
+            return str(value.get("name") or value.get("legalName") or "").strip()
+        return ""
+
+    @staticmethod
+    def _location(value: Any) -> tuple[str, str, str, str]:
+        locations = value if isinstance(value, list) else [value]
+        parts: list[str] = []
+        region = postal = country = ""
+        for location in locations:
+            if not isinstance(location, dict):
+                continue
+            address = location.get("address", location)
+            if isinstance(address, str):
+                parts.append(address)
+                continue
+            if not isinstance(address, dict):
+                continue
+            locality = str(address.get("addressLocality") or "").strip()
+            region = region or str(address.get("addressRegion") or "").strip()
+            postal = postal or str(address.get("postalCode") or "").strip()
+            country_value = address.get("addressCountry")
+            country = (
+                country
+                or JsonLdJobPostingAdapter._name(country_value)
+                or str(country_value or "").strip()
+            )
+            label = ", ".join(part for part in [locality, region, postal, country] if part)
+            if label:
+                parts.append(label)
+        return " | ".join(dict.fromkeys(parts)), region, postal, country
+
+    @staticmethod
+    def _employment_types(value: Any) -> list[str]:
+        if isinstance(value, list):
+            return [str(item).strip() for item in value if str(item).strip()]
+        if value:
+            return [str(value).strip()]
+        return []
+
+    @staticmethod
+    def _identifier(value: Any) -> str:
+        if isinstance(value, dict):
+            return str(value.get("value") or value.get("name") or "").strip()
+        return str(value or "").strip()
+
+    @staticmethod
+    def _salary(value: Any) -> dict[str, Any]:
+        if not isinstance(value, dict):
+            return {}
+        result: dict[str, Any] = {}
+        currency = value.get("currency")
+        if currency:
+            result["currency"] = currency
+        raw_value = value.get("value")
+        if isinstance(raw_value, dict):
+            for key in ("minValue", "maxValue", "value", "unitText"):
+                if key in raw_value:
+                    result[key] = raw_value[key]
+        elif raw_value is not None:
+            result["value"] = raw_value
+        return result
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        soup = BeautifulSoup(content, "lxml")
+        records: list[dict[str, Any]] = []
+        warnings: list[str] = []
+        for script in soup.find_all(
+            "script", attrs={"type": lambda value: value and "ld+json" in value}
+        ):
+            raw = script.string or script.get_text("", strip=True)
+            if not raw:
+                continue
+            try:
+                payload = json.loads(raw)
+            except json.JSONDecodeError:
+                warnings.append("Ongeldige JSON-LD overgeslagen")
+                continue
+            records.extend(self._walk(payload))
+
+        jobs: list[ExtractedJob] = []
+        for record in records:
+            title = str(record.get("title") or record.get("name") or "").strip()
+            if not title:
+                warnings.append("JobPosting zonder titel overgeslagen")
+                continue
+            description_html = str(record.get("description") or "").strip()
+            description_text = BeautifulSoup(description_html, "lxml").get_text("\n", strip=True)
+            employer = self._name(record.get("hiringOrganization"))
+            location, region, postal, country = self._location(record.get("jobLocation"))
+            workplace_type = str(record.get("jobLocationType") or "").strip()
+            if not location and record.get("applicantLocationRequirements"):
+                location, region, postal, country = self._location(
+                    record.get("applicantLocationRequirements")
+                )
+            job_url = str(record.get("url") or url).strip()
+            job_url = urljoin(url, job_url)
+            evidence = [
+                FieldEvidence("title", "jsonld", 0.98, title[:240]),
+                FieldEvidence("employer_name", "jsonld", 0.95, employer[:240]),
+                FieldEvidence("location_text", "jsonld", 0.92, location[:240]),
+                FieldEvidence("description", "jsonld", 0.95, description_text[:300]),
+            ]
+            jobs.append(
+                ExtractedJob(
+                    url=job_url,
+                    title=title,
+                    employer_name=employer,
+                    external_id=self._identifier(record.get("identifier")),
+                    location_text=location,
+                    region=region,
+                    postal_code=postal,
+                    country=country,
+                    description_html=description_html,
+                    description_text=description_text,
+                    language=str(record.get("inLanguage") or "").strip(),
+                    date_posted=str(record.get("datePosted") or "").strip(),
+                    valid_through=str(record.get("validThrough") or "").strip(),
+                    employment_types=self._employment_types(record.get("employmentType")),
+                    workplace_type=workplace_type,
+                    compensation=self._salary(record.get("baseSalary")),
+                    raw=record,
+                    evidence=evidence,
+                )
+            )
+        confidence = 0.95 if jobs else 0.0
+        return ExtractionResult(jobs, self.parser_key, self.parser_version, confidence, warnings)

apps/sources/adapters/registry.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from apps.sources.models import RawDocument
+
+from .base import ExtractionResult
+from .ats import (
+    GreenhouseAdapter,
+    LeverAdapter,
+    RecruiteeAdapter,
+    SmartRecruitersAdapter,
+    WorkableAdapter,
+)
+from .generic_html import GenericHtmlAdapter
+from .jsonld import JsonLdJobPostingAdapter
+from .rss import RssAdapter
+
+
+class AdapterRegistry:
+    def __init__(self) -> None:
+        self.greenhouse = GreenhouseAdapter()
+        self.lever = LeverAdapter()
+        self.recruitee = RecruiteeAdapter()
+        self.smartrecruiters = SmartRecruitersAdapter()
+        self.workable = WorkableAdapter()
+        self.jsonld = JsonLdJobPostingAdapter()
+        self.generic = GenericHtmlAdapter()
+        self.rss = RssAdapter()
+        self.providers = [
+            self.greenhouse,
+            self.lever,
+            self.recruitee,
+            self.smartrecruiters,
+            self.workable,
+        ]
+
+    def extract(self, document: RawDocument) -> ExtractionResult:
+        content = document.body_text
+        url = document.final_url or document.url
+        content_type = (document.content_type or "").lower()
+        if document.kind == RawDocument.Kind.XML or "rss" in content_type or "atom" in content_type:
+            return self.rss.extract(content, url=url)
+
+        for provider in self.providers:
+            if provider._supports_url(url):
+                result = provider.extract(content, url=url)
+                if any(
+                    msg in result.warnings
+                    for msg in ("Geen parseerbare ATS-response", "Geen herkenbare ATS-markup voor deze adapter")
+                ):
+                    continue
+                return result
+
+        jsonld_result = self.jsonld.extract(content, url=url)
+        if jsonld_result.jobs:
+            return jsonld_result
+        generic_result = self.generic.extract(content, url=url)
+        generic_result.warnings = jsonld_result.warnings + generic_result.warnings
+        return generic_result
+
+
+registry = AdapterRegistry()

apps/sources/adapters/rss.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from datetime import datetime
+from time import mktime
+
+import feedparser
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class RssAdapter:
+    parser_key = "rss-atom"
+    parser_version = "1.0.0"
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        feed = feedparser.parse(content)
+        jobs: list[ExtractedJob] = []
+        for entry in feed.entries:
+            title = str(entry.get("title") or "").strip()
+            link = str(entry.get("link") or "").strip()
+            if not title or not link:
+                continue
+            summary = str(entry.get("summary") or entry.get("description") or "")
+            text = BeautifulSoup(summary, "lxml").get_text("\n", strip=True)
+            published = ""
+            if entry.get("published_parsed"):
+                published = datetime.fromtimestamp(mktime(entry.published_parsed)).isoformat()
+            jobs.append(
+                ExtractedJob(
+                    url=link,
+                    title=title,
+                    external_id=str(entry.get("id") or ""),
+                    description_html=summary,
+                    description_text=text,
+                    date_posted=published,
+                    raw=dict(entry),
+                    evidence=[
+                        FieldEvidence("title", "rss", 0.88, title[:240]),
+                        FieldEvidence("description", "rss", 0.80, text[:300]),
+                    ],
+                )
+            )
+        warnings: list[str] = []
+        if getattr(feed, "bozo", False):
+            warnings.append(str(getattr(feed, "bozo_exception", "Ongeldige feed")))
+        return ExtractionResult(
+            jobs,
+            self.parser_key,
+            self.parser_version,
+            0.82 if jobs else 0.0,
+            warnings,
+        )

apps/sources/admin.py

@@ -0,0 +1,87 @@
+from django.contrib import admin
+
+from .models import (
+    EmailMessageRecord,
+    RawDocument,
+    Source,
+    SourceLease,
+    SourceOriginState,
+    SourcePolicyReview,
+    SourceRobotsCache,
+    SourceRun,
+)
+
+
+@admin.register(Source)
+class SourceAdmin(admin.ModelAdmin):
+    list_display = (
+        "name",
+        "domain",
+        "source_type",
+        "status",
+        "policy",
+        "last_success_at",
+        "failure_count",
+        "next_run_at",
+    )
+    list_filter = ("source_type", "status", "policy", "strict_mode")
+    search_fields = ("name", "domain", "base_url")
+
+
+@admin.register(SourceRun)
+class SourceRunAdmin(admin.ModelAdmin):
+    list_display = ("source", "status", "started_at", "finished_at", "extracted_count")
+    list_filter = ("status", "error_category")
+    readonly_fields = [field.name for field in SourceRun._meta.fields]
+
+
+@admin.register(SourcePolicyReview)
+class SourcePolicyReviewAdmin(admin.ModelAdmin):
+    list_display = (
+        "source",
+        "decision",
+        "scope",
+        "actor",
+        "expires_at",
+        "created_at",
+    )
+    list_filter = ("decision", "scope", "expires_at")
+    search_fields = ("source__name", "source__domain", "actor__username", "reason")
+    readonly_fields = [field.name for field in SourcePolicyReview._meta.fields]
+
+
+@admin.register(SourceRobotsCache)
+class SourceRobotsCacheAdmin(admin.ModelAdmin):
+    list_display = ("origin", "expires_at", "byte_length", "updated_at", "error")
+    list_filter = ("error",)
+    search_fields = ("origin",)
+    readonly_fields = [field.name for field in SourceRobotsCache._meta.fields]
+
+
+@admin.register(SourceLease)
+class SourceLeaseAdmin(admin.ModelAdmin):
+    list_display = ("source", "token", "worker_id", "expires_at", "updated_at")
+    search_fields = ("source__name", "source__domain", "worker_id")
+    readonly_fields = [field.name for field in SourceLease._meta.fields]
+
+
+@admin.register(SourceOriginState)
+class SourceOriginStateAdmin(admin.ModelAdmin):
+    list_display = ("domain", "next_allowed_at", "updated_at")
+    search_fields = ("domain",)
+    readonly_fields = [field.name for field in SourceOriginState._meta.fields]
+
+
+@admin.register(RawDocument)
+class RawDocumentAdmin(admin.ModelAdmin):
+    list_display = ("source", "kind", "http_status", "byte_length", "quarantined", "created_at")
+    list_filter = ("kind", "quarantined", "http_status")
+    search_fields = ("url", "final_url", "content_hash")
+    readonly_fields = [field.name for field in RawDocument._meta.fields]
+
+
+@admin.register(EmailMessageRecord)
+class EmailMessageRecordAdmin(admin.ModelAdmin):
+    list_display = ("subject", "sender", "received_at", "processed", "created_at")
+    list_filter = ("processed", "mailbox")
+    search_fields = ("subject", "sender", "message_id")

apps/sources/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class SourcesConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "apps.sources"
+    verbose_name = "Bronnen"

apps/sources/forms.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from django import forms
+
+
+class ManualImportForm(forms.Form):
+    source_url = forms.URLField(
+        required=False,
+        label="Vacature-URL",
+        max_length=1000,
+        widget=forms.URLInput(attrs={"autocomplete": "off"}),
+    )
+    pasted_text = forms.CharField(
+        required=False,
+        label="Tekst plakken",
+        widget=forms.Textarea(
+            attrs={
+                "rows": 6,
+                "placeholder": "Plak hier de vacaturetekst of relevante pagina-inhoud.",
+            }
+        ),
+    )
+
+    def clean(self):
+        data = super().clean()
+        source_url = (data.get("source_url") or "").strip()
+        pasted_text = (data.get("pasted_text") or "").strip()
+        if not source_url and not pasted_text:
+            raise forms.ValidationError("Vul een URL of een tekstfragment in.")
+        return {"source_url": source_url, "pasted_text": pasted_text}

apps/sources/management/commands/discover_sources.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from django.core.management.base import BaseCommand, CommandError
+
+from apps.sources.services.discovery import (
+    discover_from_email,
+    discover_from_feed,
+    discover_from_html,
+    discover_from_sitemap,
+    persist_discovery_candidates,
+)
+
+
+class Command(BaseCommand):
+    help = (
+        "Voert bronontdekking uit op lokaal aangeleverde content "
+        "en slaat alleen kandidaatregels op met policy review."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "mode",
+            choices=["html", "sitemap", "feed", "email"],
+            help="Bronmodus voor ontdekking",
+        )
+        parser.add_argument(
+            "--input",
+            required=True,
+            dest="input_path",
+            help="Pad naar inputbestand",
+        )
+        parser.add_argument(
+            "--base-url",
+            dest="base_url",
+            help="Base URL voor relativiteitsresolutie",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Geen database-write; toon alleen ontdekte kandidaten",
+        )
+
+    def handle(self, *args, **options):
+        mode = options["mode"]
+        input_path = Path(options["input_path"]).resolve()
+        base_url = (options["base_url"] or "").strip()
+        if mode in {"html", "sitemap", "feed"} and not base_url:
+            raise CommandError("--base-url is verplicht voor html/sitemap/feed")
+
+        if not input_path.is_file():
+            raise CommandError(f"Inputbestand niet gevonden: {input_path}")
+
+        if mode in {"html", "sitemap", "feed"}:
+            content = input_path.read_text(encoding="utf-8")
+        else:
+            content = input_path.read_bytes()
+
+        if mode == "html":
+            candidates = discover_from_html(content, base_url=base_url)
+        elif mode == "sitemap":
+            candidates = discover_from_sitemap(content, base_url=base_url)
+        elif mode == "feed":
+            candidates = discover_from_feed(content, base_url=base_url)
+        else:
+            candidates = discover_from_email(content)
+
+        self.stdout.write(f"Ontdekt {len(candidates)} kandidaten (mode={mode}).")
+        if not candidates:
+            self.stdout.write(self.style.WARNING("Geen bruikbare kandidaten gevonden."))
+            return
+
+        if options["dry_run"]:
+            self.stdout.write(self.style.WARNING("Dry-run modus: er wordt niet geschreven."))
+            for candidate in candidates:
+                self.stdout.write(f"- {candidate.source_type} {candidate.url} [{candidate.reason}]")
+            return
+
+        created, updated, skipped = persist_discovery_candidates(candidates)
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Persist result: {created} nieuw, {updated} bijgewerkt, {skipped} ongewijzigd."
+            )
+        )

apps/sources/management/commands/import_job_fixture.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import timedelta
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+
+from apps.jobs.services.pipeline import process_raw_document
+from apps.sources.models import RawDocument, Source, SourceRun
+
+
+class Command(BaseCommand):
+    help = "Importeert een lokale HTML/XML-fixture via dezelfde pipeline als een live bron."
+
+    def add_arguments(self, parser):
+        parser.add_argument("path")
+        parser.add_argument("--url", required=True)
+        parser.add_argument("--source-name", default="Lokale fixture")
+        parser.add_argument("--source-type", default=Source.Type.EMPLOYER)
+
+    def handle(self, *args, **options):
+        path = Path(options["path"]).resolve()
+        if not path.is_file():
+            raise CommandError(f"Bestand bestaat niet: {path}")
+        content = path.read_text(encoding="utf-8")
+        hostname = (urlsplit(options["url"]).hostname or "").lower()
+        if not hostname:
+            raise CommandError("--url bevat geen geldig domein")
+        source, _ = Source.objects.get_or_create(
+            domain=hostname,
+            source_type=options["source_type"],
+            defaults={
+                "name": options["source_name"],
+                "base_url": options["url"],
+                "status": Source.Status.ACTIVE,
+                "policy": Source.Policy.ALLOW,
+                "parser_key": "auto",
+            },
+        )
+        run = SourceRun.objects.create(source=source)
+        suffix = path.suffix.lower()
+        kind = RawDocument.Kind.XML if suffix in {".xml", ".rss"} else RawDocument.Kind.HTML
+        document = RawDocument.objects.create(
+            source=source,
+            source_run=run,
+            url=options["url"],
+            final_url=options["url"],
+            kind=kind,
+            content_type="application/xml" if kind == RawDocument.Kind.XML else "text/html",
+            content_hash=hashlib.sha256(content.encode()).hexdigest(),
+            body_text=content,
+            byte_length=len(content.encode()),
+            retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
+        )
+        metrics = process_raw_document(document)
+        run.finish(
+            SourceRun.Status.SUCCESS,
+            extracted_count=int(metrics["extracted"]),
+            created_count=int(metrics["created"]),
+            updated_count=int(metrics["updated"]),
+            duplicate_count=int(metrics["duplicates"]),
+            metrics=metrics,
+        )
+        self.stdout.write(self.style.SUCCESS(str(metrics)))

apps/sources/management/commands/seed_sources.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import yaml
+from django.core.management.base import BaseCommand, CommandError
+
+from apps.sources.models import Source
+from apps.sources.services.policy import is_denied_domain
+
+
+class Command(BaseCommand):
+    help = "Laadt gecontroleerde bronseeds uit YAML."
+
+    def add_arguments(self, parser):
+        parser.add_argument("path", nargs="?", default="config-data/seed_sources.yaml")
+
+    def handle(self, *args, **options):
+        path = Path(options["path"])
+        if not path.is_file():
+            raise CommandError(f"Seedbestand ontbreekt: {path}")
+        payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+        created = updated = 0
+        for item in payload.get("sources", []):
+            url = str(item.get("url") or "").strip()
+            domain = (urlsplit(url).hostname or "").lower()
+            if not domain:
+                self.stderr.write(f"Overgeslagen zonder domein: {item}")
+                continue
+            deny = is_denied_domain(domain)
+            source, was_created = Source.objects.update_or_create(
+                domain=domain,
+                source_type=item.get("type", Source.Type.EMPLOYER),
+                defaults={
+                    "name": item.get("name") or domain,
+                    "base_url": url,
+                    "status": Source.Status.PAUSED
+                    if deny
+                    else item.get("status", Source.Status.TRIAL),
+                    "policy": Source.Policy.DENY
+                    if deny
+                    else item.get("policy", Source.Policy.REVIEW),
+                    "policy_reason": "Standaard platformdenylist"
+                    if deny
+                    else item.get("policy_reason", ""),
+                    "parser_key": item.get("parser", "auto"),
+                    "crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)),
+                },
+            )
+            created += int(was_created)
+            updated += int(not was_created)
+        self.stdout.write(
+            self.style.SUCCESS(f"Bronnen: {created} aangemaakt, {updated} bijgewerkt")
+        )

apps/sources/migrations/0001_initial.py

@@ -0,0 +1,136 @@
+# Generated by Django 5.2.16 on 2026-07-20 23:57
+
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RawDocument',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('url', models.URLField(blank=True, max_length=2000)),
+                ('final_url', models.URLField(blank=True, max_length=2000)),
+                ('kind', models.CharField(choices=[('html', 'HTML'), ('xml', 'XML'), ('email', 'E-mail'), ('json', 'JSON'), ('text', 'Tekst')], max_length=16)),
+                ('content_type', models.CharField(blank=True, max_length=200)),
+                ('http_status', models.PositiveSmallIntegerField(blank=True, null=True)),
+                ('response_headers', models.JSONField(blank=True, default=dict)),
+                ('content_hash', models.CharField(db_index=True, max_length=64)),
+                ('body_text', models.TextField(blank=True)),
+                ('byte_length', models.PositiveIntegerField(default=0)),
+                ('parser_key', models.CharField(blank=True, max_length=120)),
+                ('parser_version', models.CharField(blank=True, max_length=80)),
+                ('extraction_confidence', models.DecimalField(decimal_places=3, default=0, max_digits=4)),
+                ('retain_until', models.DateTimeField(blank=True, null=True)),
+                ('quarantined', models.BooleanField(default=False)),
+                ('quarantine_reason', models.CharField(blank=True, max_length=500)),
+                ('metadata', models.JSONField(blank=True, default=dict)),
+            ],
+            options={
+                'ordering': ['-created_at'],
+            },
+        ),
+        migrations.CreateModel(
+            name='EmailMessageRecord',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('message_id', models.CharField(max_length=998, unique=True)),
+                ('mailbox', models.CharField(default='INBOX', max_length=255)),
+                ('sender', models.CharField(blank=True, max_length=500)),
+                ('subject', models.CharField(blank=True, max_length=998)),
+                ('received_at', models.DateTimeField(blank=True, null=True)),
+                ('links', models.JSONField(blank=True, default=list)),
+                ('processed', models.BooleanField(default=False)),
+                ('error_message', models.CharField(blank=True, max_length=1000)),
+                ('raw_document', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='email_record', to='sources.rawdocument')),
+            ],
+            options={
+                'ordering': ['-received_at', '-created_at'],
+            },
+        ),
+        migrations.CreateModel(
+            name='Source',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(max_length=200)),
+                ('source_type', models.CharField(choices=[('employer', 'Werkgeverspagina'), ('ats', 'Publieke ATS-pagina'), ('rss', 'RSS/Atom'), ('sitemap', 'Sitemap'), ('email', 'Vacaturemail'), ('manual', 'Handmatige import')], max_length=24)),
+                ('base_url', models.URLField(blank=True, max_length=1000)),
+                ('domain', models.CharField(db_index=True, max_length=255)),
+                ('status', models.CharField(choices=[('candidate', 'Kandidaat'), ('trial', 'Proefrun'), ('active', 'Actief'), ('quarantined', 'Quarantaine'), ('paused', 'Gepauzeerd'), ('disabled', 'Uitgeschakeld')], default='candidate', max_length=24)),
+                ('policy', models.CharField(choices=[('allow', 'Toestaan'), ('review', 'Te beoordelen'), ('deny', 'Blokkeren')], default='review', max_length=16)),
+                ('policy_reason', models.CharField(blank=True, max_length=500)),
+                ('parser_key', models.CharField(default='auto', max_length=120)),
+                ('strict_mode', models.BooleanField(default=True)),
+                ('allow_public_endpoint', models.BooleanField(default=False)),
+                ('honor_robots', models.BooleanField(default=True)),
+                ('crawl_interval_minutes', models.PositiveIntegerField(default=720)),
+                ('minimum_interval_seconds', models.PositiveIntegerField(default=30)),
+                ('max_concurrency', models.PositiveSmallIntegerField(default=1)),
+                ('next_run_at', models.DateTimeField(blank=True, db_index=True, null=True)),
+                ('last_success_at', models.DateTimeField(blank=True, null=True)),
+                ('last_failure_at', models.DateTimeField(blank=True, null=True)),
+                ('failure_count', models.PositiveIntegerField(default=0)),
+                ('etag', models.CharField(blank=True, max_length=500)),
+                ('last_modified', models.CharField(blank=True, max_length=500)),
+                ('robots_checked_at', models.DateTimeField(blank=True, null=True)),
+                ('terms_checked_at', models.DateTimeField(blank=True, null=True)),
+                ('metadata', models.JSONField(blank=True, default=dict)),
+            ],
+            options={
+                'ordering': ['name'],
+                'constraints': [models.UniqueConstraint(fields=('domain', 'source_type'), name='unique_domain_source_type')],
+            },
+        ),
+        migrations.AddField(
+            model_name='rawdocument',
+            name='source',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='sources.source'),
+        ),
+        migrations.CreateModel(
+            name='SourceRun',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('status', models.CharField(choices=[('running', 'Bezig'), ('success', 'Geslaagd'), ('partial', 'Gedeeltelijk'), ('failed', 'Mislukt'), ('skipped', 'Overgeslagen')], default='running', max_length=16)),
+                ('started_at', models.DateTimeField(default=django.utils.timezone.now)),
+                ('finished_at', models.DateTimeField(blank=True, null=True)),
+                ('http_status', models.PositiveSmallIntegerField(blank=True, null=True)),
+                ('discovered_count', models.PositiveIntegerField(default=0)),
+                ('extracted_count', models.PositiveIntegerField(default=0)),
+                ('created_count', models.PositiveIntegerField(default=0)),
+                ('updated_count', models.PositiveIntegerField(default=0)),
+                ('duplicate_count', models.PositiveIntegerField(default=0)),
+                ('error_category', models.CharField(blank=True, max_length=80)),
+                ('error_message', models.CharField(blank=True, max_length=1000)),
+                ('metrics', models.JSONField(blank=True, default=dict)),
+                ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='runs', to='sources.source')),
+            ],
+            options={
+                'ordering': ['-started_at'],
+            },
+        ),
+        migrations.AddField(
+            model_name='rawdocument',
+            name='source_run',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='sources.sourcerun'),
+        ),
+        migrations.AddIndex(
+            model_name='rawdocument',
+            index=models.Index(fields=['source', 'content_hash'], name='sources_raw_source__4a7b2a_idx'),
+        ),
+    ]

apps/sources/migrations/0002_policy_review_and_robots_cache.py

@@ -0,0 +1,51 @@
+# Generated by Django 5.2.16 on 2026-07-21 for VR-102
+
+import django.conf
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sources", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="SourcePolicyReview",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("scope", models.CharField(choices=[("source", "Bronspecifiek"), ("domain", "Domeinbrede controle")], default="source", max_length=20)),
+                ("decision", models.CharField(choices=[("allow", "Toestaan"), ("trial", "Proefrun"), ("pause", "Pauzeren"), ("deny", "Blokkeren"),], max_length=16)),
+                ("reason", models.CharField(max_length=500)),
+                ("evidence_link", models.URLField(blank=True, max_length=500)),
+                ("notes", models.TextField(blank=True)),
+                ("expires_at", models.DateTimeField(blank=True, null=True)),
+                ("metadata", models.JSONField(blank=True, default=dict)),
+                ("actor", models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="policy_reviews", to=django.conf.settings.AUTH_USER_MODEL)),
+                ("source", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name="policy_reviews", to="sources.source")),
+            ],
+            options={"ordering": ["-created_at"]},
+        ),
+        migrations.CreateModel(
+            name="SourceRobotsCache",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("origin", models.CharField(max_length=255, unique=True)),
+                ("expires_at", models.DateTimeField()),
+                ("etag", models.CharField(blank=True, max_length=255)),
+                ("last_modified", models.CharField(blank=True, max_length=255)),
+                ("allow_rules", models.JSONField(blank=True, default=dict)),
+                ("disallow_rules", models.JSONField(blank=True, default=dict)),
+                ("error", models.CharField(blank=True, max_length=500)),
+                ("byte_length", models.PositiveIntegerField(default=0)),
+            ],
+            options={"ordering": ["origin"]},
+        ),
+    ]

apps/sources/migrations/0003_source_lease_and_origin_state.py

@@ -0,0 +1,45 @@
+# Generated by Django 5.2.16 on 2026-07-21 for VR-103
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sources", "0002_policy_review_and_robots_cache"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="SourceLease",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("token", models.CharField(default="", max_length=64)),
+                ("worker_id", models.CharField(blank=True, max_length=128)),
+                ("expires_at", models.DateTimeField(db_index=True)),
+                (
+                    "source",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="lease",
+                        to="sources.source",
+                    ),
+                ),
+            ],
+            options={"ordering": ["source"]},
+        ),
+        migrations.CreateModel(
+            name="SourceOriginState",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("domain", models.CharField(max_length=255, unique=True)),
+                ("next_allowed_at", models.DateTimeField()),
+            ],
+            options={"ordering": ["domain"]},
+        ),
+    ]

apps/sources/models.py

@@ -0,0 +1,326 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from uuid import uuid4
+from urllib.parse import urlparse
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.utils import timezone
+
+from apps.core.models import TimeStampedModel
+
+
+class Source(TimeStampedModel):
+    class Type(models.TextChoices):
+        EMPLOYER = "employer", "Werkgeverspagina"
+        ATS = "ats", "Publieke ATS-pagina"
+        RSS = "rss", "RSS/Atom"
+        SITEMAP = "sitemap", "Sitemap"
+        EMAIL = "email", "Vacaturemail"
+        MANUAL = "manual", "Handmatige import"
+
+    class Status(models.TextChoices):
+        CANDIDATE = "candidate", "Kandidaat"
+        TRIAL = "trial", "Proefrun"
+        ACTIVE = "active", "Actief"
+        QUARANTINED = "quarantined", "Quarantaine"
+        PAUSED = "paused", "Gepauzeerd"
+        DISABLED = "disabled", "Uitgeschakeld"
+
+    class Policy(models.TextChoices):
+        ALLOW = "allow", "Toestaan"
+        REVIEW = "review", "Te beoordelen"
+        DENY = "deny", "Blokkeren"
+
+    name = models.CharField(max_length=200)
+    source_type = models.CharField(max_length=24, choices=Type.choices)
+    base_url = models.URLField(max_length=1000, blank=True)
+    domain = models.CharField(max_length=255, db_index=True)
+    status = models.CharField(max_length=24, choices=Status.choices, default=Status.CANDIDATE)
+    policy = models.CharField(max_length=16, choices=Policy.choices, default=Policy.REVIEW)
+    policy_reason = models.CharField(max_length=500, blank=True)
+    parser_key = models.CharField(max_length=120, default="auto")
+    strict_mode = models.BooleanField(default=True)
+    allow_public_endpoint = models.BooleanField(default=False)
+    honor_robots = models.BooleanField(default=True)
+
+    crawl_interval_minutes = models.PositiveIntegerField(default=720)
+    minimum_interval_seconds = models.PositiveIntegerField(default=30)
+    max_concurrency = models.PositiveSmallIntegerField(default=1)
+    next_run_at = models.DateTimeField(null=True, blank=True, db_index=True)
+    last_success_at = models.DateTimeField(null=True, blank=True)
+    last_failure_at = models.DateTimeField(null=True, blank=True)
+    failure_count = models.PositiveIntegerField(default=0)
+    etag = models.CharField(max_length=500, blank=True)
+    last_modified = models.CharField(max_length=500, blank=True)
+    robots_checked_at = models.DateTimeField(null=True, blank=True)
+    terms_checked_at = models.DateTimeField(null=True, blank=True)
+    metadata = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["name"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["domain", "source_type"], name="unique_domain_source_type"
+            )
+        ]
+
+    def __str__(self) -> str:
+        return self.name
+
+    def clean(self) -> None:
+        if self.base_url:
+            parsed = urlparse(self.base_url)
+            if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+                raise ValidationError({"base_url": "Alleen geldige http(s)-URLs zijn toegestaan."})
+            if self.domain and parsed.hostname.lower().rstrip(".") != self.domain.lower().rstrip(
+                "."
+            ):
+                raise ValidationError({"domain": "Domein moet overeenkomen met de base URL."})
+        if self.policy == self.Policy.DENY and self.status == self.Status.ACTIVE:
+            raise ValidationError("Een geblokkeerde bron kan niet actief zijn.")
+
+    @property
+    def discovery_evidence(self) -> list[dict[str, str | float]]:
+        raw = self.metadata if isinstance(self.metadata, dict) else {}
+        entries = raw.get("discovery", [])
+        if not isinstance(entries, list):
+            return []
+        return [entry for entry in entries if isinstance(entry, dict)]
+
+    @property
+    def latest_policy_review(self) -> "SourcePolicyReview | None":
+        return self.policy_reviews.order_by("-created_at").first()
+
+    @property
+    def policy_review_state(self) -> str:
+        review = self.latest_policy_review
+        if not review:
+            return "missing"
+        if review.is_expired:
+            return "expired"
+        return "active"
+
+    @property
+    def policy_review_summary(self) -> str:
+        review = self.latest_policy_review
+        if not review:
+            return "Geen review geregistreerd"
+        if review.is_expired:
+            return f"Review verlopen op {review.expires_at:%Y-%m-%d}" if review.expires_at else "Review verlopen"
+        if review.expires_at:
+            return f"{review.get_decision_display()} geldig tot {review.expires_at:%Y-%m-%d}"
+        return f"{review.get_decision_display()} zonder vervaldatum"
+
+    @property
+    def requires_terms_review(self) -> bool:
+        return self.policy == Source.Policy.ALLOW and self.policy_review_state in {"missing", "expired"}
+
+    def schedule_after_success(self, *, now=None, jitter_seconds: int = 0) -> None:
+        now = now or timezone.now()
+        self.last_success_at = now
+        self.failure_count = 0
+        self.next_run_at = now + timedelta(minutes=self.crawl_interval_minutes) + timedelta(
+            seconds=max(0, jitter_seconds)
+        )
+        self.save(update_fields=["last_success_at", "failure_count", "next_run_at", "updated_at"])
+
+    def schedule_after_failure(
+        self,
+        *, now=None,
+        backoff_minutes: int | None = None,
+        backoff_seconds: int | None = None,
+    ) -> None:
+        now = now or timezone.now()
+        self.last_failure_at = now
+        self.failure_count += 1
+        if backoff_seconds is None:
+            backoff = backoff_minutes or min(24 * 60, max(15, 2 ** min(self.failure_count, 10)))
+            backoff_seconds = backoff * 60
+        self.next_run_at = now + timedelta(seconds=max(0, backoff_seconds))
+        self.save(
+            update_fields=[
+                "last_failure_at",
+                "failure_count",
+                "next_run_at",
+                "updated_at",
+            ]
+        )
+
+
+class SourceRun(TimeStampedModel):
+    class Status(models.TextChoices):
+        RUNNING = "running", "Bezig"
+        SUCCESS = "success", "Geslaagd"
+        PARTIAL = "partial", "Gedeeltelijk"
+        FAILED = "failed", "Mislukt"
+        SKIPPED = "skipped", "Overgeslagen"
+
+    source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="runs")
+    status = models.CharField(max_length=16, choices=Status.choices, default=Status.RUNNING)
+    started_at = models.DateTimeField(default=timezone.now)
+    finished_at = models.DateTimeField(null=True, blank=True)
+    http_status = models.PositiveSmallIntegerField(null=True, blank=True)
+    discovered_count = models.PositiveIntegerField(default=0)
+    extracted_count = models.PositiveIntegerField(default=0)
+    created_count = models.PositiveIntegerField(default=0)
+    updated_count = models.PositiveIntegerField(default=0)
+    duplicate_count = models.PositiveIntegerField(default=0)
+    error_category = models.CharField(max_length=80, blank=True)
+    error_message = models.CharField(max_length=1000, blank=True)
+    metrics = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["-started_at"]
+
+    def finish(self, status: str, **metrics) -> None:
+        self.status = status
+        self.finished_at = timezone.now()
+        for key, value in metrics.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+        self.save()
+
+
+class SourceLease(TimeStampedModel):
+    source = models.OneToOneField(Source, on_delete=models.CASCADE, related_name="lease")
+    token = models.CharField(max_length=64, default=lambda: str(uuid4()))
+    worker_id = models.CharField(max_length=128, blank=True)
+    expires_at = models.DateTimeField(db_index=True)
+
+    class Meta:
+        ordering = ["source"]
+
+    def __str__(self) -> str:
+        return f"{self.source_id}:{self.token}"
+
+    @property
+    def is_expired(self) -> bool:
+        return self.expires_at <= timezone.now()
+
+
+class SourceOriginState(TimeStampedModel):
+    domain = models.CharField(max_length=255, unique=True)
+    next_allowed_at = models.DateTimeField()
+
+    class Meta:
+        ordering = ["domain"]
+
+    def __str__(self) -> str:
+        return self.domain
+
+
+class SourcePolicyReview(TimeStampedModel):
+    class Scope(models.TextChoices):
+        SOURCE = "source", "Bronspecifiek"
+        DOMAIN = "domain", "Domeinbrede controle"
+
+    class Decision(models.TextChoices):
+        ALLOW = "allow", "Toestaan"
+        TRIAL = "trial", "Proefrun"
+        PAUSE = "pause", "Pauzeren"
+        DENY = "deny", "Blokkeren"
+
+    source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="policy_reviews")
+    actor = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="policy_reviews",
+    )
+    scope = models.CharField(max_length=20, choices=Scope.choices, default=Scope.SOURCE)
+    decision = models.CharField(max_length=16, choices=Decision.choices)
+    reason = models.CharField(max_length=500)
+    evidence_link = models.URLField(max_length=500, blank=True)
+    notes = models.TextField(blank=True)
+    expires_at = models.DateTimeField(null=True, blank=True)
+    metadata = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["-created_at"]
+
+    @property
+    def is_expired(self) -> bool:
+        if not self.expires_at:
+            return False
+        return self.expires_at <= timezone.now()
+
+
+class SourceRobotsCache(TimeStampedModel):
+    origin = models.CharField(max_length=255, unique=True)
+    expires_at = models.DateTimeField()
+    etag = models.CharField(max_length=255, blank=True)
+    last_modified = models.CharField(max_length=255, blank=True)
+    allow_rules = models.JSONField(default=dict, blank=True)
+    disallow_rules = models.JSONField(default=dict, blank=True)
+    error = models.CharField(max_length=500, blank=True)
+    byte_length = models.PositiveIntegerField(default=0)
+
+    class Meta:
+        ordering = ["origin"]
+
+    @property
+    def is_fresh(self) -> bool:
+        return self.expires_at > timezone.now()
+
+
+class RawDocument(TimeStampedModel):
+    class Kind(models.TextChoices):
+        HTML = "html", "HTML"
+        XML = "xml", "XML"
+        EMAIL = "email", "E-mail"
+        JSON = "json", "JSON"
+        TEXT = "text", "Tekst"
+
+    source = models.ForeignKey(
+        Source, on_delete=models.SET_NULL, null=True, related_name="documents"
+    )
+    source_run = models.ForeignKey(
+        SourceRun, on_delete=models.SET_NULL, null=True, blank=True, related_name="documents"
+    )
+    url = models.URLField(max_length=2000, blank=True)
+    final_url = models.URLField(max_length=2000, blank=True)
+    kind = models.CharField(max_length=16, choices=Kind.choices)
+    content_type = models.CharField(max_length=200, blank=True)
+    http_status = models.PositiveSmallIntegerField(null=True, blank=True)
+    response_headers = models.JSONField(default=dict, blank=True)
+    content_hash = models.CharField(max_length=64, db_index=True)
+    body_text = models.TextField(blank=True)
+    byte_length = models.PositiveIntegerField(default=0)
+    parser_key = models.CharField(max_length=120, blank=True)
+    parser_version = models.CharField(max_length=80, blank=True)
+    extraction_confidence = models.DecimalField(max_digits=4, decimal_places=3, default=0)
+    retain_until = models.DateTimeField(null=True, blank=True)
+    quarantined = models.BooleanField(default=False)
+    quarantine_reason = models.CharField(max_length=500, blank=True)
+    metadata = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["-created_at"]
+        indexes = [models.Index(fields=["source", "content_hash"])]
+
+    def __str__(self) -> str:
+        return self.final_url or self.url or f"Document {self.pk}"
+
+
+class EmailMessageRecord(TimeStampedModel):
+    message_id = models.CharField(max_length=998, unique=True)
+    mailbox = models.CharField(max_length=255, default="INBOX")
+    sender = models.CharField(max_length=500, blank=True)
+    subject = models.CharField(max_length=998, blank=True)
+    received_at = models.DateTimeField(null=True, blank=True)
+    raw_document = models.OneToOneField(
+        RawDocument, on_delete=models.SET_NULL, null=True, blank=True, related_name="email_record"
+    )
+    links = models.JSONField(default=list, blank=True)
+    processed = models.BooleanField(default=False)
+    error_message = models.CharField(max_length=1000, blank=True)
+
+    class Meta:
+        ordering = ["-received_at", "-created_at"]
+
+    def __str__(self) -> str:
+        return self.subject or self.message_id

apps/sources/services/canonicalize.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import posixpath
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+TRACKING_PARAMETERS = {
+    "fbclid",
+    "gclid",
+    "mc_cid",
+    "mc_eid",
+    "ref",
+    "referrer",
+    "source",
+    "trk",
+    "trackingid",
+}
+TRACKING_PREFIXES = ("utm_", "pk_")
+
+
+def canonicalize_url(url: str) -> str:
+    value = (url or "").strip()
+    if not value:
+        return ""
+    parts = urlsplit(value)
+    scheme = parts.scheme.lower()
+    host = (parts.hostname or "").lower().rstrip(".")
+    if not scheme or not host:
+        return value
+    port = parts.port
+    if port and not ((scheme == "http" and port == 80) or (scheme == "https" and port == 443)):
+        netloc = f"{host}:{port}"
+    else:
+        netloc = host
+    path = parts.path or "/"
+    normalized_path = posixpath.normpath(path)
+    if path.endswith("/") and not normalized_path.endswith("/"):
+        normalized_path += "/"
+    if not normalized_path.startswith("/"):
+        normalized_path = "/" + normalized_path
+    query_pairs = []
+    for key, value in parse_qsl(parts.query, keep_blank_values=True):
+        lower = key.lower()
+        if lower in TRACKING_PARAMETERS or any(
+            lower.startswith(prefix) for prefix in TRACKING_PREFIXES
+        ):
+            continue
+        query_pairs.append((key, value))
+    query_pairs.sort()
+    return urlunsplit((scheme, netloc, normalized_path, urlencode(query_pairs, doseq=True), ""))
+
+
+def domain_matches(hostname: str, domain: str) -> bool:
+    host = hostname.lower().rstrip(".")
+    target = domain.lower().rstrip(".")
+    return host == target or host.endswith("." + target)

apps/sources/services/discovery.py

@@ -0,0 +1,391 @@
+from __future__ import annotations
+
+import ipaddress
+import json
+import re
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from urllib.parse import urljoin, urlsplit
+from xml.etree import ElementTree
+
+from bs4 import BeautifulSoup
+from django.db import transaction
+
+from apps.sources.adapters.email_alert import EmailAlertAdapter
+from apps.sources.adapters.rss import RssAdapter
+from apps.sources.models import Source
+
+from .canonicalize import canonicalize_url, domain_matches
+from .policy import is_denied_domain
+
+CAREER_PATTERN = re.compile(
+    r"\b(career|careers|jobs|job|vacature|vacatures|werken-bij|werken bij|emploi|emplois|offres)\b",
+    re.I,
+)
+FEED_TYPE_PATTERN = re.compile(r"application/(?:atom|rss)\+xml|text/xml|application/xml", re.I)
+
+
+@dataclass(frozen=True)
+class SourceCandidate:
+    url: str
+    domain: str
+    source_type: str
+    label: str
+    confidence: float
+    reason: str
+    discovered_from: str
+
+
+def discover_career_links(html: str, *, base_url: str) -> list[SourceCandidate]:
+    return _discover_html(
+        html,
+        base_url=base_url,
+        include_jsonld=False,
+        include_feed_links=False,
+    )
+
+
+def discover_from_html(html: str, *, base_url: str) -> list[SourceCandidate]:
+    return _discover_html(
+        html,
+        base_url=base_url,
+        include_jsonld=True,
+        include_feed_links=True,
+    )
+
+
+def discover_from_feed(feed_content: str, *, base_url: str) -> list[SourceCandidate]:
+    adapter = RssAdapter()
+    result = adapter.extract(feed_content, url=base_url)
+    base_domain = _hostname(base_url)
+    candidates: list[SourceCandidate] = []
+    for extracted in result.jobs:
+        candidate = _build_candidate(
+            raw_url=extracted.url,
+            base_domain=base_domain,
+            source_type=Source.Type.RSS,
+            label=extracted.title[:120],
+            reason="rss",
+            discovered_from="feed",
+            confidence=0.82,
+            allow_off_domain=False,
+        )
+        if candidate:
+            candidates.append(candidate)
+    return _dedupe(candidates)
+
+
+def discover_from_sitemap(xml_content: str, *, base_url: str) -> list[SourceCandidate]:
+    try:
+        root = ElementTree.fromstring(xml_content)
+    except ElementTree.ParseError:
+        return []
+
+    root_name = _local_tag(root.tag)
+    if root_name == "sitemapindex":
+        discovered_from = "sitemap-index"
+    elif root_name == "urlset":
+        discovered_from = "sitemap-urlset"
+    else:
+        return []
+
+    base_domain = _hostname(base_url)
+    candidates: list[SourceCandidate] = []
+    for location in root.findall(".//{*}loc"):
+        raw = (location.text or "").strip()
+        if not raw:
+            continue
+        candidate = _build_candidate(
+            raw_url=urljoin(base_url, raw),
+            base_domain=base_domain,
+            source_type=Source.Type.SITEMAP,
+            label="Sitemaplocatie",
+            reason=discovered_from,
+            discovered_from=discovered_from,
+            confidence=0.86 if discovered_from == "sitemap-urlset" else 0.75,
+            allow_off_domain=False,
+        )
+        if candidate:
+            candidates.append(candidate)
+
+    return _dedupe(candidates)
+
+
+def discover_from_email(raw_message: bytes) -> list[SourceCandidate]:
+    adapter = EmailAlertAdapter()
+    result = adapter.extract_message(raw_message)
+    candidates: list[SourceCandidate] = []
+    for extracted in result.jobs:
+        candidate_url = _to_domain_root_url(extracted.url)
+        candidate = _build_candidate(
+            raw_url=candidate_url,
+            base_domain=_hostname(extracted.url),
+            source_type=Source.Type.EMPLOYER,
+            label=extracted.title[:120],
+            reason="email",
+            discovered_from="email",
+            confidence=0.62,
+            allow_off_domain=True,
+        )
+        if candidate:
+            candidates.append(candidate)
+
+    return _dedupe(candidates)
+
+
+def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int, int, int]:
+    created = updated = skipped = 0
+    ordered = _dedupe(candidates)
+    with transaction.atomic():
+        for candidate in ordered:
+            provenance = _provenance_entry(candidate)
+            now = _utc_now()
+            source, was_created = Source.objects.get_or_create(
+                domain=candidate.domain,
+                source_type=candidate.source_type,
+                defaults={
+                    "name": _candidate_name(candidate),
+                    "base_url": candidate.url,
+                    "status": Source.Status.CANDIDATE,
+                    "policy": Source.Policy.REVIEW,
+                    "policy_reason": "Automatisch ontdekt",
+                    "parser_key": "auto",
+                    "strict_mode": True,
+                    "metadata": {
+                        "discovery": [provenance],
+                        "discovered_at": now,
+                    },
+                },
+            )
+            if was_created:
+                created += 1
+                continue
+
+            updated_fields: list[str] = ["updated_at"]
+            metadata = source.metadata if isinstance(source.metadata, dict) else {}
+            evidence = metadata.get("discovery", [])
+            if not isinstance(evidence, list):
+                evidence = []
+            if not any(item.get("url") == candidate.url for item in evidence if isinstance(item, dict)):
+                evidence.append(provenance)
+                metadata["discovery"] = evidence[-20:]
+                metadata["discovered_at"] = now
+                source.metadata = metadata
+                updated_fields.append("metadata")
+            else:
+                skipped += 1
+
+            if not source.name:
+                source.name = _candidate_name(candidate)
+                updated_fields.append("name")
+            if not source.base_url:
+                source.base_url = candidate.url
+                updated_fields.append("base_url")
+
+            if len(updated_fields) > 1:
+                source.save(update_fields=sorted(set(updated_fields)))
+                updated += 1
+            else:
+                skipped += 1
+
+    return (created, updated, skipped)
+
+
+def _candidate_name(candidate: SourceCandidate) -> str:
+    label = candidate.label.strip()
+    if label:
+        return label
+    return candidate.domain
+
+
+def _provenance_entry(candidate: SourceCandidate) -> dict[str, str | float]:
+    return {
+        "url": candidate.url,
+        "source_type": candidate.source_type,
+        "discovered_from": candidate.discovered_from,
+        "confidence": candidate.confidence,
+        "reason": candidate.reason,
+        "label": candidate.label[:240],
+        "seen_at": _utc_now(),
+    }
+
+
+def _discover_html(
+    html: str,
+    *,
+    base_url: str,
+    include_jsonld: bool,
+    include_feed_links: bool,
+) -> list[SourceCandidate]:
+    soup = BeautifulSoup(html, "lxml")
+    base_domain = _hostname(base_url)
+    candidates: list[SourceCandidate] = []
+
+    for anchor in soup.find_all("a", href=True):
+        label = " ".join(anchor.get_text(" ", strip=True).split())
+        candidate = _build_candidate(
+            raw_url=urljoin(base_url, str(anchor["href"])),
+            base_domain=base_domain,
+            source_type=Source.Type.EMPLOYER,
+            label=label,
+            reason="career-link",
+            discovered_from="html",
+            confidence=0.9,
+            allow_off_domain=False,
+        )
+        if not candidate:
+            continue
+        if not CAREER_PATTERN.search(label) and not CAREER_PATTERN.search(urlsplit(candidate.url).path):
+            continue
+        candidates.append(candidate)
+
+    if include_jsonld:
+        for js in soup.find_all("script", type=re.compile(r"application/ld\+json", re.I)):
+            raw = js.get_text("", strip=True)
+            if not raw:
+                continue
+            try:
+                parsed = json.loads(raw)
+            except json.JSONDecodeError:
+                continue
+            for url in _collect_jsonld_urls(parsed):
+                candidate = _build_candidate(
+                    raw_url=urljoin(base_url, url),
+                    base_domain=base_domain,
+                    source_type=Source.Type.EMPLOYER,
+                    label="JSON-LD job",
+                    reason="jsonld",
+                    discovered_from="html-jsonld",
+                    confidence=0.95,
+                    allow_off_domain=False,
+                )
+                if candidate:
+                    candidates.append(candidate)
+
+    if include_feed_links:
+        for link in soup.find_all("link", href=True):
+            rel = {str(item).lower() for item in (link.get("rel") or [])}
+            if "alternate" not in rel:
+                continue
+            link_type = str(link.get("type") or "")
+            if not FEED_TYPE_PATTERN.search(link_type):
+                continue
+            candidate = _build_candidate(
+                raw_url=urljoin(base_url, str(link["href"])),
+                base_domain=base_domain,
+                source_type=Source.Type.RSS,
+                label=(str(link.get("title") or "Feedlink")).strip()[:120],
+                reason="feed",
+                discovered_from="html",
+                confidence=0.86,
+                allow_off_domain=False,
+            )
+            if candidate:
+                candidates.append(candidate)
+
+    return _dedupe(candidates)
+
+
+def _collect_jsonld_urls(payload) -> list[str]:
+    urls: list[str] = []
+
+    def walk(node):
+        if isinstance(node, dict):
+            candidate_type = str(node.get("@type", "")).lower()
+            if candidate_type == "jobposting":
+                for key in ("url", "applyUrl", "application", "applicationurl"):
+                    value = node.get(key)
+                    if isinstance(value, str) and value:
+                        urls.append(value)
+            for key in ("@graph", "itemListElement", "item", "jobs", "jobPosting"):
+                nested = node.get(key)
+                if nested is not None:
+                    walk(nested)
+        elif isinstance(node, list):
+            for item in node:
+                walk(item)
+
+    walk(payload)
+    return urls
+
+
+def _build_candidate(
+    *,
+    raw_url: str,
+    base_domain: str,
+    source_type: str,
+    label: str,
+    reason: str,
+    discovered_from: str,
+    confidence: float,
+    allow_off_domain: bool,
+) -> SourceCandidate | None:
+    canonical = canonicalize_url(raw_url)
+    if not canonical:
+        return None
+    if not _url_is_http(canonical):
+        return None
+    hostname = _hostname(canonical)
+    if not hostname:
+        return None
+    if is_denied_domain(hostname):
+        return None
+    if _is_private_host(hostname):
+        return None
+    if (not allow_off_domain) and base_domain and not domain_matches(hostname, base_domain):
+        return None
+    return SourceCandidate(
+        url=canonical,
+        domain=hostname,
+        source_type=source_type,
+        label=label,
+        confidence=confidence,
+        reason=reason,
+        discovered_from=discovered_from,
+    )
+
+
+def _url_is_http(url: str) -> bool:
+    return urlsplit(url).scheme.lower() in {"http", "https"}
+
+
+def _hostname(value: str) -> str:
+    return (urlsplit(value).hostname or "").lower()
+
+
+def _is_private_host(hostname: str) -> bool:
+    host = hostname.lower().rstrip(".")
+    if host in {"localhost", "127.0.0.1", "::1"} or host.endswith((".local", ".localhost")):
+        return True
+    try:
+        ip = ipaddress.ip_address(host)
+    except ValueError:
+        return False
+    return not ip.is_global
+
+
+def _dedupe(candidates: list[SourceCandidate]) -> list[SourceCandidate]:
+    seen: set[tuple[str, str]] = set()
+    result: list[SourceCandidate] = []
+    for candidate in sorted(candidates, key=lambda item: item.confidence, reverse=True):
+        key = (candidate.url, candidate.source_type)
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(candidate)
+    return result
+
+
+def _local_tag(tag_name: str) -> str:
+    return tag_name.rsplit("}", 1)[-1].lower()
+
+
+def _to_domain_root_url(url: str) -> str:
+    parsed = urlsplit(url)
+    if not parsed.scheme or not parsed.hostname:
+        return url
+    return f"{parsed.scheme}://{parsed.hostname}"
+
+
+def _utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()

apps/sources/services/email_import.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import timedelta
+from email import policy
+from email.parser import BytesParser
+from email.utils import parsedate_to_datetime
+
+from django.conf import settings
+from django.db import transaction
+from django.utils import timezone
+
+from apps.jobs.services.normalization import normalize_extracted_job
+from apps.jobs.services.pipeline import persist_draft
+from apps.sources.adapters.email_alert import EmailAlertAdapter
+from apps.sources.models import EmailMessageRecord, RawDocument, Source
+
+
+def message_identity(raw_message: bytes) -> str:
+    """Return the RFC Message-ID or a deterministic hash when it is absent."""
+    parsed = BytesParser(policy=policy.default).parsebytes(raw_message, headersonly=True)
+    fallback_id = hashlib.sha256(raw_message).hexdigest()
+    return str(parsed.get("message-id") or f"sha256:{fallback_id}").strip()
+
+
+@transaction.atomic
+def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageRecord:
+    parsed = BytesParser(policy=policy.default).parsebytes(raw_message)
+    message_id = message_identity(raw_message)
+    existing = EmailMessageRecord.objects.filter(message_id=message_id).first()
+    if existing:
+        return existing
+
+    source, _ = Source.objects.get_or_create(
+        domain="mailbox.local",
+        source_type=Source.Type.EMAIL,
+        defaults={
+            "name": "Vacaturemailbox",
+            "status": Source.Status.ACTIVE,
+            "policy": Source.Policy.ALLOW,
+            "parser_key": "email-alert",
+            "crawl_interval_minutes": 10,
+        },
+    )
+    content_hash = hashlib.sha256(raw_message).hexdigest()
+    retain_until = timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS)
+    document = RawDocument.objects.create(
+        source=source,
+        kind=RawDocument.Kind.EMAIL,
+        content_type="message/rfc822",
+        content_hash=content_hash,
+        body_text=raw_message.decode("utf-8", errors="replace"),
+        byte_length=len(raw_message),
+        retain_until=retain_until,
+        metadata={"mailbox": mailbox},
+    )
+    received_at = None
+    if parsed.get("date"):
+        try:
+            received_at = parsedate_to_datetime(str(parsed.get("date")))
+            if timezone.is_naive(received_at):
+                received_at = timezone.make_aware(received_at, timezone.get_current_timezone())
+        except (TypeError, ValueError, OverflowError):
+            received_at = None
+
+    adapter = EmailAlertAdapter()
+    result = adapter.extract_message(raw_message)
+    document.parser_key = result.parser_key
+    document.parser_version = result.parser_version
+    document.extraction_confidence = result.confidence
+    document.save(
+        update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"]
+    )
+    links: list[str] = []
+    errors: list[str] = []
+    for extracted in result.jobs:
+        links.append(extracted.url)
+        try:
+            draft = normalize_extracted_job(extracted)
+            persist_draft(
+                draft,
+                document=document,
+                parser_key=result.parser_key,
+                parser_version=result.parser_version,
+                extraction_confidence=result.confidence,
+            )
+        except Exception as exc:
+            errors.append(f"{exc.__class__.__name__}: {exc}")
+    return EmailMessageRecord.objects.create(
+        message_id=message_id,
+        mailbox=mailbox,
+        sender=str(parsed.get("from") or "")[:500],
+        subject=str(parsed.get("subject") or "")[:998],
+        received_at=received_at,
+        raw_document=document,
+        links=links,
+        processed=not errors,
+        error_message="; ".join(errors)[:1000],
+    )

apps/sources/services/fetcher.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import datetime, timezone as utc
+from email.utils import parsedate_to_datetime
+from collections.abc import Mapping
+from dataclasses import dataclass
+from urllib.parse import urljoin
+
+import httpx
+from django.conf import settings
+
+from apps.sources.models import Source
+
+from .policy import assess_url
+from .url_security import validate_public_url
+
+ALLOWED_CONTENT_TYPES = (
+    "text/html",
+    "application/xhtml+xml",
+    "application/xml",
+    "text/xml",
+    "application/json",
+    "application/ld+json",
+    "text/plain",
+    "application/rss+xml",
+    "application/atom+xml",
+)
+
+
+class FetchError(RuntimeError):
+    pass
+
+
+class PolicyBlockedError(FetchError):
+    pass
+
+
+class ContentRejectedError(FetchError):
+    pass
+
+
+class RateLimitedError(FetchError):
+    def __init__(self, retry_after_seconds: int | None, message: str = "HTTP 429") -> None:
+        self.retry_after_seconds = retry_after_seconds
+        suffix = f"; Retry-After={retry_after_seconds}s" if retry_after_seconds else ""
+        super().__init__(f"{message}{suffix}".strip())
+
+
+class FetchTimeoutError(FetchError):
+    pass
+
+
+def parse_retry_after(value: str | None) -> int | None:
+    if not value:
+        return None
+    value = value.strip()
+    if not value:
+        return None
+    try:
+        return max(0, int(value))
+    except ValueError:
+        pass
+    try:
+        retry_datetime = parsedate_to_datetime(value)
+    except (TypeError, ValueError):
+        return None
+    if retry_datetime is None:
+        return None
+    if retry_datetime.tzinfo is None:
+        retry_datetime = retry_datetime.replace(tzinfo=utc)
+    retry_aware = retry_datetime.astimezone(utc)
+    now = datetime.now(utc)
+    delta = (retry_aware - now).total_seconds()
+    return max(0, int(delta))
+
+
+@dataclass(slots=True)
+class FetchedDocument:
+    requested_url: str
+    final_url: str
+    status_code: int
+    headers: dict[str, str]
+    content: bytes
+
+    @property
+    def text(self) -> str:
+        encoding = "utf-8"
+        content_type = self.headers.get("content-type", "")
+        if "charset=" in content_type:
+            encoding = content_type.split("charset=", 1)[1].split(";", 1)[0].strip()
+        return self.content.decode(encoding, errors="replace")
+
+    @property
+    def sha256(self) -> str:
+        return hashlib.sha256(self.content).hexdigest()
+
+
+def fetch_url(
+    url: str,
+    *,
+    source: Source | None = None,
+    conditional_headers: Mapping[str, str] | None = None,
+    client: httpx.Client | None = None,
+) -> FetchedDocument:
+    decision = assess_url(url, source=source)
+    if not decision.allowed:
+        raise PolicyBlockedError(decision.reason)
+
+    headers = {
+        "User-Agent": settings.FETCHER_USER_AGENT,
+        "Accept": (
+            "text/html,application/xhtml+xml,application/xml,application/json,"
+            "text/plain;q=0.8,*/*;q=0.1"
+        ),
+    }
+    if conditional_headers:
+        headers.update(conditional_headers)
+
+    own_client = client is None
+    http_client = client or httpx.Client(
+        timeout=httpx.Timeout(settings.FETCHER_TIMEOUT_SECONDS),
+        follow_redirects=False,
+        headers=headers,
+    )
+    current_url = url
+    try:
+        for _ in range(settings.FETCHER_MAX_REDIRECTS + 1):
+            validation = validate_public_url(
+                current_url,
+                allow_nonstandard_ports=settings.FETCHER_ALLOW_NONSTANDARD_PORTS,
+            )
+            response = http_client.get(current_url, headers=headers)
+            post_validation = validate_public_url(
+                str(response.url) if response.url else current_url,
+                allow_nonstandard_ports=settings.FETCHER_ALLOW_NONSTANDARD_PORTS,
+            )
+            if not set(validation.addresses).intersection(set(post_validation.addresses)):
+                raise FetchError("DNS-rebindcontrole faalde bij het benaderen van bron.")
+            if response.status_code in {301, 302, 303, 307, 308}:
+                location = response.headers.get("location")
+                if not location:
+                    raise FetchError("Redirect zonder Location-header.")
+                current_url = urljoin(current_url, location)
+                next_decision = assess_url(current_url, source=source)
+                if not next_decision.allowed:
+                    raise PolicyBlockedError(next_decision.reason)
+                continue
+            if response.status_code == 304:
+                return FetchedDocument(url, current_url, 304, dict(response.headers), b"")
+            if response.status_code == 429:
+                retry_after = parse_retry_after(response.headers.get("retry-after"))
+                raise RateLimitedError(retry_after)
+            if response.status_code >= 400:
+                raise FetchError(f"HTTP {response.status_code}")
+            content_type = response.headers.get("content-type", "").lower()
+            if content_type and not any(
+                allowed in content_type for allowed in ALLOWED_CONTENT_TYPES
+            ):
+                raise ContentRejectedError(f"Content-Type niet toegestaan: {content_type}")
+            content = response.content
+            if len(content) > settings.FETCHER_MAX_BYTES:
+                raise ContentRejectedError("Document overschrijdt de ingestelde groottebeperking.")
+            return FetchedDocument(
+                url, current_url, response.status_code, dict(response.headers), content
+            )
+        raise FetchError("Te veel redirects.")
+    except httpx.TimeoutException as exc:
+        raise FetchTimeoutError("Timeout tijdens HTTP-opvraag") from exc
+    except httpx.RequestError as exc:
+        raise FetchError(f"Netwerkfout tijdens HTTP-opvraag: {exc}") from exc
+    finally:
+        if own_client:
+            http_client.close()

apps/sources/services/health.py

@@ -0,0 +1,349 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+from django.utils import timezone
+
+from apps.sources.models import Source, SourcePolicyReview, SourceRun
+
+from .policy import create_policy_review
+
+
+@dataclass(frozen=True)
+class SourceHealth:
+    source_id: int
+    source_name: str
+    source_type: str
+    source_status: str
+    monitored_runs: int
+    success_ratio: float
+    avg_latency_ms: float | None
+    error_counts: dict[str, int]
+    http_status_counts: dict[str, int]
+    extracted_count: int
+    updated_count: int
+    duplicate_count: int
+    last_parser: str | None
+    last_parser_warnings: int
+    last_health_action: str | None
+    last_health_reason: str | None
+
+
+SOURCE_HEALTH_RUN_WINDOW = 30
+SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE = 6
+SOURCE_HEALTH_TEMPORARY_ERROR_RATIO = 0.7
+SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK = 3
+SOURCE_HEALTH_PARSER_MIN_WARNINGS = 2
+SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX = 0
+SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS = 12
+
+TEMPORARY_ERROR_CATEGORIES = {
+    "timeout",
+    "rate_limited",
+    "FetchTimeoutError",
+    "FetchError",
+    "unexpected",
+    "NetworkError",
+}
+POLICY_ERROR_CATEGORIES = {"policy"}
+
+
+def _normalize_metrics(raw: object) -> dict[str, object]:
+    if isinstance(raw, dict):
+        return raw
+    return {}
+
+
+def _warnings_from_run(run: SourceRun) -> list[str]:
+    metrics = _normalize_metrics(run.metrics)
+    warnings = metrics.get("warnings", [])
+    if not isinstance(warnings, list):
+        return []
+    return [str(item) for item in warnings if isinstance(item, str)]
+
+
+def _parser_from_run(run: SourceRun) -> str | None:
+    metrics = _normalize_metrics(run.metrics)
+    parser = metrics.get("parser")
+    if isinstance(parser, str) and parser:
+        return parser
+    return None
+
+
+def _int(value: object) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
+def _float(value: object) -> float:
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def _to_iso(dt: datetime | None) -> str | None:
+    if dt is None:
+        return None
+    return dt.isoformat()
+
+
+def _from_iso(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        dt = datetime.fromisoformat(value)
+    except ValueError:
+        return None
+    if timezone.is_naive(dt):
+        return timezone.make_aware(dt)
+    return dt
+
+
+def _health_metadata(source: Source) -> dict[str, object]:
+    metadata = source.metadata
+    if not isinstance(metadata, dict):
+        return {}
+    health = metadata.get("source_health")
+    return health if isinstance(health, dict) else {}
+
+
+def _set_health_metadata(source: Source, health_data: dict[str, object]) -> None:
+    metadata = source.metadata
+    if not isinstance(metadata, dict):
+        metadata = {}
+    metadata["source_health"] = health_data
+    source.metadata = metadata
+    source.save(update_fields=["metadata", "updated_at"])
+
+
+def _run_counts(runs: Iterable[SourceRun]) -> tuple[dict[str, int], dict[str, int]]:
+    error_counts: dict[str, int] = {}
+    http_status_counts: dict[str, int] = {}
+    for run in runs:
+        if run.status != SourceRun.Status.SUCCESS:
+            category = run.error_category or "unknown"
+            error_counts[category] = error_counts.get(category, 0) + 1
+        if run.http_status:
+            status = str(run.http_status)
+            http_status_counts[status] = http_status_counts.get(status, 0) + 1
+    return error_counts, http_status_counts
+
+
+def _last_parser_output(runs: list[SourceRun]) -> tuple[str | None, int]:
+    for run in runs:
+        if run.status != SourceRun.Status.SUCCESS:
+            continue
+        parser = _parser_from_run(run)
+        if parser:
+            warnings = _warnings_from_run(run)
+            return parser, len(warnings)
+    return None, 0
+
+
+def _latency_ms(runs: list[SourceRun]) -> float | None:
+    latencies = []
+    for run in runs:
+        if run.finished_at is None or run.started_at is None:
+            continue
+        latencies.append(max(0.0, (run.finished_at - run.started_at).total_seconds() * 1000))
+    if not latencies:
+        return None
+    return sum(latencies) / len(latencies)
+
+
+def _is_parser_drift_run(run: SourceRun) -> bool:
+    if run.status != SourceRun.Status.SUCCESS:
+        return False
+    if _int(run.extracted_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
+        return False
+    if _int(run.created_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
+        return False
+    if _int(run.updated_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
+        return False
+    if _parser_from_run(run) in {None, "not-modified"}:
+        return False
+    warnings = _warnings_from_run(run)
+    return len(warnings) >= SOURCE_HEALTH_PARSER_MIN_WARNINGS
+
+
+def _determine_health_action(runs: list[SourceRun]) -> tuple[str | None, str | None]:
+    recent_runs = runs[:SOURCE_HEALTH_RUN_WINDOW]
+    considered_failures = [
+        run for run in recent_runs if run.status in {SourceRun.Status.FAILED, SourceRun.Status.SKIPPED}
+    ]
+
+    if any(
+        run.error_category in POLICY_ERROR_CATEGORIES
+        for run in considered_failures[:3]
+        if run.error_category
+    ):
+        return "quarantine", "Herhaald beleid-/securityprobleem in bronruns."
+
+    if len(considered_failures) >= SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE:
+        temporary_count = sum(
+            1
+            for run in considered_failures
+            if (run.error_category or "") in TEMPORARY_ERROR_CATEGORIES
+        )
+        ratio = _float(temporary_count) / _float(len(considered_failures))
+        if ratio >= SOURCE_HEALTH_TEMPORARY_ERROR_RATIO:
+            return "quarantine", "Herhaald tijdelijk foutgedrag tijdens bronruns."
+
+    streak = 0
+    for run in recent_runs:
+        if _is_parser_drift_run(run):
+            streak += 1
+            if streak >= SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK:
+                return (
+                    "quarantine",
+                    "Parserdrift vermoed: opeenvolgende succesvolle runs met minimale output.",
+                )
+            continue
+        streak = 0
+
+    return None, None
+
+
+def collect_source_health(*, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW) -> list[SourceHealth]:
+    sources = Source.objects.order_by("name").all()
+    rows: list[SourceHealth] = []
+
+    for source in sources:
+        run_queryset = SourceRun.objects.filter(source=source).order_by("-started_at")
+        runs = list(run_queryset[:runs_to_consider])
+        monitored_runs = len(runs)
+        success_runs = [run for run in runs if run.status == SourceRun.Status.SUCCESS]
+
+        success_ratio = _float(len(success_runs) / monitored_runs) if monitored_runs else 0.0
+        avg_latency_ms = _latency_ms(runs)
+        error_counts, http_status_counts = _run_counts(runs)
+
+        extracted_count = sum(_int(run.extracted_count) for run in runs)
+        updated_count = sum(_int(run.updated_count) for run in runs)
+        duplicate_count = sum(_int(run.duplicate_count) for run in runs)
+
+        last_parser, last_warnings = _last_parser_output(runs)
+        action, reason = _determine_health_action(runs)
+
+        rows.append(
+            SourceHealth(
+                source_id=source.pk,
+                source_name=source.name,
+                source_type=source.source_type,
+                source_status=source.status,
+                monitored_runs=monitored_runs,
+                success_ratio=success_ratio,
+                avg_latency_ms=avg_latency_ms,
+                error_counts=error_counts,
+                http_status_counts=http_status_counts,
+                extracted_count=extracted_count,
+                updated_count=updated_count,
+                duplicate_count=duplicate_count,
+                last_parser=last_parser,
+                last_parser_warnings=last_warnings,
+                last_health_action=action,
+                last_health_reason=reason,
+            )
+        )
+
+    return rows
+
+
+def evaluate_source_health(*, now: datetime | None = None) -> dict[str, int]:
+    now = now or timezone.now()
+    rows = [row for row in collect_source_health() if row.source_status in {Source.Status.ACTIVE, Source.Status.TRIAL}]
+    counts = {"evaluated": len(rows), "quarantined": 0}
+
+    for row in rows:
+        if row.last_health_action != "quarantine":
+            continue
+        counts["evaluated"] += 1
+
+        source = Source.objects.get(pk=row.source_id)
+        source.status = Source.Status.QUARANTINED
+        source.policy = Source.Policy.DENY
+        source.policy_reason = row.last_health_reason or "Bronhealth detecteert instabiele bron"
+        _set_health_metadata(
+            source,
+            {
+                "state": "quarantined",
+                "quarantine_reason": source.policy_reason,
+                "quarantined_at": _to_iso(now),
+                "recovery_due_at": _to_iso(
+                    now + timedelta(hours=SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS)
+                ),
+                "canary_started": False,
+            },
+        )
+        source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
+
+        latest_review = source.policy_reviews.order_by("-created_at").first()
+        if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.DENY:
+            create_policy_review(
+                source,
+                actor=None,
+                decision=SourcePolicyReview.Decision.DENY,
+                reason=source.policy_reason,
+                scope=SourcePolicyReview.Scope.SOURCE,
+                notes="Automatische bronhealth",
+            )
+
+        counts["quarantined"] += 1
+
+    return counts
+
+
+def canary_recovery_sources(*, now: datetime | None = None) -> list[Source]:
+    now = now or timezone.now()
+    sources: list[Source] = []
+    for source in Source.objects.filter(status=Source.Status.QUARANTINED):
+        health = _health_metadata(source)
+        if health.get("state") != "quarantined":
+            continue
+
+        if bool(health.get("canary_started", False)):
+            continue
+
+        recovery_due = _from_iso(health.get("recovery_due_at") if isinstance(health, dict) else None)
+        if recovery_due and recovery_due > now:
+            continue
+
+        sources.append(source)
+
+    return sources
+
+
+def start_health_canary(source: Source, *, now: datetime | None = None) -> None:
+    now = now or timezone.now()
+    health = _health_metadata(source)
+
+    latest_review = source.policy_reviews.order_by("-created_at").first()
+    if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.TRIAL:
+        create_policy_review(
+            source,
+            actor=None,
+            decision=SourcePolicyReview.Decision.TRIAL,
+            reason="Automatische bronrecovery via canary",
+            scope=SourcePolicyReview.Scope.SOURCE,
+            notes="Canaryherstel",
+        )
+
+    source.status = Source.Status.TRIAL
+    source.policy = Source.Policy.REVIEW
+    source.policy_reason = "Bronherstel via geautomatiseerde canary"
+    source.next_run_at = now
+    _set_health_metadata(
+        source,
+        {
+            **health,
+            "state": "canary_in_progress",
+            "canary_started": True,
+            "canary_started_at": _to_iso(now),
+        },
+    )
+    source.save(update_fields=["status", "policy", "policy_reason", "next_run_at", "updated_at"])

apps/sources/services/manual_import.py

@@ -0,0 +1,355 @@
+from __future__ import annotations
+
+import hashlib
+import html
+from dataclasses import dataclass
+from datetime import timedelta
+from urllib.parse import urlsplit
+from uuid import uuid4
+
+import bleach
+from django.conf import settings
+from django.db import transaction
+from django.db.models import QuerySet
+from django.utils import timezone
+
+from apps.jobs.models import JobSourceAlias
+from apps.jobs.services.pipeline import process_raw_document
+from apps.sources.models import RawDocument, Source, SourcePolicyReview, SourceRun
+from apps.sources.services.canonicalize import canonicalize_url
+from apps.sources.services.fetcher import (
+    FetchError,
+    FetchTimeoutError,
+    FetchedDocument,
+    PolicyBlockedError,
+    RateLimitedError,
+    fetch_url,
+)
+from apps.sources.services.policy import assess_url, create_policy_review
+
+
+class ManualImportError(RuntimeError):
+    pass
+
+
+@dataclass(frozen=True)
+class ManualImportSummary:
+    source_id: int
+    source_name: str
+    source_url: str
+    mode: str
+    extracted_count: int
+    created_count: int
+    duplicate_count: int
+    warnings: list[str]
+    jobs: list[dict[str, str]]
+
+    def to_session_payload(self) -> dict[str, object]:
+        return {
+            "source_id": self.source_id,
+            "source_name": self.source_name,
+            "source_url": self.source_url,
+            "mode": self.mode,
+            "extracted_count": self.extracted_count,
+            "created_count": self.created_count,
+            "duplicate_count": self.duplicate_count,
+            "warnings": self.warnings,
+            "jobs": self.jobs,
+        }
+
+
+def _sanitize_pasted_text(value: str) -> str:
+    text = bleach.clean(value or "", tags=[], attributes={}, strip=True).strip()
+    if not text:
+        raise ManualImportError("Het geplakte tekstveld bevat geen bruikbare inhoud.")
+    if len(text.encode("utf-8")) > settings.MANUAL_IMPORT_PASTE_MAX_BYTES:
+        raise ManualImportError(
+            "Het tekstveld is te groot voor veilige import. Verwijder overtollige tekst."
+        )
+    return text
+
+
+def _kind_for(content_type: str) -> str:
+    content_type = (content_type or "").lower()
+    if "html" in content_type:
+        return RawDocument.Kind.HTML
+    if "xml" in content_type or "rss" in content_type or "atom" in content_type:
+        return RawDocument.Kind.XML
+    if "json" in content_type:
+        return RawDocument.Kind.JSON
+    return RawDocument.Kind.TEXT
+
+
+def _mode_source_name(domain: str, *, mode: str) -> str:
+    if mode == "paste":
+        return f"Handmatige tekstimport {domain}"
+    return f"Handmatige URL-import {domain}"
+
+
+def _ensure_manual_review(source: Source, actor) -> None:
+    review = source.latest_policy_review
+    if (
+        review
+        and not review.is_expired
+        and review.decision == SourcePolicyReview.Decision.ALLOW
+    ):
+        return
+    create_policy_review(
+        source,
+        actor=actor if actor and getattr(actor, "pk", None) else None,
+        decision=SourcePolicyReview.Decision.ALLOW,
+        reason="Handmatige import uitgevoerd.",
+        scope=SourcePolicyReview.Scope.SOURCE,
+    )
+
+
+def _ensure_manual_source(*, domain: str, source_url: str, actor, mode: str) -> Source:
+    defaults = {
+        "name": _mode_source_name(domain, mode=mode),
+        "base_url": source_url,
+        "status": Source.Status.CANDIDATE,
+        "policy": Source.Policy.ALLOW,
+    }
+    source, created = Source.objects.get_or_create(
+        domain=domain,
+        source_type=Source.Type.MANUAL,
+        defaults=defaults,
+    )
+    if not created:
+        source.name = _mode_source_name(domain, mode=mode)
+        source.base_url = source_url
+        source.status = Source.Status.CANDIDATE
+        source.policy = Source.Policy.ALLOW
+        source.save(
+            update_fields=["name", "base_url", "status", "policy", "updated_at"]
+        )
+    _ensure_manual_review(source, actor=actor)
+    return source
+
+
+def _create_raw_document(
+    source: Source,
+    *,
+    source_run: SourceRun,
+    requested_url: str,
+    final_url: str,
+    content_type: str,
+    content: bytes,
+) -> RawDocument:
+    return RawDocument.objects.create(
+        source=source,
+        source_run=source_run,
+        url=requested_url,
+        final_url=final_url,
+        kind=_kind_for(content_type),
+        content_type=content_type[:200],
+        http_status=None,
+        response_headers={},
+        content_hash=hashlib.sha256(content).hexdigest(),
+        body_text=content.decode("utf-8", errors="replace"),
+        byte_length=len(content),
+        retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
+    )
+
+
+def _build_jobs_from_document(document: RawDocument) -> list[dict[str, str]]:
+    alias_qs: QuerySet[JobSourceAlias] = JobSourceAlias.objects.select_related("job").filter(
+        raw_document=document
+    )
+    jobs: list[dict[str, str]] = []
+    for alias in alias_qs:
+        title = alias.source_title or (alias.job.original_title if alias.job else "Vacature")
+        employer = alias.source_employer or "Onbekende werkgever"
+        jobs.append(
+            {
+                "id": str(alias.job_id),
+                "title": title,
+                "employer": employer,
+            }
+        )
+    return jobs
+
+
+def _run_pipeline(document: RawDocument, *, source_run: SourceRun) -> ManualImportSummary:
+    metrics = process_raw_document(document)
+    source = document.source
+    source_name = source.name if source else ""
+    warnings: list[str] = list(metrics.get("warnings", []))
+    warnings_count = len(warnings)
+    source_run.finish(
+        SourceRun.Status.SUCCESS,
+        http_status=document.source_run.http_status if document.source_run else None,
+        extracted_count=int(metrics["extracted"]),
+        created_count=int(metrics["created"]),
+        updated_count=int(metrics["updated"]),
+        duplicate_count=int(metrics["duplicates"]),
+        metrics={"parser": metrics["parser"], "warnings": warnings},
+    )
+    jobs = _build_jobs_from_document(document)
+    if metrics["created"] == 0 and metrics["updated"] == 0:
+        if not warnings:
+            warnings.append("De bron leverde geen herkenbare vacaturedata op.")
+    if not warnings:
+        # keep stable, machine-readable payload shape
+        warnings = []
+    return ManualImportSummary(
+        source_id=source.pk,
+        source_name=source_name,
+        source_url=document.final_url,
+        mode="url",
+        extracted_count=int(metrics["extracted"]),
+        created_count=int(metrics["created"]),
+        duplicate_count=int(metrics["duplicates"]),
+        warnings=warnings,
+        jobs=jobs,
+    )
+
+
+def _build_synthetic_paste_payload(raw_text: str) -> tuple[str, bytes, str]:
+    lines = [html.escape(line.strip()) for line in raw_text.splitlines() if line.strip()]
+    title = lines[0] if lines else "Handmatige vacature"
+    body = "".join(f"<p>{line}</p>" for line in lines)
+    source_domain = f"manual-{uuid4().hex[:16]}"
+    synthetic_url = f"https://{source_domain}.vacature.local/"
+    html_content = (
+        f"<html><body><main><h1>{title}</h1>{body}</main>"
+        "<footer>Handmatige import; geen externe fetch</footer></body></html>"
+    )
+    return synthetic_url, html_content.encode("utf-8"), source_domain
+
+
+def import_manual_source(
+    *, actor, source_url: str | None = None, pasted_text: str | None = None
+) -> ManualImportSummary:
+    source_url = (source_url or "").strip()
+    pasted_text = (pasted_text or "").strip()
+    if not source_url and not pasted_text:
+        raise ManualImportError("Vul een URL of tekst in.")
+
+    with transaction.atomic():
+        if source_url:
+            normalized = canonicalize_url(source_url)
+            parsed = urlsplit(normalized)
+            domain = (parsed.hostname or "").lower()
+            if not domain:
+                raise ManualImportError("De bron-URL bevat geen geldig domein.")
+            decision = assess_url(normalized)
+            if not decision.allowed:
+                raise ManualImportError(f"Import geblokkeerd: {decision.reason}")
+            source = _ensure_manual_source(
+                domain=domain,
+                source_url=normalized,
+                actor=actor,
+                mode="url",
+            )
+            source_run = SourceRun.objects.create(source=source)
+            try:
+                fetched: FetchedDocument = fetch_url(normalized, source=source)
+            except PolicyBlockedError as exc:
+                source.status = Source.Status.QUARANTINED
+                source.policy = Source.Policy.DENY
+                source.policy_reason = str(exc)[:1000]
+                source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
+                source_run.finish(
+                    SourceRun.Status.SKIPPED,
+                    error_category="policy",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Import geblokkeerd: {exc}") from exc
+            except RateLimitedError as exc:
+                source_run.finish(
+                    SourceRun.Status.FAILED,
+                    error_category="rate_limited",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Rate limiting tijdens import: {exc}") from exc
+            except FetchTimeoutError as exc:
+                source_run.finish(
+                    SourceRun.Status.FAILED,
+                    error_category="timeout",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Time-out tijdens import: {exc}") from exc
+            except FetchError as exc:
+                source_run.finish(
+                    SourceRun.Status.FAILED,
+                    error_category="fetch",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Fetch mislukt: {exc}") from exc
+
+            if fetched.status_code == 304:
+                source_run.finish(
+                    SourceRun.Status.SUCCESS,
+                    http_status=304,
+                    extracted_count=0,
+                    created_count=0,
+                    updated_count=0,
+                    duplicate_count=0,
+                    metrics={"parser": "not-modified", "warnings": []},
+                )
+                return ManualImportSummary(
+                    source_id=source.pk,
+                    source_name=source.name,
+                    source_url=source.base_url,
+                    mode="url",
+                    extracted_count=0,
+                    created_count=0,
+                    duplicate_count=0,
+                    warnings=["Geen inhoudsverandering (304)."],
+                    jobs=[],
+                )
+
+            document = _create_raw_document(
+                source,
+                source_run=source_run,
+                requested_url=fetched.requested_url,
+                final_url=fetched.final_url,
+                content_type=fetched.headers.get("content-type", ""),
+                content=fetched.content,
+            )
+            source.etag = fetched.headers.get("etag", source.etag)
+            source.last_modified = fetched.headers.get("last-modified", source.last_modified)
+            source.save(update_fields=["etag", "last_modified", "updated_at"])
+            summary = _run_pipeline(document, source_run=source_run)
+            return ManualImportSummary(
+                source_id=summary.source_id,
+                source_name=summary.source_name,
+                source_url=summary.source_url,
+                mode="url",
+                extracted_count=summary.extracted_count,
+                created_count=summary.created_count,
+                duplicate_count=summary.duplicate_count,
+                warnings=summary.warnings,
+                jobs=summary.jobs,
+            )
+
+        text = _sanitize_pasted_text(pasted_text)
+        synthetic_url, payload, synthetic_domain = _build_synthetic_paste_payload(text)
+        source = _ensure_manual_source(
+            domain=synthetic_domain,
+            source_url=synthetic_url,
+            actor=actor,
+            mode="paste",
+        )
+        source_run = SourceRun.objects.create(source=source)
+        document = _create_raw_document(
+            source,
+            source_run=source_run,
+            requested_url=synthetic_url,
+            final_url=synthetic_url,
+            content_type="text/html; charset=utf-8",
+            content=payload,
+        )
+        summary = _run_pipeline(document, source_run=source_run)
+        return ManualImportSummary(
+            source_id=summary.source_id,
+            source_name=source.name,
+            source_url=synthetic_url,
+            mode="paste",
+            extracted_count=summary.extracted_count,
+            created_count=summary.created_count,
+            duplicate_count=summary.duplicate_count,
+            warnings=summary.warnings,
+            jobs=summary.jobs,
+        )

apps/sources/services/policy.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from urllib.parse import urlsplit
+
+from django.conf import settings
+from django.utils import timezone
+
+from apps.sources.models import Source, SourcePolicyReview
+
+from .canonicalize import domain_matches
+from .robots import assess_robots
+
+DEFAULT_DENYLIST = {
+    "linkedin.com",
+    "indeed.com",
+    "indeed.be",
+    "stepstone.be",
+    "jobat.be",
+    "vdab.be",
+}
+
+
+@dataclass(frozen=True)
+class PolicyDecision:
+    allowed: bool
+    status: str
+    reason: str
+
+
+def _review_expiry(now: datetime | None = None) -> datetime:
+    return (now or timezone.now()) + timedelta(days=getattr(settings, "SOURCE_REVIEW_TTL_DAYS", 90))
+
+
+def is_denied_domain(hostname: str, denylist: set[str] | None = None) -> bool:
+    denylist = denylist or DEFAULT_DENYLIST
+    return any(domain_matches(hostname, domain) for domain in denylist)
+
+
+def create_policy_review(
+    source: Source,
+    *,
+    actor,
+    decision: SourcePolicyReview.Decision,
+    reason: str,
+    scope: SourcePolicyReview.Scope = SourcePolicyReview.Scope.SOURCE,
+    notes: str = "",
+    evidence_link: str = "",
+    expires_at: datetime | None = None,
+    metadata: dict | None = None,
+) -> SourcePolicyReview:
+    return SourcePolicyReview.objects.create(
+        source=source,
+        actor=actor if actor and getattr(actor, "pk", None) else None,
+        decision=decision,
+        scope=scope,
+        reason=reason,
+        notes=notes,
+        evidence_link=evidence_link,
+        expires_at=expires_at or _review_expiry(),
+        metadata=metadata or {},
+    )
+
+
+def _active_review(source: Source) -> SourcePolicyReview | None:
+    review = source.latest_policy_review
+    if review and not review.is_expired:
+        return review
+    return None
+
+
+def _check_review_gate(source: Source) -> PolicyDecision | None:
+    review = _active_review(source)
+    if source.policy == Source.Policy.REVIEW and source.status in {
+        Source.Status.CANDIDATE,
+        Source.Status.TRIAL,
+    }:
+        if not review:
+            return PolicyDecision(False, Source.Policy.REVIEW, "Review vereist")
+        if review.decision == SourcePolicyReview.Decision.DENY:
+            return PolicyDecision(False, Source.Policy.DENY, review.reason or "Review blokkeert bron")
+        if review.decision == SourcePolicyReview.Decision.PAUSE:
+            return PolicyDecision(False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze")
+        return None
+
+    if source.policy == Source.Policy.ALLOW:
+        if not review:
+            return PolicyDecision(False, Source.Policy.REVIEW, "Review ontbreekt voor actief beleid")
+        if review.decision == SourcePolicyReview.Decision.DENY:
+            return PolicyDecision(False, Source.Policy.DENY, review.reason or "Review blokkeert bron")
+        if review.decision == SourcePolicyReview.Decision.PAUSE:
+            return PolicyDecision(False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze")
+    return None
+
+
+def assess_url(url: str, *, source: Source | None = None) -> PolicyDecision:
+    hostname = (urlsplit(url).hostname or "").lower()
+    if not hostname:
+        return PolicyDecision(False, Source.Policy.DENY, "URL zonder hostname")
+    if is_denied_domain(hostname):
+        return PolicyDecision(False, Source.Policy.DENY, "Platformdomein staat op de denylist")
+    if source:
+        if source.status in {Source.Status.DISABLED, Source.Status.PAUSED}:
+            return PolicyDecision(False, Source.Policy.REVIEW, f"Bronstatus: {source.status}")
+        if source.policy == Source.Policy.DENY:
+            return PolicyDecision(
+                False, Source.Policy.DENY, source.policy_reason or "Bron geblokkeerd"
+            )
+        review_decision = _check_review_gate(source)
+        if review_decision is not None:
+            return review_decision
+
+    robots = assess_robots(url, source=source)
+    if not robots.allowed:
+        return PolicyDecision(False, Source.Policy.REVIEW, robots.reason)
+
+    return PolicyDecision(True, Source.Policy.ALLOW, "Toegestane publieke bron")

apps/sources/services/robots.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from urllib.parse import urlsplit, urlunsplit
+
+import httpx
+from django.conf import settings
+from django.utils import timezone
+
+from apps.sources.models import Source, SourceRobotsCache
+from .url_security import UnsafeUrlError, validate_public_url
+
+ALLOW = "allow"
+DISALLOW = "disallow"
+
+
+@dataclass(frozen=True)
+class RobotsDecision:
+    allowed: bool
+    reason: str
+
+
+def _origin_for(url: str) -> str:
+    parts = urlsplit(url)
+    if not parts.scheme:
+        raise ValueError("Ongeldige URL voor robotscontrole.")
+    host = (parts.hostname or "").lower().rstrip(".")
+    if not host:
+        raise ValueError("Host ontbreekt voor robotscontrole.")
+    port = parts.port
+    if (parts.scheme == "http" and port == 80) or (parts.scheme == "https" and port == 443) or not port:
+        netloc = host
+    else:
+        netloc = f"{host}:{port}"
+    return f"{parts.scheme}://{netloc}"
+
+
+def _path_for(url: str) -> str:
+    path = urlsplit(url).path or "/"
+    return path if path.startswith("/") else f"/{path}"
+
+
+def _robots_url(origin: str) -> str:
+    parts = urlsplit(origin)
+    return urlunsplit((parts.scheme, parts.netloc, "/robots.txt", "", ""))
+
+
+def _rules_from_text(text: str) -> dict[str, dict[str, list[str]]]:
+    bucket: dict[str, dict[str, list[str]]] = {}
+    active_agents: set[str] = set()
+
+    for raw_line in text.splitlines():
+        line = raw_line.split("#", 1)[0].strip()
+        if ":" not in line:
+            continue
+        key, value = (part.strip() for part in line.split(":", 1))
+        if not key:
+            continue
+        key_lower = key.lower()
+        if key_lower == "user-agent":
+            token = value.lower()
+            if token:
+                active_agents = {token}
+            else:
+                active_agents = set()
+            continue
+        if key_lower not in {ALLOW, DISALLOW}:
+            continue
+        if not active_agents:
+            continue
+        for agent in active_agents:
+            section = bucket.setdefault(agent, {ALLOW: [], DISALLOW: []})
+            section[key_lower].append(value.strip() or "/")
+
+    for entry in bucket.values():
+        entry[ALLOW] = list(dict.fromkeys(entry[ALLOW]))
+        entry[DISALLOW] = list(dict.fromkeys(entry[DISALLOW]))
+    return bucket
+
+
+def _pick_rules(rules: dict[str, dict[str, list[str]]], user_agent: str) -> dict[str, list[str]]:
+    normalized = user_agent.lower()
+    selected = {ALLOW: [], DISALLOW: []}
+
+    for agent, values in rules.items():
+        if agent == "*" or agent and agent in normalized:
+            selected[ALLOW].extend(values[ALLOW])
+            selected[DISALLOW].extend(values[DISALLOW])
+
+    selected[ALLOW] = list(dict.fromkeys(selected[ALLOW]))
+    selected[DISALLOW] = list(dict.fromkeys(selected[DISALLOW]))
+    return selected
+
+
+def _longest_prefix(path: str, rules: Iterable[str]) -> int:
+    return max((len(rule.rstrip("/")) for rule in rules if rule and path.startswith(rule)), default=0)
+
+
+def _evaluate_path(path: str, user_agent: str, rules: dict[str, dict[str, list[str]]]) -> RobotsDecision:
+    selected = _pick_rules(rules, user_agent=user_agent)
+    allow_len = _longest_prefix(path, selected[ALLOW])
+    disallow_len = _longest_prefix(path, selected[DISALLOW])
+    if disallow_len > allow_len:
+        return RobotsDecision(False, "Toegang geblokkeerd door robotsregels")
+    return RobotsDecision(True, "Robotsregels staan toegang toe")
+
+
+def _cache_ttl_seconds() -> float:
+    return float(getattr(settings, "ROBOTS_CACHE_TTL_SECONDS", 3600))
+
+
+def _max_bytes() -> int:
+    return int(getattr(settings, "ROBOTS_MAX_BYTES", 131072))
+
+
+def _fetch_robots(origin: str, *, client: httpx.Client | None = None) -> tuple[str, int, str, str]:
+    robots_url = _robots_url(origin)
+    validate_public_url(
+        robots_url,
+        allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
+    )
+    own_client = client is None
+    http_client = client or httpx.Client(
+        timeout=httpx.Timeout(getattr(settings, "ROBOTS_FETCH_TIMEOUT_SECONDS", 5)),
+        follow_redirects=False,
+    )
+    try:
+        response = http_client.get(
+            robots_url,
+            headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
+        )
+    finally:
+        if own_client:
+            http_client.close()
+    return (
+        response.text,
+        response.status_code,
+        response.headers.get("etag", ""),
+        response.headers.get("last-modified", ""),
+    )
+
+
+def _load_cached(origin: str, now: datetime) -> SourceRobotsCache | None:
+    try:
+        cache = SourceRobotsCache.objects.get(origin=origin)
+    except SourceRobotsCache.DoesNotExist:
+        return None
+    if cache.expires_at <= now:
+        return None
+    return cache
+
+
+def _load_stale_cache(origin: str) -> SourceRobotsCache | None:
+    try:
+        return SourceRobotsCache.objects.get(origin=origin)
+    except SourceRobotsCache.DoesNotExist:
+        return None
+
+
+def _persist_cache(
+    origin: str,
+    *,
+    status_code: int,
+    content: str,
+    etag: str = "",
+    last_modified: str = "",
+    now: datetime,
+) -> SourceRobotsCache:
+    max_bytes = _max_bytes()
+    byte_length = len(content.encode("utf-8"))
+    if byte_length > max_bytes:
+        return SourceRobotsCache.objects.update_or_create(
+            origin=origin,
+            defaults={
+                "expires_at": now + timedelta(seconds=_cache_ttl_seconds()),
+                "allow_rules": {},
+                "disallow_rules": {},
+                "error": f"robots.txt te groot ({byte_length} bytes)",
+                "etag": etag,
+                "last_modified": last_modified,
+                "byte_length": byte_length,
+            },
+        )[0]
+
+    if status_code in {404, 410}:
+        rules = {}
+    elif status_code >= 400:
+        rules = {}
+    else:
+        rules = _rules_from_text(content)
+
+    return SourceRobotsCache.objects.update_or_create(
+        origin=origin,
+        defaults={
+            "expires_at": now + timedelta(seconds=_cache_ttl_seconds()),
+            "allow_rules": {agent: value[ALLOW] for agent, value in rules.items()},
+            "disallow_rules": {agent: value[DISALLOW] for agent, value in rules.items()},
+            "error": "",
+            "etag": etag,
+            "last_modified": last_modified,
+            "byte_length": byte_length,
+        },
+    )[0]
+
+
+def _build_ruleset(cache: SourceRobotsCache) -> dict[str, dict[str, list[str]]]:
+    rules = {"allow": {}, "disallow": {}}
+    for agent, entries in cache.allow_rules.items():
+        rules["allow"][agent] = list(entries)
+    for agent, entries in cache.disallow_rules.items():
+        rules["disallow"][agent] = list(entries)
+
+    normalized_rules: dict[str, dict[str, list[str]]] = {}
+    all_agents = set(rules["allow"].keys()) | set(rules["disallow"].keys())
+    for agent in all_agents:
+        normalized_rules[agent] = {
+            ALLOW: rules["allow"].get(agent, []),
+            DISALLOW: rules["disallow"].get(agent, []),
+        }
+    return normalized_rules
+
+
+def assess_robots(
+    url: str,
+    *,
+    source: Source | None = None,
+    user_agent: str | None = None,
+    client: httpx.Client | None = None,
+    now: datetime | None = None,
+) -> RobotsDecision:
+    if source is None or not source.honor_robots:
+        return RobotsDecision(True, "Robotscontrole niet vereist")
+
+    now = now or timezone.now()
+    try:
+        origin = _origin_for(url)
+        path = _path_for(url)
+    except ValueError as exc:
+        return RobotsDecision(False, str(exc))
+
+    cache = _load_cached(origin, now=now)
+    stale = _load_stale_cache(origin)
+    if cache is None:
+        try:
+            content, status_code, etag, last_modified = _fetch_robots(origin, client=client)
+            cache = _persist_cache(
+                origin,
+                status_code=status_code,
+                content=content,
+                etag=etag,
+                last_modified=last_modified,
+                now=now,
+            )
+        except UnsafeUrlError as exc:
+            return RobotsDecision(False, f"Robotscontrole mislukt: {exc}")
+        except httpx.HTTPError as exc:
+            if stale is not None:
+                return _evaluate_path(path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=_build_ruleset(stale))
+            return RobotsDecision(True, f"Robotscontrole tijdelijk niet beschikbaar: {exc}")
+
+    if cache.error:
+        return RobotsDecision(True, cache.error)
+
+    rules = _build_ruleset(cache)
+    decision = _evaluate_path(path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=rules)
+    return decision

apps/sources/services/scheduling.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import timedelta
+from urllib.parse import urlparse
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models import Max
+from django.utils import timezone
+
+from apps.sources.models import Source, SourceLease, SourceOriginState
+
+
+def origin_key(source: Source) -> str:
+    if source.domain:
+        return source.domain.lower()
+    hostname = (urlparse(source.base_url).hostname or "").lower()
+    return hostname.rstrip(".")
+
+
+def max_origin_minimum_interval_seconds(source: Source) -> int:
+    domain = origin_key(source)
+    aggregate = Source.objects.filter(domain=domain).aggregate(
+        maximum_interval=Max("minimum_interval_seconds")
+    )
+    max_interval = aggregate["maximum_interval"]
+    if max_interval is None:
+        return int(source.minimum_interval_seconds)
+    return int(max_interval)
+
+
+def _lease_ttl_seconds() -> int:
+    ttl = settings.SOURCE_LEASE_TTL_SECONDS
+    if ttl <= 0:
+        return 180
+    return int(ttl)
+
+
+def _max_origin_interval_seconds(source: Source) -> int:
+    explicit = settings.SOURCE_ORIGIN_MIN_INTERVAL_SECONDS
+    if explicit > 0:
+        return int(explicit)
+    return max_origin_minimum_interval_seconds(source)
+
+
+def calculate_jitter_seconds(source: Source, *, max_seconds: int) -> int:
+    max_seconds = int(max_seconds)
+    if max_seconds <= 0:
+        return 0
+    jitter_seed = f"{source.pk}:{source.domain}:{source.minimum_interval_seconds}"
+    digest = hashlib.sha256(jitter_seed.encode("utf-8")).digest()
+    jitter_span = max_seconds + 1
+    return int(int.from_bytes(digest[:8], "big") % jitter_span)
+
+
+def calculate_failure_backoff_seconds(
+    source: Source,
+    *,
+    failure_count: int,
+    retry_after_seconds: int | None = None,
+    timeout: bool = False,
+) -> int:
+    failure_count = max(1, failure_count)
+    base_seconds = settings.SOURCE_FAILURE_BACKOFF_BASE_SECONDS
+    if timeout:
+        base_seconds = max(base_seconds, settings.SOURCE_FAILURE_TIMEOUT_BASE_SECONDS)
+    factor = 2 ** min(failure_count - 1, 10)
+    backoff_seconds = base_seconds * factor
+    if retry_after_seconds:
+        backoff_seconds = max(backoff_seconds, retry_after_seconds)
+    max_seconds = settings.SOURCE_FAILURE_BACKOFF_MAX_SECONDS
+    return min(max_seconds, backoff_seconds) + calculate_jitter_seconds(
+        source, max_seconds=settings.SOURCE_FAILURE_JITTER_SECONDS
+    )
+
+
+def calculate_success_jitter_seconds(source: Source) -> int:
+    return calculate_jitter_seconds(
+        source, max_seconds=settings.SOURCE_SUCCESS_JITTER_SECONDS
+    )
+
+
+def acquire_source_lease(
+    *, source_id: int, worker_token: str, now=None
+) -> SourceLease | None:
+    now = now or timezone.now()
+    with transaction.atomic():
+        source = Source.objects.select_for_update().get(pk=source_id)
+        if not source.domain:
+            return None
+
+        domain = origin_key(source)
+        origin_state = SourceOriginState.objects.select_for_update().get_or_create(
+            domain=domain, defaults={"next_allowed_at": now - timedelta(seconds=1)}
+        )[0]
+
+        if origin_state.next_allowed_at and origin_state.next_allowed_at > now:
+            existing_lease = SourceLease.objects.select_for_update().filter(source=source).first()
+            if not existing_lease or existing_lease.token != worker_token:
+                return None
+
+        lease = SourceLease.objects.select_for_update().filter(source=source).first()
+        if lease is not None and not lease.is_expired:
+            if lease.token != worker_token:
+                return None
+
+        active_leases = SourceLease.objects.select_for_update().filter(
+            source__domain=domain, expires_at__gt=now
+        )
+        active_count = active_leases.exclude(source=source).count()
+        max_concurrency = max(1, int(source.max_concurrency))
+        if active_count >= max_concurrency:
+            return None
+
+        if lease is None:
+            lease = SourceLease(source=source)
+        lease.token = worker_token
+        lease.worker_id = worker_token
+        lease.expires_at = now + timedelta(seconds=_lease_ttl_seconds())
+        lease.save(update_fields=["token", "worker_id", "expires_at", "updated_at"])
+        return lease
+
+
+def release_source_lease(*, source_id: int, worker_token: str, now=None) -> bool:
+    now = now or timezone.now()
+    with transaction.atomic():
+        source = Source.objects.select_for_update().get(pk=source_id)
+        if not source.domain:
+            return False
+
+        lease = SourceLease.objects.select_for_update().filter(
+            source=source, token=worker_token
+        ).first()
+        if not lease:
+            return False
+
+        lease.expires_at = now
+        lease.save(update_fields=["expires_at", "updated_at"])
+
+        interval_seconds = _max_origin_interval_seconds(source)
+        domain = origin_key(source)
+        origin_state = SourceOriginState.objects.select_for_update().get_or_create(
+            domain=domain, defaults={"next_allowed_at": now}
+        )[0]
+        origin_state.next_allowed_at = now + timedelta(seconds=max(0, interval_seconds))
+        origin_state.save(update_fields=["next_allowed_at", "updated_at"])
+        return True

apps/sources/services/url_security.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import ipaddress
+import socket
+from collections.abc import Callable
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
+Resolver = Callable[..., list[tuple]]
+
+
+class UnsafeUrlError(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class ValidatedUrl:
+    url: str
+    hostname: str
+    port: int
+    addresses: tuple[str, ...]
+
+
+def _extract_addresses(results: list[tuple]) -> tuple[str, ...]:
+    addresses: list[str] = []
+    for result in results:
+        sockaddr = result[4]
+        if sockaddr:
+            addresses.append(str(sockaddr[0]))
+    return tuple(dict.fromkeys(addresses))
+
+
+def validate_public_url(
+    url: str,
+    *,
+    resolver: Resolver = socket.getaddrinfo,
+    allow_nonstandard_ports: bool = False,
+) -> ValidatedUrl:
+    parts = urlsplit(url)
+    if parts.scheme.lower() not in {"http", "https"}:
+        raise UnsafeUrlError("Alleen http en https zijn toegestaan.")
+    if parts.username or parts.password:
+        raise UnsafeUrlError("URLs met ingebedde credentials zijn niet toegestaan.")
+    hostname = (parts.hostname or "").lower().rstrip(".")
+    if not hostname:
+        raise UnsafeUrlError("URL bevat geen hostname.")
+    if hostname == "localhost" or hostname.endswith(".localhost"):
+        raise UnsafeUrlError("Lokale hostnames zijn niet toegestaan.")
+    port = parts.port or (443 if parts.scheme.lower() == "https" else 80)
+    if not allow_nonstandard_ports and port not in {80, 443}:
+        raise UnsafeUrlError("Niet-standaard poorten zijn niet toegestaan.")
+    try:
+        direct_ip = ipaddress.ip_address(hostname.strip("[]"))
+        addresses = (str(direct_ip),)
+    except ValueError:
+        try:
+            results = resolver(hostname, port, type=socket.SOCK_STREAM)
+        except OSError as exc:
+            raise UnsafeUrlError("Hostname kan niet veilig worden opgelost.") from exc
+        addresses = _extract_addresses(results)
+    if not addresses:
+        raise UnsafeUrlError("Hostname leverde geen IP-adressen op.")
+    for raw in addresses:
+        try:
+            ip = ipaddress.ip_address(raw)
+        except ValueError as exc:
+            raise UnsafeUrlError("Ongeldig IP-adres na DNS-resolutie.") from exc
+        if not ip.is_global:
+            raise UnsafeUrlError(f"Niet-publiek IP-adres geblokkeerd: {ip}")
+    return ValidatedUrl(url=url, hostname=hostname, port=port, addresses=addresses)

apps/sources/tasks.py

@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+import imaplib
+from contextlib import suppress
+from datetime import timedelta
+from uuid import uuid4
+
+from celery import shared_task
+from django.conf import settings
+from django.db.models import Q
+from django.utils import timezone
+
+from apps.jobs.services.pipeline import process_raw_document
+from apps.sources.models import EmailMessageRecord, RawDocument, Source, SourceRun
+from apps.sources.services.email_import import ingest_email, message_identity
+from apps.sources.services.fetcher import (
+    FetchError,
+    FetchTimeoutError,
+    PolicyBlockedError,
+    RateLimitedError,
+    fetch_url,
+)
+from apps.sources.services.health import canary_recovery_sources, evaluate_source_health, start_health_canary
+from apps.sources.services.policy import assess_url
+from apps.sources.services.scheduling import (
+    acquire_source_lease,
+    calculate_failure_backoff_seconds,
+    calculate_success_jitter_seconds,
+    release_source_lease,
+)
+
+
+def _kind_for(content_type: str) -> str:
+    content_type = (content_type or "").lower()
+    if "html" in content_type:
+        return RawDocument.Kind.HTML
+    if "xml" in content_type or "rss" in content_type or "atom" in content_type:
+        return RawDocument.Kind.XML
+    if "json" in content_type:
+        return RawDocument.Kind.JSON
+    return RawDocument.Kind.TEXT
+
+
+@shared_task(
+    bind=True,
+    name="apps.sources.tasks.fetch_source",
+)
+def fetch_source(self, source_id: int, force: bool = False) -> dict[str, object]:
+    source = Source.objects.get(pk=source_id)
+    run = SourceRun.objects.create(source=source)
+    now = timezone.now()
+    lease_token = str(self.request.id or uuid4())
+    lease = None
+    decision = assess_url(source.base_url, source=source)
+    if source.status not in {Source.Status.ACTIVE, Source.Status.TRIAL, Source.Status.CANDIDATE}:
+        run.finish(SourceRun.Status.SKIPPED, error_category="status")
+        return {"status": "skipped", "reason": "status"}
+    if not decision.allowed:
+        run.finish(
+            SourceRun.Status.SKIPPED,
+            error_category="policy",
+            error_message=decision.reason,
+        )
+        return {"status": "skipped", "reason": "policy"}
+    if not force and source.next_run_at and source.next_run_at > now:
+        run.finish(SourceRun.Status.SKIPPED, error_category="not_due")
+        return {"status": "skipped", "reason": "not_due"}
+
+    lease = acquire_source_lease(source_id=source.pk, worker_token=lease_token, now=now)
+    if lease is None:
+        run.finish(
+            SourceRun.Status.SKIPPED,
+            error_category="lease",
+            error_message="Bron staat al in verwerking.",
+        )
+        return {"status": "skipped", "reason": "lease"}
+    headers: dict[str, str] = {}
+    if source.etag:
+        headers["If-None-Match"] = source.etag
+    if source.last_modified:
+        headers["If-Modified-Since"] = source.last_modified
+    try:
+        fetched = fetch_url(source.base_url, source=source, conditional_headers=headers)
+        if fetched.status_code == 304:
+            source.schedule_after_success(
+                jitter_seconds=calculate_success_jitter_seconds(source)
+            )
+            run.finish(SourceRun.Status.SUCCESS, http_status=304)
+            return {"status": "not_modified"}
+        content_type = fetched.headers.get("content-type", "")
+        document = RawDocument.objects.create(
+            source=source,
+            source_run=run,
+            url=fetched.requested_url,
+            final_url=fetched.final_url,
+            kind=_kind_for(content_type),
+            content_type=content_type[:200],
+            http_status=fetched.status_code,
+            response_headers={
+                key: value
+                for key, value in fetched.headers.items()
+                if key.lower() in {"etag", "last-modified", "content-type", "cache-control"}
+            },
+            content_hash=fetched.sha256,
+            body_text=fetched.text,
+            byte_length=len(fetched.content),
+            retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
+        )
+        metrics = process_raw_document(document)
+        source.etag = fetched.headers.get("etag", source.etag)
+        source.last_modified = fetched.headers.get("last-modified", source.last_modified)
+        source.save(update_fields=["etag", "last_modified", "updated_at"])
+        source.schedule_after_success(
+            jitter_seconds=calculate_success_jitter_seconds(source)
+        )
+        run.finish(
+            SourceRun.Status.SUCCESS,
+            http_status=fetched.status_code,
+            extracted_count=int(metrics["extracted"]),
+            created_count=int(metrics["created"]),
+            updated_count=int(metrics["updated"]),
+            duplicate_count=int(metrics["duplicates"]),
+            metrics={"parser": metrics["parser"], "warnings": metrics["warnings"]},
+        )
+        return metrics
+    except PolicyBlockedError as exc:
+        source.status = Source.Status.QUARANTINED
+        source.policy = Source.Policy.DENY
+        source.policy_reason = str(exc)
+        source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
+        run.finish(
+            SourceRun.Status.SKIPPED,
+            error_category="policy",
+            error_message=str(exc)[:1000],
+        )
+        return {"status": "blocked", "reason": str(exc)}
+    except RateLimitedError as exc:
+        backoff = calculate_failure_backoff_seconds(
+            source,
+            failure_count=source.failure_count + 1,
+            retry_after_seconds=exc.retry_after_seconds,
+        )
+        source.schedule_after_failure(now=timezone.now(), backoff_seconds=backoff)
+        run.finish(
+            SourceRun.Status.FAILED,
+            error_category="rate_limited",
+            error_message=str(exc)[:1000],
+        )
+        return {"status": "rate_limited", "backoff": backoff}
+    except FetchTimeoutError as exc:
+        backoff = calculate_failure_backoff_seconds(
+            source, failure_count=source.failure_count + 1, timeout=True
+        )
+        source.schedule_after_failure(now=timezone.now(), backoff_seconds=backoff)
+        run.finish(
+            SourceRun.Status.FAILED,
+            error_category="timeout",
+            error_message=str(exc)[:1000],
+        )
+        return {"status": "timeout", "backoff": backoff}
+    except FetchError as exc:
+        backoff = calculate_failure_backoff_seconds(
+            source, failure_count=source.failure_count + 1
+        )
+        source.schedule_after_failure(now=timezone.now(), backoff_seconds=backoff)
+        run.finish(
+            SourceRun.Status.FAILED,
+            error_category=exc.__class__.__name__,
+            error_message=str(exc)[:1000],
+        )
+        return {"status": "failed", "error_category": exc.__class__.__name__, "backoff": backoff}
+    except Exception as exc:
+        source.schedule_after_failure()
+        run.finish(
+            SourceRun.Status.FAILED,
+            error_category="unexpected",
+            error_message=str(exc)[:1000],
+        )
+        return {"status": "failed", "error_category": "unexpected"}
+    finally:
+        if lease is not None:
+            release_source_lease(source_id=source.pk, worker_token=lease_token)
+
+
+@shared_task(name="apps.sources.tasks.schedule_due_sources")
+def schedule_due_sources() -> dict[str, int]:
+    now = timezone.now()
+    due = Source.objects.filter(
+        status__in=[Source.Status.ACTIVE, Source.Status.TRIAL],
+    ).filter(Q(next_run_at__isnull=True) | Q(next_run_at__lte=now))
+    ids = list(due.values_list("id", flat=True).distinct()[:200])
+    for source_id in ids:
+        fetch_source.delay(source_id)
+    return {"scheduled": len(ids)}
+
+
+@shared_task(name="apps.sources.tasks.poll_imap_inbox")
+def poll_imap_inbox() -> dict[str, int | str]:
+    if not settings.IMAP_ENABLED:
+        return {"status": "disabled", "imported": 0}
+    if not settings.IMAP_HOST or not settings.IMAP_USER or not settings.IMAP_PASSWORD:
+        return {"status": "not_configured", "imported": 0}
+    client_cls = imaplib.IMAP4_SSL if settings.IMAP_USE_SSL else imaplib.IMAP4
+    client = client_cls(settings.IMAP_HOST, settings.IMAP_PORT)
+    imported = 0
+    try:
+        client.login(settings.IMAP_USER, settings.IMAP_PASSWORD)
+        status, _ = client.select(settings.IMAP_MAILBOX, readonly=not settings.IMAP_MARK_SEEN)
+        if status != "OK":
+            raise RuntimeError("IMAP-mailbox kon niet worden geopend")
+        status, data = client.uid("search", None, "ALL")
+        if status != "OK":
+            raise RuntimeError("IMAP-zoekopdracht mislukt")
+        uids = (data[0] or b"").split()[-200:]
+        for uid in uids:
+            status, payload = client.uid("fetch", uid, "(RFC822)")
+            if status != "OK" or not payload:
+                continue
+            raw = next((part[1] for part in payload if isinstance(part, tuple)), None)
+            if not raw:
+                continue
+            message_id = message_identity(raw)
+            was_known = EmailMessageRecord.objects.filter(message_id=message_id).exists()
+            ingest_email(raw, mailbox=settings.IMAP_MAILBOX)
+            if not was_known:
+                imported += 1
+        return {"status": "ok", "imported": imported}
+    finally:
+        with suppress(Exception):
+            client.logout()
+
+
+@shared_task(name="apps.sources.tasks.cleanup_raw_documents")
+def cleanup_raw_documents() -> dict[str, int]:
+    deleted, _ = RawDocument.objects.filter(
+        retain_until__lt=timezone.now(), quarantined=False
+    ).delete()
+    return {"deleted": deleted}
+
+
+@shared_task(name="apps.sources.tasks.enforce_source_health")
+def enforce_source_health() -> dict[str, int]:
+    return evaluate_source_health()
+
+
+@shared_task(name="apps.sources.tasks.recover_source_health")
+def recover_source_health() -> dict[str, int]:
+    started = 0
+    for source in canary_recovery_sources():
+        start_health_canary(source)
+        fetch_source.delay(source.pk, force=True)
+        started += 1
+    return {"canary_started": started}

apps/sources/urls.py

@@ -0,0 +1,10 @@
+from django.urls import path
+
+from .views import SourceListView, bulk_candidate_action, manual_import_view, retry_source
+
+urlpatterns = [
+    path("", SourceListView.as_view(), name="list"),
+    path("manual-import/", manual_import_view, name="manual_import"),
+    path("bulk/", bulk_candidate_action, name="bulk"),
+    path("<int:pk>/retry/", retry_source, name="retry"),
+]

apps/sources/views.py

@@ -0,0 +1,222 @@
+from __future__ import annotations
+
+from django.conf import settings
+from django.contrib import messages
+from django.contrib.auth.decorators import login_required
+from django.contrib.auth.mixins import LoginRequiredMixin
+from django.http import HttpRequest
+from django.shortcuts import get_object_or_404, redirect, render
+from django.urls import reverse
+from django.views.decorators.cache import never_cache
+from django.views.decorators.http import require_POST
+from django.views.generic import ListView
+
+from .forms import ManualImportForm
+from .models import Source
+from .models import SourcePolicyReview
+from .services.manual_import import ManualImportError, ManualImportSummary, import_manual_source
+from .services.policy import create_policy_review
+from .tasks import fetch_source
+from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
+
+
+def _source_list_queryset():
+    return Source.objects.prefetch_related("policy_reviews").all().order_by("name")
+
+
+def _manual_import_context(
+    request: HttpRequest, *, form: ManualImportForm, manual_result=None
+):
+    base_queryset = _source_list_queryset()
+    return {
+        "sources": base_queryset,
+        "active_source_count": Source.objects.filter(status=Source.Status.ACTIVE).count(),
+        "source_total_count": base_queryset.count(),
+        "review_pending_count": Source.objects.filter(policy=Source.Policy.REVIEW).count(),
+        "manual_import_form": form,
+        "manual_import_bookmarklet_endpoint": request.build_absolute_uri(
+            reverse("sources:manual_import")
+        ),
+        "manual_import_paste_max_bytes": settings.MANUAL_IMPORT_PASTE_MAX_BYTES,
+        "manual_import_result": manual_result,
+    }
+
+
+class SourceListView(LoginRequiredMixin, ListView):
+    model = Source
+    template_name = "sources/list.html"
+    context_object_name = "sources"
+
+    def get_queryset(self):
+        return _source_list_queryset()
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context.update(
+            _manual_import_context(
+                self.request,
+                form=ManualImportForm(
+                    initial={"source_url": self.request.GET.get("source_url", "")}
+                ),
+            )
+        )
+        return context
+
+
+def _bulk_summary(action: str) -> str:
+    mapping = {
+        "allow": "toestaan",
+        "approve_to_trial": "proefrun",
+        "pause": "pauzeren",
+        "dismiss": "afwijzen",
+    }
+    return mapping.get(action, "onbekend")
+
+
+def _action_to_review(
+    action: str,
+) -> tuple[SourcePolicyReview.Decision, str, str, str]:
+    if action == "allow":
+        return (
+            SourcePolicyReview.Decision.ALLOW,
+            Source.Status.ACTIVE,
+            Source.Policy.ALLOW,
+            "Bron actief gemaakt door expliciete goedkeuring",
+        )
+    if action == "approve_to_trial":
+        return (
+            SourcePolicyReview.Decision.TRIAL,
+            Source.Status.TRIAL,
+            Source.Policy.REVIEW,
+            "Bron naar proefrun voor technische controle",
+        )
+    if action == "pause":
+        return (
+            SourcePolicyReview.Decision.PAUSE,
+            Source.Status.PAUSED,
+            Source.Policy.REVIEW,
+            "Bron tijdelijk gepauzeerd",
+        )
+    return (
+        SourcePolicyReview.Decision.DENY,
+        Source.Status.DISABLED,
+        Source.Policy.DENY,
+        "Bron afgewezen",
+    )
+
+
+@login_required
+@require_POST
+def bulk_candidate_action(request):
+    action = request.POST.get("action")
+    source_ids = [int(raw_id) for raw_id in request.POST.getlist("source_ids") if raw_id.isdigit()]
+    sources = Source.objects.filter(id__in=source_ids).exclude(policy=Source.Policy.DENY)
+    if not sources.exists():
+        messages.info(request, "Geen bruikbare bronregels geselecteerd.")
+        return redirect("sources:list")
+
+    if action not in {"allow", "approve_to_trial", "pause", "dismiss"}:
+        messages.error(request, "Onbekende actie.")
+        return redirect("sources:list")
+
+    decision, status, policy, reason = _action_to_review(action)
+    for source in sources:
+        create_policy_review(
+            source,
+            actor=request.user,
+            decision=decision,
+            scope=SourcePolicyReview.Scope.SOURCE,
+            reason=reason,
+            notes="Bulkactie",
+        )
+        source.status = status
+        source.policy = policy
+        source.policy_reason = reason
+        source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
+
+    messages.success(request, f"{sources.count()} bron(nen) naar { _bulk_summary(action) }.")
+    return redirect("sources:list")
+
+
+@login_required
+def retry_source(request, pk: int):
+    if request.method == "POST":
+        source = get_object_or_404(Source, pk=pk)
+        if source.policy == Source.Policy.DENY:
+            messages.error(request, "Deze bron is door het bronbeleid geblokkeerd.")
+        else:
+            fetch_source.delay(source.pk, force=True)
+            messages.success(request, "Broncontrole ingepland.")
+    return redirect("sources:list")
+
+
+@login_required
+@never_cache
+def manual_import_view(request):
+    rate_limit = is_rate_limited(
+        request,
+        namespace="manual_import",
+        max_attempts=settings.MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS,
+        window_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_WINDOW_SECONDS,
+        block_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_BLOCK_SECONDS,
+    )
+    if rate_limit.is_blocked:
+        messages.error(
+            request,
+            "Te veel handmatige importverzoeken. Wacht 5 minuten en probeer daarna opnieuw.",
+        )
+        return render(
+            request,
+            "sources/list.html",
+            _manual_import_context(request, form=ManualImportForm(), manual_result=None),
+        )
+
+    if request.method == "POST":
+        form = ManualImportForm(request.POST)
+        result_payload = None
+        if not form.is_valid():
+            messages.error(request, "Controleer de handmatige importvelden.")
+            return render(
+                request,
+                "sources/list.html",
+                _manual_import_context(request, form=form, manual_result=result_payload),
+            )
+
+        cleaned = form.cleaned_data
+        try:
+            summary: ManualImportSummary = import_manual_source(
+                actor=request.user,
+                source_url=cleaned.get("source_url"),
+                pasted_text=cleaned.get("pasted_text"),
+            )
+            result_payload = summary.to_session_payload()
+            messages.success(
+                request,
+                "Handmatige import uitgevoerd. Bekijk het resultaat hieronder.",
+            )
+            clear_rate_limit(request, namespace="manual_import")
+            form = ManualImportForm()
+        except ManualImportError as exc:
+            messages.error(request, str(exc))
+            register_rate_limit_failure(
+                request,
+                namespace="manual_import",
+                max_attempts=settings.MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS,
+                window_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_WINDOW_SECONDS,
+                block_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_BLOCK_SECONDS,
+            )
+
+        return render(
+            request,
+            "sources/list.html",
+            _manual_import_context(request, form=form, manual_result=result_payload),
+        )
+
+    return render(
+        request,
+        "sources/list.html",
+        _manual_import_context(
+            request,
+            form=ManualImportForm(initial={"source_url": request.GET.get("source_url", "")}),
+        ),
+    )