feat: release regional radar and mailbox integrations

apps/sources/services/discovery.py

@@ -140,23 +140,28 @@ def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int
         for candidate in ordered:
             provenance = _provenance_entry(candidate)
             now = _utc_now()
-            source, was_created = Source.objects.get_or_create(
-                domain=candidate.domain,
-                source_type=candidate.source_type,
-                defaults={
-                    "name": _candidate_name(candidate),
-                    "base_url": candidate.url,
-                    "status": Source.Status.CANDIDATE,
-                    "policy": Source.Policy.REVIEW,
-                    "policy_reason": "Automatisch ontdekt",
-                    "parser_key": "auto",
-                    "strict_mode": True,
-                    "metadata": {
-                        "discovery": [provenance],
-                        "discovered_at": now,
-                    },
-                },
+            source = (
+                Source.objects.filter(
+                    domain=candidate.domain,
+                    source_type=candidate.source_type,
+                )
+                .order_by("id")
+                .first()
             )
+            was_created = source is None
+            if source is None:
+                source = Source.objects.create(
+                    domain=candidate.domain,
+                    source_type=candidate.source_type,
+                    name=_candidate_name(candidate),
+                    base_url=candidate.url,
+                    status=Source.Status.CANDIDATE,
+                    policy=Source.Policy.REVIEW,
+                    policy_reason="Automatisch ontdekt",
+                    parser_key="auto",
+                    strict_mode=True,
+                    metadata={"discovery": [provenance], "discovered_at": now},
+                )
             if was_created:
                 created += 1
                 continue

apps/sources/services/email_import.py

@@ -13,7 +13,8 @@ from django.utils import timezone
 from apps.jobs.services.normalization import normalize_extracted_job
 from apps.jobs.services.pipeline import persist_draft
 from apps.sources.adapters.email_alert import EmailAlertAdapter
-from apps.sources.models import EmailMessageRecord, RawDocument, Source
+from apps.sources.models import EmailMessageRecord, MailboxConnection, RawDocument, Source
+from apps.sources.platforms import platform_label
 
 
 def message_identity(raw_message: bytes) -> str:
@@ -24,22 +25,32 @@ def message_identity(raw_message: bytes) -> str:
 
 
 @transaction.atomic
-def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageRecord:
+def ingest_email(
+    raw_message: bytes,
+    *,
+    mailbox: str = "INBOX",
+    mailbox_connection: MailboxConnection | None = None,
+    platform: str | None = None,
+) -> EmailMessageRecord:
     parsed = BytesParser(policy=policy.default).parsebytes(raw_message)
     message_id = message_identity(raw_message)
-    existing = EmailMessageRecord.objects.filter(message_id=message_id).first()
+    existing = EmailMessageRecord.objects.filter(
+        message_id=message_id, mailbox_connection=mailbox_connection
+    ).first()
     if existing:
         return existing
 
+    source_domain = f"{platform}.mailbox.local" if platform else "mailbox.local"
+    source_name = f"{platform_label(platform)} vacaturemailbox" if platform else "Vacaturemailbox"
     source, _ = Source.objects.get_or_create(
-        domain="mailbox.local",
+        domain=source_domain,
         source_type=Source.Type.EMAIL,
         defaults={
-            "name": "Vacaturemailbox",
+            "name": source_name,
             "status": Source.Status.ACTIVE,
             "policy": Source.Policy.ALLOW,
             "parser_key": "email-alert",
-            "crawl_interval_minutes": 10,
+            "crawl_interval_minutes": 1440,
         },
     )
     content_hash = hashlib.sha256(raw_message).hexdigest()
@@ -52,7 +63,11 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
         body_text=raw_message.decode("utf-8", errors="replace"),
         byte_length=len(raw_message),
         retain_until=retain_until,
-        metadata={"mailbox": mailbox},
+        metadata={
+            "mailbox": mailbox,
+            "mailbox_connection_id": mailbox_connection.pk if mailbox_connection else None,
+            "alert_provider": platform or "unknown",
+        },
     )
     received_at = None
     if parsed.get("date"):
@@ -64,12 +79,28 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
             received_at = None
 
     adapter = EmailAlertAdapter()
-    result = adapter.extract_message(raw_message)
+    result = adapter.extract_message(raw_message, expected_provider=platform)
+    detected_provider = next(
+        (
+            str(extracted.raw.get("alert_provider"))
+            for extracted in result.jobs
+            if extracted.raw.get("alert_provider")
+        ),
+        "unknown",
+    )
+    alert_provider = platform or detected_provider
     document.parser_key = result.parser_key
     document.parser_version = result.parser_version
     document.extraction_confidence = result.confidence
+    document.metadata = {**document.metadata, "alert_provider": alert_provider}
     document.save(
-        update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"]
+        update_fields=[
+            "parser_key",
+            "parser_version",
+            "extraction_confidence",
+            "metadata",
+            "updated_at",
+        ]
     )
     links: list[str] = []
     errors: list[str] = []
@@ -87,6 +118,7 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
         except Exception as exc:
             errors.append(f"{exc.__class__.__name__}: {exc}")
     return EmailMessageRecord.objects.create(
+        mailbox_connection=mailbox_connection,
         message_id=message_id,
         mailbox=mailbox,
         sender=str(parsed.get("from") or "")[:500],

apps/sources/services/fetcher.py

@@ -14,6 +14,7 @@ from django.conf import settings
 from apps.sources.models import Source
 
 from .policy import assess_url
+from .tls import trusted_tls_context
 from .url_security import ValidatedUrl, validate_public_url
 
 ALLOWED_CONTENT_TYPES = (
@@ -123,6 +124,7 @@ def fetch_url(
         timeout=httpx.Timeout(settings.FETCHER_TIMEOUT_SECONDS),
         follow_redirects=False,
         headers=headers,
+        verify=trusted_tls_context(),
     )
     current_url = url
     previous_validation: ValidatedUrl | None = None

apps/sources/services/imap_import.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import imaplib
+from collections.abc import Callable
+from contextlib import suppress
+from typing import Any
+
+from django.conf import settings
+
+from apps.sources.models import EmailMessageRecord, MailboxConnection
+from apps.sources.services.email_import import ingest_email, message_identity
+from apps.sources.services.mailbox_connections import decrypt_mailbox_password
+from apps.sources.services.tls import trusted_tls_context
+from apps.sources.services.url_security import validate_public_url
+
+ImapClientFactory = Callable[..., Any]
+HostValidator = Callable[[str], object]
+
+
+def poll_imap_mailbox(
+    connection: MailboxConnection,
+    *,
+    client_factory: ImapClientFactory | None = None,
+    host_validator: HostValidator = validate_public_url,
+) -> dict[str, int | str]:
+    """Import one bounded mailbox batch without letting one bad message abort the batch."""
+    if not connection.enabled:
+        return {"status": "disabled", "imported": 0}
+
+    if client_factory is None:
+        client_factory = imaplib.IMAP4_SSL
+    host_validator(f"https://{connection.imap_host}")
+    password = decrypt_mailbox_password(connection)
+    client_options = {"timeout": settings.IMAP_CONNECT_TIMEOUT_SECONDS}
+    if client_factory is imaplib.IMAP4_SSL:
+        client_options["ssl_context"] = trusted_tls_context()
+    client = client_factory(connection.imap_host, connection.port, **client_options)
+    counters = {
+        "imported": 0,
+        "duplicates": 0,
+        "skipped_oversized": 0,
+        "failed": 0,
+    }
+    try:
+        client.login(connection.username, password)
+        status, _ = client.select(connection.mailbox, readonly=True)
+        if status != "OK":
+            raise RuntimeError("IMAP-mailbox kon niet worden geopend")
+        status, data = client.uid("search", None, "ALL")
+        if status != "OK":
+            raise RuntimeError("IMAP-zoekopdracht mislukt")
+        uids = (data[0] or b"").split()[-settings.IMAP_MAX_MESSAGES_PER_POLL :]
+        for uid in uids:
+            try:
+                status, payload = client.uid("fetch", uid, "(RFC822)")
+                if status != "OK" or not payload:
+                    counters["failed"] += 1
+                    continue
+                raw = next((part[1] for part in payload if isinstance(part, tuple)), None)
+                if not isinstance(raw, bytes):
+                    counters["failed"] += 1
+                    continue
+                if len(raw) > settings.IMAP_MAX_MESSAGE_BYTES:
+                    counters["skipped_oversized"] += 1
+                    continue
+                identity = message_identity(raw)
+                if EmailMessageRecord.objects.filter(
+                    mailbox_connection=connection, message_id=identity
+                ).exists():
+                    counters["duplicates"] += 1
+                    continue
+                ingest_email(
+                    raw,
+                    mailbox=connection.mailbox,
+                    mailbox_connection=connection,
+                    platform=connection.platform,
+                )
+                counters["imported"] += 1
+            except Exception:
+                counters["failed"] += 1
+        return {"status": "ok", **counters}
+    finally:
+        with suppress(Exception):
+            client.logout()

apps/sources/services/mailbox_connections.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from uuid import uuid4
+
+from cryptography.fernet import Fernet, InvalidToken, MultiFernet
+from django.conf import settings
+from django.db import transaction
+from django.utils import timezone
+
+from apps.sources.models import MailboxConnection
+
+
+class MailboxCredentialError(RuntimeError):
+    """Safe credential configuration error without secret material."""
+
+
+def _cipher() -> MultiFernet:
+    keys = settings.MAILBOX_CREDENTIAL_KEYS
+    if not keys:
+        raise MailboxCredentialError(
+            "Credentialopslag is niet geconfigureerd. Stel MAILBOX_CREDENTIAL_KEYS in."
+        )
+    try:
+        return MultiFernet([Fernet(key.encode("ascii")) for key in keys])
+    except (ValueError, TypeError) as exc:
+        raise MailboxCredentialError(
+            "MAILBOX_CREDENTIAL_KEYS bevat een ongeldige sleutel."
+        ) from exc
+
+
+def encrypt_mailbox_password(password: str) -> str:
+    if not password:
+        raise MailboxCredentialError("Een app-wachtwoord is verplicht.")
+    return _cipher().encrypt(password.encode("utf-8")).decode("ascii")
+
+
+def decrypt_mailbox_password(connection: MailboxConnection) -> str:
+    try:
+        return _cipher().decrypt(connection.encrypted_password.encode("ascii")).decode("utf-8")
+    except (InvalidToken, ValueError, UnicodeError) as exc:
+        raise MailboxCredentialError(
+            "Het opgeslagen app-wachtwoord kan niet worden ontsleuteld. Sla het opnieuw op."
+        ) from exc
+
+
+@transaction.atomic
+def rotate_mailbox_credentials() -> int:
+    """Re-encrypt every mailbox password with the first configured key."""
+    rotated = 0
+    for connection in MailboxConnection.objects.select_for_update().all():
+        password = decrypt_mailbox_password(connection)
+        connection.encrypted_password = encrypt_mailbox_password(password)
+        connection.save(update_fields=["encrypted_password", "updated_at"])
+        rotated += 1
+    return rotated
+
+
+@transaction.atomic
+def save_mailbox_connection(
+    *,
+    user,
+    cleaned_data: dict[str, object],
+    instance: MailboxConnection | None = None,
+) -> MailboxConnection:
+    connection = instance or MailboxConnection(user=user)
+    if connection.pk and connection.user_id != user.pk:
+        raise PermissionError("Mailboxkoppeling behoort niet tot deze gebruiker.")
+
+    password = str(cleaned_data.pop("password", "") or "")
+    for field in (
+        "platform",
+        "provider",
+        "custom_host",
+        "port",
+        "username",
+        "mailbox",
+        "enabled",
+        "poll_interval_minutes",
+    ):
+        setattr(connection, field, cleaned_data[field])
+    if password:
+        connection.encrypted_password = encrypt_mailbox_password(password)
+    elif not connection.encrypted_password:
+        raise MailboxCredentialError("Een app-wachtwoord is verplicht.")
+    connection.next_poll_at = timezone.now()
+    connection.last_error_category = ""
+    connection.last_error_message = ""
+    connection.full_clean()
+    connection.save()
+    return connection
+
+
+@transaction.atomic
+def claim_mailbox_connection(
+    *, connection_id: int, worker_token: str | None = None, force: bool = False
+) -> tuple[MailboxConnection, str] | None:
+    now = timezone.now()
+    connection = MailboxConnection.objects.select_for_update().get(pk=connection_id)
+    if not connection.enabled:
+        return None
+    if connection.lease_expires_at and connection.lease_expires_at > now:
+        return None
+    if not force and connection.next_poll_at > now:
+        return None
+    token = worker_token or uuid4().hex
+    connection.lease_token = token
+    connection.lease_expires_at = now + timedelta(minutes=5)
+    connection.save(update_fields=["lease_token", "lease_expires_at", "updated_at"])
+    return connection, token
+
+
+@transaction.atomic
+def finish_mailbox_poll(
+    *,
+    connection_id: int,
+    worker_token: str,
+    success: bool,
+    error_category: str = "",
+    error_message: str = "",
+) -> None:
+    connection = MailboxConnection.objects.select_for_update().get(pk=connection_id)
+    if connection.lease_token != worker_token:
+        return
+    now = timezone.now()
+    connection.last_polled_at = now
+    connection.next_poll_at = now + timedelta(minutes=connection.poll_interval_minutes)
+    connection.lease_token = ""
+    connection.lease_expires_at = None
+    if success:
+        connection.last_success_at = now
+        connection.last_error_category = ""
+        connection.last_error_message = ""
+    else:
+        connection.last_error_category = error_category[:80]
+        connection.last_error_message = error_message[:500]
+    connection.save(
+        update_fields=[
+            "last_polled_at",
+            "next_poll_at",
+            "lease_token",
+            "lease_expires_at",
+            "last_success_at",
+            "last_error_category",
+            "last_error_message",
+            "updated_at",
+        ]
+    )

apps/sources/services/policy.py

@@ -19,6 +19,10 @@ DEFAULT_DENYLIST = {
     "stepstone.be",
     "jobat.be",
     "vdab.be",
+    "ictjob.be",
+    "careerjet.be",
+    "randstad.be",
+    "roberthalf.com",
 }
 
 

apps/sources/services/robots.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from collections.abc import Iterable
 from dataclasses import dataclass
 from datetime import datetime, timedelta
-from urllib.parse import urlsplit, urlunsplit
+from urllib.parse import urljoin, urlsplit, urlunsplit
 
 import httpx
 from django.conf import settings
@@ -11,6 +11,7 @@ from django.utils import timezone
 
 from apps.sources.models import Source, SourceRobotsCache
 
+from .tls import trusted_tls_context
 from .url_security import UnsafeUrlError, validate_public_url
 
 ALLOW = "allow"
@@ -123,20 +124,37 @@ def _max_bytes() -> int:
 
 def _fetch_robots(origin: str, *, client: httpx.Client | None = None) -> tuple[str, int, str, str]:
     robots_url = _robots_url(origin)
-    validate_public_url(
-        robots_url,
-        allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
-    )
     own_client = client is None
     http_client = client or httpx.Client(
         timeout=httpx.Timeout(getattr(settings, "ROBOTS_FETCH_TIMEOUT_SECONDS", 5)),
         follow_redirects=False,
+        verify=trusted_tls_context(),
     )
     try:
-        response = http_client.get(
-            robots_url,
-            headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
-        )
+        current_url = robots_url
+        for _ in range(getattr(settings, "FETCHER_MAX_REDIRECTS", 5) + 1):
+            validate_public_url(
+                current_url,
+                allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
+            )
+            response = http_client.get(
+                current_url,
+                headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
+            )
+            if response.status_code not in {301, 302, 303, 307, 308}:
+                break
+            location = response.headers.get("location")
+            if not location:
+                raise httpx.HTTPStatusError(
+                    "Robotsredirect zonder Location-header",
+                    request=response.request,
+                    response=response,
+                )
+            current_url = urljoin(current_url, location)
+        else:
+            raise httpx.TooManyRedirects(
+                "Te veel redirects bij robotscontrole", request=response.request
+            )
     finally:
         if own_client:
             http_client.close()

apps/sources/services/tls.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+import ssl
+from functools import lru_cache
+from pathlib import Path
+
+import certifi
+
+
+@lru_cache(maxsize=1)
+def trusted_tls_context() -> ssl.SSLContext:
+    """Build a strict TLS context from the pinned application CA bundle."""
+    bundle = Path(certifi.where()).resolve(strict=True)
+    if not bundle.is_file():
+        raise RuntimeError("De vertrouwde CA-bundel ontbreekt.")
+    return ssl.create_default_context(cafile=str(bundle))