feat: release regional radar and mailbox integrations

apps/sources/tasks.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-import imaplib
-from contextlib import suppress
 from datetime import timedelta
 from uuid import uuid4
 
@@ -11,8 +9,7 @@ from django.db.models import Q
 from django.utils import timezone
 
 from apps.jobs.services.pipeline import process_raw_document
-from apps.sources.models import EmailMessageRecord, RawDocument, Source, SourceRun
-from apps.sources.services.email_import import ingest_email, message_identity
+from apps.sources.models import MailboxConnection, RawDocument, Source, SourceRun
 from apps.sources.services.fetcher import (
     FetchError,
     FetchTimeoutError,
@@ -25,6 +22,12 @@ from apps.sources.services.health import (
     evaluate_source_health,
     start_health_canary,
 )
+from apps.sources.services.imap_import import poll_imap_mailbox
+from apps.sources.services.mailbox_connections import (
+    MailboxCredentialError,
+    claim_mailbox_connection,
+    finish_mailbox_poll,
+)
 from apps.sources.services.policy import assess_url
 from apps.sources.services.scheduling import (
     acquire_source_lease,
@@ -192,40 +195,63 @@ def schedule_due_sources() -> dict[str, int]:
     return {"scheduled": len(ids)}
 
 
-@shared_task(name="apps.sources.tasks.poll_imap_inbox")
-def poll_imap_inbox() -> dict[str, int | str]:
-    if not settings.IMAP_ENABLED:
-        return {"status": "disabled", "imported": 0}
-    if not settings.IMAP_HOST or not settings.IMAP_USER or not settings.IMAP_PASSWORD:
-        return {"status": "not_configured", "imported": 0}
-    client_cls = imaplib.IMAP4_SSL if settings.IMAP_USE_SSL else imaplib.IMAP4
-    client = client_cls(settings.IMAP_HOST, settings.IMAP_PORT)
-    imported = 0
+@shared_task(name="apps.sources.tasks.schedule_mailbox_polls")
+def schedule_mailbox_polls() -> dict[str, int]:
+    now = timezone.now()
+    due_ids = list(
+        MailboxConnection.objects.filter(enabled=True, next_poll_at__lte=now)
+        .filter(Q(lease_expires_at__isnull=True) | Q(lease_expires_at__lte=now))
+        .values_list("id", flat=True)[:100]
+    )
+    for connection_id in due_ids:
+        poll_mailbox.delay(connection_id)
+    return {"scheduled": len(due_ids)}
+
+
+@shared_task(bind=True, name="apps.sources.tasks.poll_mailbox")
+def poll_mailbox(self, connection_id: int, force: bool = False) -> dict[str, int | str]:
+    worker_token = str(self.request.id or uuid4())
+    claimed = claim_mailbox_connection(
+        connection_id=connection_id, worker_token=worker_token, force=force
+    )
+    if claimed is None:
+        return {"status": "skipped", "imported": 0}
+    connection, token = claimed
     try:
-        client.login(settings.IMAP_USER, settings.IMAP_PASSWORD)
-        status, _ = client.select(settings.IMAP_MAILBOX, readonly=not settings.IMAP_MARK_SEEN)
-        if status != "OK":
-            raise RuntimeError("IMAP-mailbox kon niet worden geopend")
-        status, data = client.uid("search", None, "ALL")
-        if status != "OK":
-            raise RuntimeError("IMAP-zoekopdracht mislukt")
-        uids = (data[0] or b"").split()[-200:]
-        for uid in uids:
-            status, payload = client.uid("fetch", uid, "(RFC822)")
-            if status != "OK" or not payload:
-                continue
-            raw = next((part[1] for part in payload if isinstance(part, tuple)), None)
-            if not raw:
-                continue
-            message_id = message_identity(raw)
-            was_known = EmailMessageRecord.objects.filter(message_id=message_id).exists()
-            ingest_email(raw, mailbox=settings.IMAP_MAILBOX)
-            if not was_known:
-                imported += 1
-        return {"status": "ok", "imported": imported}
-    finally:
-        with suppress(Exception):
-            client.logout()
+        result = poll_imap_mailbox(connection)
+        finish_mailbox_poll(
+            connection_id=connection.pk,
+            worker_token=token,
+            success=result["status"] == "ok",
+        )
+        return result
+    except MailboxCredentialError as exc:
+        finish_mailbox_poll(
+            connection_id=connection.pk,
+            worker_token=token,
+            success=False,
+            error_category="credentials",
+            error_message=str(exc),
+        )
+        return {"status": "failed", "error_category": "credentials", "imported": 0}
+    except Exception as exc:
+        error_category = "imap"
+        if exc.__class__.__name__.lower().startswith("unsafe"):
+            error_category = "host_policy"
+        finish_mailbox_poll(
+            connection_id=connection.pk,
+            worker_token=token,
+            success=False,
+            error_category=error_category,
+            error_message="IMAP-verbinding of authenticatie mislukt.",
+        )
+        return {"status": "failed", "error_category": error_category, "imported": 0}
+
+
+@shared_task(name="apps.sources.tasks.poll_imap_inbox")
+def poll_imap_inbox() -> dict[str, int]:
+    """Compatibility alias for old deployments; it now schedules due per-platform mailboxes."""
+    return schedule_mailbox_polls()
 
 
 @shared_task(name="apps.sources.tasks.cleanup_raw_documents")