Initial deploy setup

2026-07-21 14:00:00 +02:00
commit b8091e59bd
285 changed files with 27854 additions and 0 deletions
@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import timedelta
+from email import policy
+from email.parser import BytesParser
+from email.utils import parsedate_to_datetime
+
+from django.conf import settings
+from django.db import transaction
+from django.utils import timezone
+
+from apps.jobs.services.normalization import normalize_extracted_job
+from apps.jobs.services.pipeline import persist_draft
+from apps.sources.adapters.email_alert import EmailAlertAdapter
+from apps.sources.models import EmailMessageRecord, RawDocument, Source
+
+
+def message_identity(raw_message: bytes) -> str:
+    """Return the RFC Message-ID or a deterministic hash when it is absent."""
+    parsed = BytesParser(policy=policy.default).parsebytes(raw_message, headersonly=True)
+    fallback_id = hashlib.sha256(raw_message).hexdigest()
+    return str(parsed.get("message-id") or f"sha256:{fallback_id}").strip()
+
+
+@transaction.atomic
+def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageRecord:
+    parsed = BytesParser(policy=policy.default).parsebytes(raw_message)
+    message_id = message_identity(raw_message)
+    existing = EmailMessageRecord.objects.filter(message_id=message_id).first()
+    if existing:
+        return existing
+
+    source, _ = Source.objects.get_or_create(
+        domain="mailbox.local",
+        source_type=Source.Type.EMAIL,
+        defaults={
+            "name": "Vacaturemailbox",
+            "status": Source.Status.ACTIVE,
+            "policy": Source.Policy.ALLOW,
+            "parser_key": "email-alert",
+            "crawl_interval_minutes": 10,
+        },
+    )
+    content_hash = hashlib.sha256(raw_message).hexdigest()
+    retain_until = timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS)
+    document = RawDocument.objects.create(
+        source=source,
+        kind=RawDocument.Kind.EMAIL,
+        content_type="message/rfc822",
+        content_hash=content_hash,
+        body_text=raw_message.decode("utf-8", errors="replace"),
+        byte_length=len(raw_message),
+        retain_until=retain_until,
+        metadata={"mailbox": mailbox},
+    )
+    received_at = None
+    if parsed.get("date"):
+        try:
+            received_at = parsedate_to_datetime(str(parsed.get("date")))
+            if timezone.is_naive(received_at):
+                received_at = timezone.make_aware(received_at, timezone.get_current_timezone())
+        except (TypeError, ValueError, OverflowError):
+            received_at = None
+
+    adapter = EmailAlertAdapter()
+    result = adapter.extract_message(raw_message)
+    document.parser_key = result.parser_key
+    document.parser_version = result.parser_version
+    document.extraction_confidence = result.confidence
+    document.save(
+        update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"]
+    )
+    links: list[str] = []
+    errors: list[str] = []
+    for extracted in result.jobs:
+        links.append(extracted.url)
+        try:
+            draft = normalize_extracted_job(extracted)
+            persist_draft(
+                draft,
+                document=document,
+                parser_key=result.parser_key,
+                parser_version=result.parser_version,
+                extraction_confidence=result.confidence,
+            )
+        except Exception as exc:
+            errors.append(f"{exc.__class__.__name__}: {exc}")
+    return EmailMessageRecord.objects.create(
+        message_id=message_id,
+        mailbox=mailbox,
+        sender=str(parsed.get("from") or "")[:500],
+        subject=str(parsed.get("subject") or "")[:998],
+        received_at=received_at,
+        raw_document=document,
+        links=links,
+        processed=not errors,
+        error_message="; ".join(errors)[:1000],
+    )