from __future__ import annotations import hashlib from datetime import timedelta from email import policy from email.parser import BytesParser from email.utils import parsedate_to_datetime from django.conf import settings from django.db import transaction from django.utils import timezone from apps.jobs.services.normalization import normalize_extracted_job from apps.jobs.services.pipeline import persist_draft from apps.sources.adapters.email_alert import EmailAlertAdapter from apps.sources.models import EmailMessageRecord, RawDocument, Source def message_identity(raw_message: bytes) -> str: """Return the RFC Message-ID or a deterministic hash when it is absent.""" parsed = BytesParser(policy=policy.default).parsebytes(raw_message, headersonly=True) fallback_id = hashlib.sha256(raw_message).hexdigest() return str(parsed.get("message-id") or f"sha256:{fallback_id}").strip() @transaction.atomic def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageRecord: parsed = BytesParser(policy=policy.default).parsebytes(raw_message) message_id = message_identity(raw_message) existing = EmailMessageRecord.objects.filter(message_id=message_id).first() if existing: return existing source, _ = Source.objects.get_or_create( domain="mailbox.local", source_type=Source.Type.EMAIL, defaults={ "name": "Vacaturemailbox", "status": Source.Status.ACTIVE, "policy": Source.Policy.ALLOW, "parser_key": "email-alert", "crawl_interval_minutes": 10, }, ) content_hash = hashlib.sha256(raw_message).hexdigest() retain_until = timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS) document = RawDocument.objects.create( source=source, kind=RawDocument.Kind.EMAIL, content_type="message/rfc822", content_hash=content_hash, body_text=raw_message.decode("utf-8", errors="replace"), byte_length=len(raw_message), retain_until=retain_until, metadata={"mailbox": mailbox}, ) received_at = None if parsed.get("date"): try: received_at = parsedate_to_datetime(str(parsed.get("date"))) if timezone.is_naive(received_at): received_at = timezone.make_aware(received_at, timezone.get_current_timezone()) except (TypeError, ValueError, OverflowError): received_at = None adapter = EmailAlertAdapter() result = adapter.extract_message(raw_message) document.parser_key = result.parser_key document.parser_version = result.parser_version document.extraction_confidence = result.confidence document.save( update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"] ) links: list[str] = [] errors: list[str] = [] for extracted in result.jobs: links.append(extracted.url) try: draft = normalize_extracted_job(extracted) persist_draft( draft, document=document, parser_key=result.parser_key, parser_version=result.parser_version, extraction_confidence=result.confidence, ) except Exception as exc: errors.append(f"{exc.__class__.__name__}: {exc}") return EmailMessageRecord.objects.create( message_id=message_id, mailbox=mailbox, sender=str(parsed.get("from") or "")[:500], subject=str(parsed.get("subject") or "")[:998], received_at=received_at, raw_document=document, links=links, processed=not errors, error_message="; ".join(errors)[:1000], )