from __future__ import annotations import hashlib from datetime import timedelta from email import policy from email.parser import BytesParser from email.utils import parsedate_to_datetime from django.conf import settings from django.db import transaction from django.utils import timezone from apps.jobs.services.normalization import normalize_extracted_job from apps.jobs.services.pipeline import persist_draft from apps.sources.adapters.email_alert import EmailAlertAdapter from apps.sources.models import EmailMessageRecord, MailboxConnection, RawDocument, Source from apps.sources.platforms import platform_label def message_identity(raw_message: bytes) -> str: """Return the RFC Message-ID or a deterministic hash when it is absent.""" parsed = BytesParser(policy=policy.default).parsebytes(raw_message, headersonly=True) fallback_id = hashlib.sha256(raw_message).hexdigest() return str(parsed.get("message-id") or f"sha256:{fallback_id}").strip() @transaction.atomic def ingest_email( raw_message: bytes, *, mailbox: str = "INBOX", mailbox_connection: MailboxConnection | None = None, platform: str | None = None, ) -> EmailMessageRecord: parsed = BytesParser(policy=policy.default).parsebytes(raw_message) message_id = message_identity(raw_message) existing = EmailMessageRecord.objects.filter( message_id=message_id, mailbox_connection=mailbox_connection ).first() if existing: return existing source_domain = f"{platform}.mailbox.local" if platform else "mailbox.local" source_name = f"{platform_label(platform)} vacaturemailbox" if platform else "Vacaturemailbox" source, _ = Source.objects.get_or_create( domain=source_domain, source_type=Source.Type.EMAIL, defaults={ "name": source_name, "status": Source.Status.ACTIVE, "policy": Source.Policy.ALLOW, "parser_key": "email-alert", "crawl_interval_minutes": 1440, }, ) content_hash = hashlib.sha256(raw_message).hexdigest() retain_until = timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS) document = RawDocument.objects.create( source=source, kind=RawDocument.Kind.EMAIL, content_type="message/rfc822", content_hash=content_hash, body_text=raw_message.decode("utf-8", errors="replace"), byte_length=len(raw_message), retain_until=retain_until, metadata={ "mailbox": mailbox, "mailbox_connection_id": mailbox_connection.pk if mailbox_connection else None, "alert_provider": platform or "unknown", }, ) received_at = None if parsed.get("date"): try: received_at = parsedate_to_datetime(str(parsed.get("date"))) if timezone.is_naive(received_at): received_at = timezone.make_aware(received_at, timezone.get_current_timezone()) except (TypeError, ValueError, OverflowError): received_at = None adapter = EmailAlertAdapter() result = adapter.extract_message(raw_message, expected_provider=platform) detected_provider = next( ( str(extracted.raw.get("alert_provider")) for extracted in result.jobs if extracted.raw.get("alert_provider") ), "unknown", ) alert_provider = platform or detected_provider document.parser_key = result.parser_key document.parser_version = result.parser_version document.extraction_confidence = result.confidence document.metadata = {**document.metadata, "alert_provider": alert_provider} document.save( update_fields=[ "parser_key", "parser_version", "extraction_confidence", "metadata", "updated_at", ] ) links: list[str] = [] errors: list[str] = [] for extracted in result.jobs: links.append(extracted.url) try: draft = normalize_extracted_job(extracted) persist_draft( draft, document=document, parser_key=result.parser_key, parser_version=result.parser_version, extraction_confidence=result.confidence, ) except Exception as exc: errors.append(f"{exc.__class__.__name__}: {exc}") return EmailMessageRecord.objects.create( mailbox_connection=mailbox_connection, message_id=message_id, mailbox=mailbox, sender=str(parsed.get("from") or "")[:500], subject=str(parsed.get("subject") or "")[:998], received_at=received_at, raw_document=document, links=links, processed=not errors, error_message="; ".join(errors)[:1000], )