100 lines
3.7 KiB
Python
100 lines
3.7 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from datetime import timedelta
|
|
from email import policy
|
|
from email.parser import BytesParser
|
|
from email.utils import parsedate_to_datetime
|
|
|
|
from django.conf import settings
|
|
from django.db import transaction
|
|
from django.utils import timezone
|
|
|
|
from apps.jobs.services.normalization import normalize_extracted_job
|
|
from apps.jobs.services.pipeline import persist_draft
|
|
from apps.sources.adapters.email_alert import EmailAlertAdapter
|
|
from apps.sources.models import EmailMessageRecord, RawDocument, Source
|
|
|
|
|
|
def message_identity(raw_message: bytes) -> str:
|
|
"""Return the RFC Message-ID or a deterministic hash when it is absent."""
|
|
parsed = BytesParser(policy=policy.default).parsebytes(raw_message, headersonly=True)
|
|
fallback_id = hashlib.sha256(raw_message).hexdigest()
|
|
return str(parsed.get("message-id") or f"sha256:{fallback_id}").strip()
|
|
|
|
|
|
@transaction.atomic
|
|
def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageRecord:
|
|
parsed = BytesParser(policy=policy.default).parsebytes(raw_message)
|
|
message_id = message_identity(raw_message)
|
|
existing = EmailMessageRecord.objects.filter(message_id=message_id).first()
|
|
if existing:
|
|
return existing
|
|
|
|
source, _ = Source.objects.get_or_create(
|
|
domain="mailbox.local",
|
|
source_type=Source.Type.EMAIL,
|
|
defaults={
|
|
"name": "Vacaturemailbox",
|
|
"status": Source.Status.ACTIVE,
|
|
"policy": Source.Policy.ALLOW,
|
|
"parser_key": "email-alert",
|
|
"crawl_interval_minutes": 10,
|
|
},
|
|
)
|
|
content_hash = hashlib.sha256(raw_message).hexdigest()
|
|
retain_until = timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS)
|
|
document = RawDocument.objects.create(
|
|
source=source,
|
|
kind=RawDocument.Kind.EMAIL,
|
|
content_type="message/rfc822",
|
|
content_hash=content_hash,
|
|
body_text=raw_message.decode("utf-8", errors="replace"),
|
|
byte_length=len(raw_message),
|
|
retain_until=retain_until,
|
|
metadata={"mailbox": mailbox},
|
|
)
|
|
received_at = None
|
|
if parsed.get("date"):
|
|
try:
|
|
received_at = parsedate_to_datetime(str(parsed.get("date")))
|
|
if timezone.is_naive(received_at):
|
|
received_at = timezone.make_aware(received_at, timezone.get_current_timezone())
|
|
except (TypeError, ValueError, OverflowError):
|
|
received_at = None
|
|
|
|
adapter = EmailAlertAdapter()
|
|
result = adapter.extract_message(raw_message)
|
|
document.parser_key = result.parser_key
|
|
document.parser_version = result.parser_version
|
|
document.extraction_confidence = result.confidence
|
|
document.save(
|
|
update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"]
|
|
)
|
|
links: list[str] = []
|
|
errors: list[str] = []
|
|
for extracted in result.jobs:
|
|
links.append(extracted.url)
|
|
try:
|
|
draft = normalize_extracted_job(extracted)
|
|
persist_draft(
|
|
draft,
|
|
document=document,
|
|
parser_key=result.parser_key,
|
|
parser_version=result.parser_version,
|
|
extraction_confidence=result.confidence,
|
|
)
|
|
except Exception as exc:
|
|
errors.append(f"{exc.__class__.__name__}: {exc}")
|
|
return EmailMessageRecord.objects.create(
|
|
message_id=message_id,
|
|
mailbox=mailbox,
|
|
sender=str(parsed.get("from") or "")[:500],
|
|
subject=str(parsed.get("subject") or "")[:998],
|
|
received_at=received_at,
|
|
raw_document=document,
|
|
links=links,
|
|
processed=not errors,
|
|
error_message="; ".join(errors)[:1000],
|
|
)
|