132 lines
4.7 KiB
Python
132 lines
4.7 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from datetime import timedelta
|
|
from email import policy
|
|
from email.parser import BytesParser
|
|
from email.utils import parsedate_to_datetime
|
|
|
|
from django.conf import settings
|
|
from django.db import transaction
|
|
from django.utils import timezone
|
|
|
|
from apps.jobs.services.normalization import normalize_extracted_job
|
|
from apps.jobs.services.pipeline import persist_draft
|
|
from apps.sources.adapters.email_alert import EmailAlertAdapter
|
|
from apps.sources.models import EmailMessageRecord, MailboxConnection, RawDocument, Source
|
|
from apps.sources.platforms import platform_label
|
|
|
|
|
|
def message_identity(raw_message: bytes) -> str:
|
|
"""Return the RFC Message-ID or a deterministic hash when it is absent."""
|
|
parsed = BytesParser(policy=policy.default).parsebytes(raw_message, headersonly=True)
|
|
fallback_id = hashlib.sha256(raw_message).hexdigest()
|
|
return str(parsed.get("message-id") or f"sha256:{fallback_id}").strip()
|
|
|
|
|
|
@transaction.atomic
|
|
def ingest_email(
|
|
raw_message: bytes,
|
|
*,
|
|
mailbox: str = "INBOX",
|
|
mailbox_connection: MailboxConnection | None = None,
|
|
platform: str | None = None,
|
|
) -> EmailMessageRecord:
|
|
parsed = BytesParser(policy=policy.default).parsebytes(raw_message)
|
|
message_id = message_identity(raw_message)
|
|
existing = EmailMessageRecord.objects.filter(
|
|
message_id=message_id, mailbox_connection=mailbox_connection
|
|
).first()
|
|
if existing:
|
|
return existing
|
|
|
|
source_domain = f"{platform}.mailbox.local" if platform else "mailbox.local"
|
|
source_name = f"{platform_label(platform)} vacaturemailbox" if platform else "Vacaturemailbox"
|
|
source, _ = Source.objects.get_or_create(
|
|
domain=source_domain,
|
|
source_type=Source.Type.EMAIL,
|
|
defaults={
|
|
"name": source_name,
|
|
"status": Source.Status.ACTIVE,
|
|
"policy": Source.Policy.ALLOW,
|
|
"parser_key": "email-alert",
|
|
"crawl_interval_minutes": 1440,
|
|
},
|
|
)
|
|
content_hash = hashlib.sha256(raw_message).hexdigest()
|
|
retain_until = timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS)
|
|
document = RawDocument.objects.create(
|
|
source=source,
|
|
kind=RawDocument.Kind.EMAIL,
|
|
content_type="message/rfc822",
|
|
content_hash=content_hash,
|
|
body_text=raw_message.decode("utf-8", errors="replace"),
|
|
byte_length=len(raw_message),
|
|
retain_until=retain_until,
|
|
metadata={
|
|
"mailbox": mailbox,
|
|
"mailbox_connection_id": mailbox_connection.pk if mailbox_connection else None,
|
|
"alert_provider": platform or "unknown",
|
|
},
|
|
)
|
|
received_at = None
|
|
if parsed.get("date"):
|
|
try:
|
|
received_at = parsedate_to_datetime(str(parsed.get("date")))
|
|
if timezone.is_naive(received_at):
|
|
received_at = timezone.make_aware(received_at, timezone.get_current_timezone())
|
|
except (TypeError, ValueError, OverflowError):
|
|
received_at = None
|
|
|
|
adapter = EmailAlertAdapter()
|
|
result = adapter.extract_message(raw_message, expected_provider=platform)
|
|
detected_provider = next(
|
|
(
|
|
str(extracted.raw.get("alert_provider"))
|
|
for extracted in result.jobs
|
|
if extracted.raw.get("alert_provider")
|
|
),
|
|
"unknown",
|
|
)
|
|
alert_provider = platform or detected_provider
|
|
document.parser_key = result.parser_key
|
|
document.parser_version = result.parser_version
|
|
document.extraction_confidence = result.confidence
|
|
document.metadata = {**document.metadata, "alert_provider": alert_provider}
|
|
document.save(
|
|
update_fields=[
|
|
"parser_key",
|
|
"parser_version",
|
|
"extraction_confidence",
|
|
"metadata",
|
|
"updated_at",
|
|
]
|
|
)
|
|
links: list[str] = []
|
|
errors: list[str] = []
|
|
for extracted in result.jobs:
|
|
links.append(extracted.url)
|
|
try:
|
|
draft = normalize_extracted_job(extracted)
|
|
persist_draft(
|
|
draft,
|
|
document=document,
|
|
parser_key=result.parser_key,
|
|
parser_version=result.parser_version,
|
|
extraction_confidence=result.confidence,
|
|
)
|
|
except Exception as exc:
|
|
errors.append(f"{exc.__class__.__name__}: {exc}")
|
|
return EmailMessageRecord.objects.create(
|
|
mailbox_connection=mailbox_connection,
|
|
message_id=message_id,
|
|
mailbox=mailbox,
|
|
sender=str(parsed.get("from") or "")[:500],
|
|
subject=str(parsed.get("subject") or "")[:998],
|
|
received_at=received_at,
|
|
raw_document=document,
|
|
links=links,
|
|
processed=not errors,
|
|
error_message="; ".join(errors)[:1000],
|
|
)
|