101 lines
4.0 KiB
Python
101 lines
4.0 KiB
Python
from __future__ import annotations
|
|
|
|
import html
|
|
import re
|
|
from email import policy
|
|
from email.message import Message
|
|
from email.parser import BytesParser
|
|
from urllib.parse import urljoin, urlsplit
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from apps.sources.services.canonicalize import canonicalize_url
|
|
|
|
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
|
|
|
URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I)
|
|
SKIP_TEXT = re.compile(r"unsubscribe|afmelden|uitschrijven|privacy|view in browser", re.I)
|
|
|
|
|
|
class EmailAlertAdapter:
|
|
parser_key = "email-alert"
|
|
parser_version = "1.0.0"
|
|
|
|
@staticmethod
|
|
def _decode_parts(message: Message) -> tuple[str, str]:
|
|
plain_parts: list[str] = []
|
|
html_parts: list[str] = []
|
|
for part in message.walk() if message.is_multipart() else [message]:
|
|
disposition = str(part.get("Content-Disposition") or "")
|
|
if "attachment" in disposition.lower():
|
|
continue
|
|
content_type = part.get_content_type()
|
|
try:
|
|
payload = part.get_content()
|
|
except Exception:
|
|
raw = part.get_payload(decode=True) or b""
|
|
payload = raw.decode(part.get_content_charset() or "utf-8", errors="replace")
|
|
if content_type == "text/plain":
|
|
plain_parts.append(str(payload))
|
|
elif content_type == "text/html":
|
|
html_parts.append(str(payload))
|
|
return "\n".join(plain_parts), "\n".join(html_parts)
|
|
|
|
def extract_message(self, raw_message: bytes) -> ExtractionResult:
|
|
message = BytesParser(policy=policy.default).parsebytes(raw_message)
|
|
plain, html_body = self._decode_parts(message)
|
|
candidates: list[tuple[str, str]] = []
|
|
if html_body:
|
|
soup = BeautifulSoup(html_body, "lxml")
|
|
for anchor in soup.find_all("a", href=True):
|
|
label = " ".join(anchor.get_text(" ", strip=True).split())
|
|
href = html.unescape(str(anchor["href"]).strip())
|
|
if not href.lower().startswith(("http://", "https://")):
|
|
continue
|
|
if SKIP_TEXT.search(label) or SKIP_TEXT.search(href):
|
|
continue
|
|
candidates.append((label, href))
|
|
for href in URL_RE.findall(plain):
|
|
clean_href = href.rstrip(".,);]")
|
|
if SKIP_TEXT.search(clean_href):
|
|
continue
|
|
candidates.append(("", clean_href))
|
|
|
|
jobs: list[ExtractedJob] = []
|
|
seen: set[str] = set()
|
|
subject = str(message.get("subject") or "Vacature uit e-mail").strip()
|
|
for label, href in candidates:
|
|
canonical = canonicalize_url(urljoin("https://invalid.local/", href))
|
|
if not canonical or canonical in seen:
|
|
continue
|
|
seen.add(canonical)
|
|
hostname = urlsplit(canonical).hostname or ""
|
|
title = label if len(label) >= 4 else subject
|
|
if len(title) > 300:
|
|
title = title[:300]
|
|
jobs.append(
|
|
ExtractedJob(
|
|
url=canonical,
|
|
title=title,
|
|
employer_name="",
|
|
description_text=plain[:5000]
|
|
or BeautifulSoup(html_body, "lxml").get_text("\n", strip=True)[:5000],
|
|
raw={
|
|
"email_subject": subject,
|
|
"email_sender": str(message.get("from") or ""),
|
|
"target_domain": hostname,
|
|
},
|
|
evidence=[FieldEvidence("url", "email-anchor", 0.75, label[:240])],
|
|
)
|
|
)
|
|
return ExtractionResult(
|
|
jobs,
|
|
self.parser_key,
|
|
self.parser_version,
|
|
0.65 if jobs else 0.0,
|
|
[] if jobs else ["Geen vacaturelinks in e-mail gevonden"],
|
|
)
|
|
|
|
def extract(self, content: str, *, url: str = "") -> ExtractionResult:
|
|
return self.extract_message(content.encode("utf-8", errors="replace"))
|