from __future__ import annotations import html import re from email import policy from email.message import Message from email.parser import BytesParser from urllib.parse import urljoin, urlsplit from bs4 import BeautifulSoup from apps.sources.platforms import PLATFORM_ALERTS from apps.sources.services.canonicalize import canonicalize_url from .base import ExtractedJob, ExtractionResult, FieldEvidence URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I) SKIP_TEXT = re.compile( r"unsubscribe|afmelden|uitschrijven|privacy|view in browser|bekijk online|" r"account|aanmelden|inloggen|login|voorkeuren|preferences|voorwaarden|terms|" r"contact|help|hulp|over ons|about us", re.I, ) SKIP_PATH = re.compile( r"/(?:unsubscribe|uitschrijven|afmelden|privacy|account|login|signin|preferences|" r"settings|terms|legal|help|contact)(?:/|$)", re.I, ) def _matches_domain(hostname: str, domains: tuple[str, ...]) -> bool: return any(hostname == domain or hostname.endswith(f".{domain}") for domain in domains) class EmailAlertAdapter: parser_key = "email-alert" parser_version = "1.1.0" @staticmethod def _provider_for_candidates(candidates: list[tuple[str, str]]) -> str: hostnames = { (urlsplit(href).hostname or "").lower() for _, href in candidates if href.lower().startswith(("http://", "https://")) } for provider, alert in PLATFORM_ALERTS.items(): if any(_matches_domain(hostname, alert.domains) for hostname in hostnames): return provider return "other" @staticmethod def _is_expected_platform_link(*, label: str, href: str, provider: str) -> bool: alert = PLATFORM_ALERTS.get(provider) if alert is None or len(label.strip()) < 4: return False parsed = urlsplit(href) hostname = (parsed.hostname or "").lower() return ( parsed.scheme.lower() == "https" and _matches_domain(hostname, alert.domains) and not SKIP_TEXT.search(label) and not SKIP_PATH.search(parsed.path) ) @staticmethod def _decode_parts(message: Message) -> tuple[str, str]: plain_parts: list[str] = [] html_parts: list[str] = [] for part in message.walk() if message.is_multipart() else [message]: disposition = str(part.get("Content-Disposition") or "") if "attachment" in disposition.lower(): continue content_type = part.get_content_type() try: payload = part.get_content() except Exception: raw = part.get_payload(decode=True) or b"" payload = raw.decode(part.get_content_charset() or "utf-8", errors="replace") if content_type == "text/plain": plain_parts.append(str(payload)) elif content_type == "text/html": html_parts.append(str(payload)) return "\n".join(plain_parts), "\n".join(html_parts) def extract_message( self, raw_message: bytes, *, expected_provider: str | None = None ) -> ExtractionResult: message = BytesParser(policy=policy.default).parsebytes(raw_message) plain, html_body = self._decode_parts(message) candidates: list[tuple[str, str]] = [] if html_body: soup = BeautifulSoup(html_body, "lxml") for anchor in soup.find_all("a", href=True): label = " ".join(anchor.get_text(" ", strip=True).split()) href = html.unescape(str(anchor["href"]).strip()) if not href.lower().startswith(("http://", "https://")): continue if SKIP_TEXT.search(label) or SKIP_TEXT.search(href): continue candidates.append((label, href)) for href in URL_RE.findall(plain): clean_href = href.rstrip(".,);]") if SKIP_TEXT.search(clean_href): continue candidates.append(("", clean_href)) alert_provider = expected_provider or self._provider_for_candidates(candidates) if expected_provider: candidates = [ (label, href) for label, href in candidates if self._is_expected_platform_link( label=label, href=href, provider=expected_provider, ) ] jobs: list[ExtractedJob] = [] seen: set[str] = set() subject = str(message.get("subject") or "Vacature uit e-mail").strip() for label, href in candidates: canonical = canonicalize_url(urljoin("https://invalid.local/", href)) if not canonical or canonical in seen: continue seen.add(canonical) hostname = urlsplit(canonical).hostname or "" title = label if len(label) >= 4 else subject if len(title) > 300: title = title[:300] jobs.append( ExtractedJob( url=canonical, title=title, employer_name="", description_text=plain[:5000] or BeautifulSoup(html_body, "lxml").get_text("\n", strip=True)[:5000], raw={ "email_subject": subject, "email_sender": str(message.get("from") or ""), "target_domain": hostname, "alert_provider": alert_provider, }, evidence=[FieldEvidence("url", "email-anchor", 0.75, label[:240])], ) ) return ExtractionResult( jobs, self.parser_key, self.parser_version, 0.65 if jobs else 0.0, [] if jobs else ["Geen vacaturelinks in e-mail gevonden"], ) def extract(self, content: str, *, url: str = "") -> ExtractionResult: return self.extract_message(content.encode("utf-8", errors="replace"))