157 lines
6.0 KiB
Python
157 lines
6.0 KiB
Python
from __future__ import annotations
|
|
|
|
import html
|
|
import re
|
|
from email import policy
|
|
from email.message import Message
|
|
from email.parser import BytesParser
|
|
from urllib.parse import urljoin, urlsplit
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from apps.sources.platforms import PLATFORM_ALERTS
|
|
from apps.sources.services.canonicalize import canonicalize_url
|
|
|
|
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
|
|
|
URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I)
|
|
SKIP_TEXT = re.compile(
|
|
r"unsubscribe|afmelden|uitschrijven|privacy|view in browser|bekijk online|"
|
|
r"account|aanmelden|inloggen|login|voorkeuren|preferences|voorwaarden|terms|"
|
|
r"contact|help|hulp|over ons|about us",
|
|
re.I,
|
|
)
|
|
SKIP_PATH = re.compile(
|
|
r"/(?:unsubscribe|uitschrijven|afmelden|privacy|account|login|signin|preferences|"
|
|
r"settings|terms|legal|help|contact)(?:/|$)",
|
|
re.I,
|
|
)
|
|
|
|
|
|
def _matches_domain(hostname: str, domains: tuple[str, ...]) -> bool:
|
|
return any(hostname == domain or hostname.endswith(f".{domain}") for domain in domains)
|
|
|
|
|
|
class EmailAlertAdapter:
|
|
parser_key = "email-alert"
|
|
parser_version = "1.1.0"
|
|
|
|
@staticmethod
|
|
def _provider_for_candidates(candidates: list[tuple[str, str]]) -> str:
|
|
hostnames = {
|
|
(urlsplit(href).hostname or "").lower()
|
|
for _, href in candidates
|
|
if href.lower().startswith(("http://", "https://"))
|
|
}
|
|
for provider, alert in PLATFORM_ALERTS.items():
|
|
if any(_matches_domain(hostname, alert.domains) for hostname in hostnames):
|
|
return provider
|
|
return "other"
|
|
|
|
@staticmethod
|
|
def _is_expected_platform_link(*, label: str, href: str, provider: str) -> bool:
|
|
alert = PLATFORM_ALERTS.get(provider)
|
|
if alert is None or len(label.strip()) < 4:
|
|
return False
|
|
parsed = urlsplit(href)
|
|
hostname = (parsed.hostname or "").lower()
|
|
return (
|
|
parsed.scheme.lower() == "https"
|
|
and _matches_domain(hostname, alert.domains)
|
|
and not SKIP_TEXT.search(label)
|
|
and not SKIP_PATH.search(parsed.path)
|
|
)
|
|
|
|
@staticmethod
|
|
def _decode_parts(message: Message) -> tuple[str, str]:
|
|
plain_parts: list[str] = []
|
|
html_parts: list[str] = []
|
|
for part in message.walk() if message.is_multipart() else [message]:
|
|
disposition = str(part.get("Content-Disposition") or "")
|
|
if "attachment" in disposition.lower():
|
|
continue
|
|
content_type = part.get_content_type()
|
|
try:
|
|
payload = part.get_content()
|
|
except Exception:
|
|
raw = part.get_payload(decode=True) or b""
|
|
payload = raw.decode(part.get_content_charset() or "utf-8", errors="replace")
|
|
if content_type == "text/plain":
|
|
plain_parts.append(str(payload))
|
|
elif content_type == "text/html":
|
|
html_parts.append(str(payload))
|
|
return "\n".join(plain_parts), "\n".join(html_parts)
|
|
|
|
def extract_message(
|
|
self, raw_message: bytes, *, expected_provider: str | None = None
|
|
) -> ExtractionResult:
|
|
message = BytesParser(policy=policy.default).parsebytes(raw_message)
|
|
plain, html_body = self._decode_parts(message)
|
|
candidates: list[tuple[str, str]] = []
|
|
if html_body:
|
|
soup = BeautifulSoup(html_body, "lxml")
|
|
for anchor in soup.find_all("a", href=True):
|
|
label = " ".join(anchor.get_text(" ", strip=True).split())
|
|
href = html.unescape(str(anchor["href"]).strip())
|
|
if not href.lower().startswith(("http://", "https://")):
|
|
continue
|
|
if SKIP_TEXT.search(label) or SKIP_TEXT.search(href):
|
|
continue
|
|
candidates.append((label, href))
|
|
for href in URL_RE.findall(plain):
|
|
clean_href = href.rstrip(".,);]")
|
|
if SKIP_TEXT.search(clean_href):
|
|
continue
|
|
candidates.append(("", clean_href))
|
|
|
|
alert_provider = expected_provider or self._provider_for_candidates(candidates)
|
|
if expected_provider:
|
|
candidates = [
|
|
(label, href)
|
|
for label, href in candidates
|
|
if self._is_expected_platform_link(
|
|
label=label,
|
|
href=href,
|
|
provider=expected_provider,
|
|
)
|
|
]
|
|
|
|
jobs: list[ExtractedJob] = []
|
|
seen: set[str] = set()
|
|
subject = str(message.get("subject") or "Vacature uit e-mail").strip()
|
|
for label, href in candidates:
|
|
canonical = canonicalize_url(urljoin("https://invalid.local/", href))
|
|
if not canonical or canonical in seen:
|
|
continue
|
|
seen.add(canonical)
|
|
hostname = urlsplit(canonical).hostname or ""
|
|
title = label if len(label) >= 4 else subject
|
|
if len(title) > 300:
|
|
title = title[:300]
|
|
jobs.append(
|
|
ExtractedJob(
|
|
url=canonical,
|
|
title=title,
|
|
employer_name="",
|
|
description_text=plain[:5000]
|
|
or BeautifulSoup(html_body, "lxml").get_text("\n", strip=True)[:5000],
|
|
raw={
|
|
"email_subject": subject,
|
|
"email_sender": str(message.get("from") or ""),
|
|
"target_domain": hostname,
|
|
"alert_provider": alert_provider,
|
|
},
|
|
evidence=[FieldEvidence("url", "email-anchor", 0.75, label[:240])],
|
|
)
|
|
)
|
|
return ExtractionResult(
|
|
jobs,
|
|
self.parser_key,
|
|
self.parser_version,
|
|
0.65 if jobs else 0.0,
|
|
[] if jobs else ["Geen vacaturelinks in e-mail gevonden"],
|
|
)
|
|
|
|
def extract(self, content: str, *, url: str = "") -> ExtractionResult:
|
|
return self.extract_message(content.encode("utf-8", errors="replace"))
|