VacatureRadar/apps/sources/services/discovery.py

from __future__ import annotations

import ipaddress
import json
import re
from dataclasses import dataclass
from datetime import datetime, timezone
from urllib.parse import urljoin, urlsplit
from xml.etree import ElementTree

from bs4 import BeautifulSoup
from django.db import transaction

from apps.sources.adapters.email_alert import EmailAlertAdapter
from apps.sources.adapters.rss import RssAdapter
from apps.sources.models import Source

from .canonicalize import canonicalize_url, domain_matches
from .policy import is_denied_domain

CAREER_PATTERN = re.compile(
    r"\b(career|careers|jobs|job|vacature|vacatures|werken-bij|werken bij|emploi|emplois|offres)\b",
    re.I,
)
FEED_TYPE_PATTERN = re.compile(r"application/(?:atom|rss)\+xml|text/xml|application/xml", re.I)


@dataclass(frozen=True)
class SourceCandidate:
    url: str
    domain: str
    source_type: str
    label: str
    confidence: float
    reason: str
    discovered_from: str


def discover_career_links(html: str, *, base_url: str) -> list[SourceCandidate]:
    return _discover_html(
        html,
        base_url=base_url,
        include_jsonld=False,
        include_feed_links=False,
    )


def discover_from_html(html: str, *, base_url: str) -> list[SourceCandidate]:
    return _discover_html(
        html,
        base_url=base_url,
        include_jsonld=True,
        include_feed_links=True,
    )


def discover_from_feed(feed_content: str, *, base_url: str) -> list[SourceCandidate]:
    adapter = RssAdapter()
    result = adapter.extract(feed_content, url=base_url)
    base_domain = _hostname(base_url)
    candidates: list[SourceCandidate] = []
    for extracted in result.jobs:
        candidate = _build_candidate(
            raw_url=extracted.url,
            base_domain=base_domain,
            source_type=Source.Type.RSS,
            label=extracted.title[:120],
            reason="rss",
            discovered_from="feed",
            confidence=0.82,
            allow_off_domain=False,
        )
        if candidate:
            candidates.append(candidate)
    return _dedupe(candidates)


def discover_from_sitemap(xml_content: str, *, base_url: str) -> list[SourceCandidate]:
    try:
        root = ElementTree.fromstring(xml_content)
    except ElementTree.ParseError:
        return []

    root_name = _local_tag(root.tag)
    if root_name == "sitemapindex":
        discovered_from = "sitemap-index"
    elif root_name == "urlset":
        discovered_from = "sitemap-urlset"
    else:
        return []

    base_domain = _hostname(base_url)
    candidates: list[SourceCandidate] = []
    for location in root.findall(".//{*}loc"):
        raw = (location.text or "").strip()
        if not raw:
            continue
        candidate = _build_candidate(
            raw_url=urljoin(base_url, raw),
            base_domain=base_domain,
            source_type=Source.Type.SITEMAP,
            label="Sitemaplocatie",
            reason=discovered_from,
            discovered_from=discovered_from,
            confidence=0.86 if discovered_from == "sitemap-urlset" else 0.75,
            allow_off_domain=False,
        )
        if candidate:
            candidates.append(candidate)

    return _dedupe(candidates)


def discover_from_email(raw_message: bytes) -> list[SourceCandidate]:
    adapter = EmailAlertAdapter()
    result = adapter.extract_message(raw_message)
    candidates: list[SourceCandidate] = []
    for extracted in result.jobs:
        candidate_url = _to_domain_root_url(extracted.url)
        candidate = _build_candidate(
            raw_url=candidate_url,
            base_domain=_hostname(extracted.url),
            source_type=Source.Type.EMPLOYER,
            label=extracted.title[:120],
            reason="email",
            discovered_from="email",
            confidence=0.62,
            allow_off_domain=True,
        )
        if candidate:
            candidates.append(candidate)

    return _dedupe(candidates)


def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int, int, int]:
    created = updated = skipped = 0
    ordered = _dedupe(candidates)
    with transaction.atomic():
        for candidate in ordered:
            provenance = _provenance_entry(candidate)
            now = _utc_now()
            source, was_created = Source.objects.get_or_create(
                domain=candidate.domain,
                source_type=candidate.source_type,
                defaults={
                    "name": _candidate_name(candidate),
                    "base_url": candidate.url,
                    "status": Source.Status.CANDIDATE,
                    "policy": Source.Policy.REVIEW,
                    "policy_reason": "Automatisch ontdekt",
                    "parser_key": "auto",
                    "strict_mode": True,
                    "metadata": {
                        "discovery": [provenance],
                        "discovered_at": now,
                    },
                },
            )
            if was_created:
                created += 1
                continue

            updated_fields: list[str] = ["updated_at"]
            metadata = source.metadata if isinstance(source.metadata, dict) else {}
            evidence = metadata.get("discovery", [])
            if not isinstance(evidence, list):
                evidence = []
            if not any(item.get("url") == candidate.url for item in evidence if isinstance(item, dict)):
                evidence.append(provenance)
                metadata["discovery"] = evidence[-20:]
                metadata["discovered_at"] = now
                source.metadata = metadata
                updated_fields.append("metadata")
            else:
                skipped += 1

            if not source.name:
                source.name = _candidate_name(candidate)
                updated_fields.append("name")
            if not source.base_url:
                source.base_url = candidate.url
                updated_fields.append("base_url")

            if len(updated_fields) > 1:
                source.save(update_fields=sorted(set(updated_fields)))
                updated += 1
            else:
                skipped += 1

    return (created, updated, skipped)


def _candidate_name(candidate: SourceCandidate) -> str:
    label = candidate.label.strip()
    if label:
        return label
    return candidate.domain


def _provenance_entry(candidate: SourceCandidate) -> dict[str, str | float]:
    return {
        "url": candidate.url,
        "source_type": candidate.source_type,
        "discovered_from": candidate.discovered_from,
        "confidence": candidate.confidence,
        "reason": candidate.reason,
        "label": candidate.label[:240],
        "seen_at": _utc_now(),
    }


def _discover_html(
    html: str,
    *,
    base_url: str,
    include_jsonld: bool,
    include_feed_links: bool,
) -> list[SourceCandidate]:
    soup = BeautifulSoup(html, "lxml")
    base_domain = _hostname(base_url)
    candidates: list[SourceCandidate] = []

    for anchor in soup.find_all("a", href=True):
        label = " ".join(anchor.get_text(" ", strip=True).split())
        candidate = _build_candidate(
            raw_url=urljoin(base_url, str(anchor["href"])),
            base_domain=base_domain,
            source_type=Source.Type.EMPLOYER,
            label=label,
            reason="career-link",
            discovered_from="html",
            confidence=0.9,
            allow_off_domain=False,
        )
        if not candidate:
            continue
        if not CAREER_PATTERN.search(label) and not CAREER_PATTERN.search(urlsplit(candidate.url).path):
            continue
        candidates.append(candidate)

    if include_jsonld:
        for js in soup.find_all("script", type=re.compile(r"application/ld\+json", re.I)):
            raw = js.get_text("", strip=True)
            if not raw:
                continue
            try:
                parsed = json.loads(raw)
            except json.JSONDecodeError:
                continue
            for url in _collect_jsonld_urls(parsed):
                candidate = _build_candidate(
                    raw_url=urljoin(base_url, url),
                    base_domain=base_domain,
                    source_type=Source.Type.EMPLOYER,
                    label="JSON-LD job",
                    reason="jsonld",
                    discovered_from="html-jsonld",
                    confidence=0.95,
                    allow_off_domain=False,
                )
                if candidate:
                    candidates.append(candidate)

    if include_feed_links:
        for link in soup.find_all("link", href=True):
            rel = {str(item).lower() for item in (link.get("rel") or [])}
            if "alternate" not in rel:
                continue
            link_type = str(link.get("type") or "")
            if not FEED_TYPE_PATTERN.search(link_type):
                continue
            candidate = _build_candidate(
                raw_url=urljoin(base_url, str(link["href"])),
                base_domain=base_domain,
                source_type=Source.Type.RSS,
                label=(str(link.get("title") or "Feedlink")).strip()[:120],
                reason="feed",
                discovered_from="html",
                confidence=0.86,
                allow_off_domain=False,
            )
            if candidate:
                candidates.append(candidate)

    return _dedupe(candidates)


def _collect_jsonld_urls(payload) -> list[str]:
    urls: list[str] = []

    def walk(node):
        if isinstance(node, dict):
            candidate_type = str(node.get("@type", "")).lower()
            if candidate_type == "jobposting":
                for key in ("url", "applyUrl", "application", "applicationurl"):
                    value = node.get(key)
                    if isinstance(value, str) and value:
                        urls.append(value)
            for key in ("@graph", "itemListElement", "item", "jobs", "jobPosting"):
                nested = node.get(key)
                if nested is not None:
                    walk(nested)
        elif isinstance(node, list):
            for item in node:
                walk(item)

    walk(payload)
    return urls


def _build_candidate(
    *,
    raw_url: str,
    base_domain: str,
    source_type: str,
    label: str,
    reason: str,
    discovered_from: str,
    confidence: float,
    allow_off_domain: bool,
) -> SourceCandidate | None:
    canonical = canonicalize_url(raw_url)
    if not canonical:
        return None
    if not _url_is_http(canonical):
        return None
    hostname = _hostname(canonical)
    if not hostname:
        return None
    if is_denied_domain(hostname):
        return None
    if _is_private_host(hostname):
        return None
    if (not allow_off_domain) and base_domain and not domain_matches(hostname, base_domain):
        return None
    return SourceCandidate(
        url=canonical,
        domain=hostname,
        source_type=source_type,
        label=label,
        confidence=confidence,
        reason=reason,
        discovered_from=discovered_from,
    )


def _url_is_http(url: str) -> bool:
    return urlsplit(url).scheme.lower() in {"http", "https"}


def _hostname(value: str) -> str:
    return (urlsplit(value).hostname or "").lower()


def _is_private_host(hostname: str) -> bool:
    host = hostname.lower().rstrip(".")
    if host in {"localhost", "127.0.0.1", "::1"} or host.endswith((".local", ".localhost")):
        return True
    try:
        ip = ipaddress.ip_address(host)
    except ValueError:
        return False
    return not ip.is_global


def _dedupe(candidates: list[SourceCandidate]) -> list[SourceCandidate]:
    seen: set[tuple[str, str]] = set()
    result: list[SourceCandidate] = []
    for candidate in sorted(candidates, key=lambda item: item.confidence, reverse=True):
        key = (candidate.url, candidate.source_type)
        if key in seen:
            continue
        seen.add(key)
        result.append(candidate)
    return result


def _local_tag(tag_name: str) -> str:
    return tag_name.rsplit("}", 1)[-1].lower()


def _to_domain_root_url(url: str) -> str:
    parsed = urlsplit(url)
    if not parsed.scheme or not parsed.hostname:
        return url
    return f"{parsed.scheme}://{parsed.hostname}"


def _utc_now() -> str:
    return datetime.now(timezone.utc).isoformat()