from __future__ import annotations import ipaddress import json import re from dataclasses import dataclass from datetime import datetime, timezone from urllib.parse import urljoin, urlsplit from xml.etree import ElementTree from bs4 import BeautifulSoup from django.db import transaction from apps.sources.adapters.email_alert import EmailAlertAdapter from apps.sources.adapters.rss import RssAdapter from apps.sources.models import Source from .canonicalize import canonicalize_url, domain_matches from .policy import is_denied_domain CAREER_PATTERN = re.compile( r"\b(career|careers|jobs|job|vacature|vacatures|werken-bij|werken bij|emploi|emplois|offres)\b", re.I, ) FEED_TYPE_PATTERN = re.compile(r"application/(?:atom|rss)\+xml|text/xml|application/xml", re.I) @dataclass(frozen=True) class SourceCandidate: url: str domain: str source_type: str label: str confidence: float reason: str discovered_from: str def discover_career_links(html: str, *, base_url: str) -> list[SourceCandidate]: return _discover_html( html, base_url=base_url, include_jsonld=False, include_feed_links=False, ) def discover_from_html(html: str, *, base_url: str) -> list[SourceCandidate]: return _discover_html( html, base_url=base_url, include_jsonld=True, include_feed_links=True, ) def discover_from_feed(feed_content: str, *, base_url: str) -> list[SourceCandidate]: adapter = RssAdapter() result = adapter.extract(feed_content, url=base_url) base_domain = _hostname(base_url) candidates: list[SourceCandidate] = [] for extracted in result.jobs: candidate = _build_candidate( raw_url=extracted.url, base_domain=base_domain, source_type=Source.Type.RSS, label=extracted.title[:120], reason="rss", discovered_from="feed", confidence=0.82, allow_off_domain=False, ) if candidate: candidates.append(candidate) return _dedupe(candidates) def discover_from_sitemap(xml_content: str, *, base_url: str) -> list[SourceCandidate]: try: root = ElementTree.fromstring(xml_content) except ElementTree.ParseError: return [] root_name = _local_tag(root.tag) if root_name == "sitemapindex": discovered_from = "sitemap-index" elif root_name == "urlset": discovered_from = "sitemap-urlset" else: return [] base_domain = _hostname(base_url) candidates: list[SourceCandidate] = [] for location in root.findall(".//{*}loc"): raw = (location.text or "").strip() if not raw: continue candidate = _build_candidate( raw_url=urljoin(base_url, raw), base_domain=base_domain, source_type=Source.Type.SITEMAP, label="Sitemaplocatie", reason=discovered_from, discovered_from=discovered_from, confidence=0.86 if discovered_from == "sitemap-urlset" else 0.75, allow_off_domain=False, ) if candidate: candidates.append(candidate) return _dedupe(candidates) def discover_from_email(raw_message: bytes) -> list[SourceCandidate]: adapter = EmailAlertAdapter() result = adapter.extract_message(raw_message) candidates: list[SourceCandidate] = [] for extracted in result.jobs: candidate_url = _to_domain_root_url(extracted.url) candidate = _build_candidate( raw_url=candidate_url, base_domain=_hostname(extracted.url), source_type=Source.Type.EMPLOYER, label=extracted.title[:120], reason="email", discovered_from="email", confidence=0.62, allow_off_domain=True, ) if candidate: candidates.append(candidate) return _dedupe(candidates) def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int, int, int]: created = updated = skipped = 0 ordered = _dedupe(candidates) with transaction.atomic(): for candidate in ordered: provenance = _provenance_entry(candidate) now = _utc_now() source, was_created = Source.objects.get_or_create( domain=candidate.domain, source_type=candidate.source_type, defaults={ "name": _candidate_name(candidate), "base_url": candidate.url, "status": Source.Status.CANDIDATE, "policy": Source.Policy.REVIEW, "policy_reason": "Automatisch ontdekt", "parser_key": "auto", "strict_mode": True, "metadata": { "discovery": [provenance], "discovered_at": now, }, }, ) if was_created: created += 1 continue updated_fields: list[str] = ["updated_at"] metadata = source.metadata if isinstance(source.metadata, dict) else {} evidence = metadata.get("discovery", []) if not isinstance(evidence, list): evidence = [] if not any(item.get("url") == candidate.url for item in evidence if isinstance(item, dict)): evidence.append(provenance) metadata["discovery"] = evidence[-20:] metadata["discovered_at"] = now source.metadata = metadata updated_fields.append("metadata") else: skipped += 1 if not source.name: source.name = _candidate_name(candidate) updated_fields.append("name") if not source.base_url: source.base_url = candidate.url updated_fields.append("base_url") if len(updated_fields) > 1: source.save(update_fields=sorted(set(updated_fields))) updated += 1 else: skipped += 1 return (created, updated, skipped) def _candidate_name(candidate: SourceCandidate) -> str: label = candidate.label.strip() if label: return label return candidate.domain def _provenance_entry(candidate: SourceCandidate) -> dict[str, str | float]: return { "url": candidate.url, "source_type": candidate.source_type, "discovered_from": candidate.discovered_from, "confidence": candidate.confidence, "reason": candidate.reason, "label": candidate.label[:240], "seen_at": _utc_now(), } def _discover_html( html: str, *, base_url: str, include_jsonld: bool, include_feed_links: bool, ) -> list[SourceCandidate]: soup = BeautifulSoup(html, "lxml") base_domain = _hostname(base_url) candidates: list[SourceCandidate] = [] for anchor in soup.find_all("a", href=True): label = " ".join(anchor.get_text(" ", strip=True).split()) candidate = _build_candidate( raw_url=urljoin(base_url, str(anchor["href"])), base_domain=base_domain, source_type=Source.Type.EMPLOYER, label=label, reason="career-link", discovered_from="html", confidence=0.9, allow_off_domain=False, ) if not candidate: continue if not CAREER_PATTERN.search(label) and not CAREER_PATTERN.search(urlsplit(candidate.url).path): continue candidates.append(candidate) if include_jsonld: for js in soup.find_all("script", type=re.compile(r"application/ld\+json", re.I)): raw = js.get_text("", strip=True) if not raw: continue try: parsed = json.loads(raw) except json.JSONDecodeError: continue for url in _collect_jsonld_urls(parsed): candidate = _build_candidate( raw_url=urljoin(base_url, url), base_domain=base_domain, source_type=Source.Type.EMPLOYER, label="JSON-LD job", reason="jsonld", discovered_from="html-jsonld", confidence=0.95, allow_off_domain=False, ) if candidate: candidates.append(candidate) if include_feed_links: for link in soup.find_all("link", href=True): rel = {str(item).lower() for item in (link.get("rel") or [])} if "alternate" not in rel: continue link_type = str(link.get("type") or "") if not FEED_TYPE_PATTERN.search(link_type): continue candidate = _build_candidate( raw_url=urljoin(base_url, str(link["href"])), base_domain=base_domain, source_type=Source.Type.RSS, label=(str(link.get("title") or "Feedlink")).strip()[:120], reason="feed", discovered_from="html", confidence=0.86, allow_off_domain=False, ) if candidate: candidates.append(candidate) return _dedupe(candidates) def _collect_jsonld_urls(payload) -> list[str]: urls: list[str] = [] def walk(node): if isinstance(node, dict): candidate_type = str(node.get("@type", "")).lower() if candidate_type == "jobposting": for key in ("url", "applyUrl", "application", "applicationurl"): value = node.get(key) if isinstance(value, str) and value: urls.append(value) for key in ("@graph", "itemListElement", "item", "jobs", "jobPosting"): nested = node.get(key) if nested is not None: walk(nested) elif isinstance(node, list): for item in node: walk(item) walk(payload) return urls def _build_candidate( *, raw_url: str, base_domain: str, source_type: str, label: str, reason: str, discovered_from: str, confidence: float, allow_off_domain: bool, ) -> SourceCandidate | None: canonical = canonicalize_url(raw_url) if not canonical: return None if not _url_is_http(canonical): return None hostname = _hostname(canonical) if not hostname: return None if is_denied_domain(hostname): return None if _is_private_host(hostname): return None if (not allow_off_domain) and base_domain and not domain_matches(hostname, base_domain): return None return SourceCandidate( url=canonical, domain=hostname, source_type=source_type, label=label, confidence=confidence, reason=reason, discovered_from=discovered_from, ) def _url_is_http(url: str) -> bool: return urlsplit(url).scheme.lower() in {"http", "https"} def _hostname(value: str) -> str: return (urlsplit(value).hostname or "").lower() def _is_private_host(hostname: str) -> bool: host = hostname.lower().rstrip(".") if host in {"localhost", "127.0.0.1", "::1"} or host.endswith((".local", ".localhost")): return True try: ip = ipaddress.ip_address(host) except ValueError: return False return not ip.is_global def _dedupe(candidates: list[SourceCandidate]) -> list[SourceCandidate]: seen: set[tuple[str, str]] = set() result: list[SourceCandidate] = [] for candidate in sorted(candidates, key=lambda item: item.confidence, reverse=True): key = (candidate.url, candidate.source_type) if key in seen: continue seen.add(key) result.append(candidate) return result def _local_tag(tag_name: str) -> str: return tag_name.rsplit("}", 1)[-1].lower() def _to_domain_root_url(url: str) -> str: parsed = urlsplit(url) if not parsed.scheme or not parsed.hostname: return url return f"{parsed.scheme}://{parsed.hostname}" def _utc_now() -> str: return datetime.now(timezone.utc).isoformat()