Jens
@@ -0,0 +1,391 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
from xml.etree import ElementTree
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from django.db import transaction
|
||||
|
||||
from apps.sources.adapters.email_alert import EmailAlertAdapter
|
||||
from apps.sources.adapters.rss import RssAdapter
|
||||
from apps.sources.models import Source
|
||||
|
||||
from .canonicalize import canonicalize_url, domain_matches
|
||||
from .policy import is_denied_domain
|
||||
|
||||
CAREER_PATTERN = re.compile(
|
||||
r"\b(career|careers|jobs|job|vacature|vacatures|werken-bij|werken bij|emploi|emplois|offres)\b",
|
||||
re.I,
|
||||
)
|
||||
FEED_TYPE_PATTERN = re.compile(r"application/(?:atom|rss)\+xml|text/xml|application/xml", re.I)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceCandidate:
|
||||
url: str
|
||||
domain: str
|
||||
source_type: str
|
||||
label: str
|
||||
confidence: float
|
||||
reason: str
|
||||
discovered_from: str
|
||||
|
||||
|
||||
def discover_career_links(html: str, *, base_url: str) -> list[SourceCandidate]:
|
||||
return _discover_html(
|
||||
html,
|
||||
base_url=base_url,
|
||||
include_jsonld=False,
|
||||
include_feed_links=False,
|
||||
)
|
||||
|
||||
|
||||
def discover_from_html(html: str, *, base_url: str) -> list[SourceCandidate]:
|
||||
return _discover_html(
|
||||
html,
|
||||
base_url=base_url,
|
||||
include_jsonld=True,
|
||||
include_feed_links=True,
|
||||
)
|
||||
|
||||
|
||||
def discover_from_feed(feed_content: str, *, base_url: str) -> list[SourceCandidate]:
|
||||
adapter = RssAdapter()
|
||||
result = adapter.extract(feed_content, url=base_url)
|
||||
base_domain = _hostname(base_url)
|
||||
candidates: list[SourceCandidate] = []
|
||||
for extracted in result.jobs:
|
||||
candidate = _build_candidate(
|
||||
raw_url=extracted.url,
|
||||
base_domain=base_domain,
|
||||
source_type=Source.Type.RSS,
|
||||
label=extracted.title[:120],
|
||||
reason="rss",
|
||||
discovered_from="feed",
|
||||
confidence=0.82,
|
||||
allow_off_domain=False,
|
||||
)
|
||||
if candidate:
|
||||
candidates.append(candidate)
|
||||
return _dedupe(candidates)
|
||||
|
||||
|
||||
def discover_from_sitemap(xml_content: str, *, base_url: str) -> list[SourceCandidate]:
|
||||
try:
|
||||
root = ElementTree.fromstring(xml_content)
|
||||
except ElementTree.ParseError:
|
||||
return []
|
||||
|
||||
root_name = _local_tag(root.tag)
|
||||
if root_name == "sitemapindex":
|
||||
discovered_from = "sitemap-index"
|
||||
elif root_name == "urlset":
|
||||
discovered_from = "sitemap-urlset"
|
||||
else:
|
||||
return []
|
||||
|
||||
base_domain = _hostname(base_url)
|
||||
candidates: list[SourceCandidate] = []
|
||||
for location in root.findall(".//{*}loc"):
|
||||
raw = (location.text or "").strip()
|
||||
if not raw:
|
||||
continue
|
||||
candidate = _build_candidate(
|
||||
raw_url=urljoin(base_url, raw),
|
||||
base_domain=base_domain,
|
||||
source_type=Source.Type.SITEMAP,
|
||||
label="Sitemaplocatie",
|
||||
reason=discovered_from,
|
||||
discovered_from=discovered_from,
|
||||
confidence=0.86 if discovered_from == "sitemap-urlset" else 0.75,
|
||||
allow_off_domain=False,
|
||||
)
|
||||
if candidate:
|
||||
candidates.append(candidate)
|
||||
|
||||
return _dedupe(candidates)
|
||||
|
||||
|
||||
def discover_from_email(raw_message: bytes) -> list[SourceCandidate]:
|
||||
adapter = EmailAlertAdapter()
|
||||
result = adapter.extract_message(raw_message)
|
||||
candidates: list[SourceCandidate] = []
|
||||
for extracted in result.jobs:
|
||||
candidate_url = _to_domain_root_url(extracted.url)
|
||||
candidate = _build_candidate(
|
||||
raw_url=candidate_url,
|
||||
base_domain=_hostname(extracted.url),
|
||||
source_type=Source.Type.EMPLOYER,
|
||||
label=extracted.title[:120],
|
||||
reason="email",
|
||||
discovered_from="email",
|
||||
confidence=0.62,
|
||||
allow_off_domain=True,
|
||||
)
|
||||
if candidate:
|
||||
candidates.append(candidate)
|
||||
|
||||
return _dedupe(candidates)
|
||||
|
||||
|
||||
def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int, int, int]:
|
||||
created = updated = skipped = 0
|
||||
ordered = _dedupe(candidates)
|
||||
with transaction.atomic():
|
||||
for candidate in ordered:
|
||||
provenance = _provenance_entry(candidate)
|
||||
now = _utc_now()
|
||||
source, was_created = Source.objects.get_or_create(
|
||||
domain=candidate.domain,
|
||||
source_type=candidate.source_type,
|
||||
defaults={
|
||||
"name": _candidate_name(candidate),
|
||||
"base_url": candidate.url,
|
||||
"status": Source.Status.CANDIDATE,
|
||||
"policy": Source.Policy.REVIEW,
|
||||
"policy_reason": "Automatisch ontdekt",
|
||||
"parser_key": "auto",
|
||||
"strict_mode": True,
|
||||
"metadata": {
|
||||
"discovery": [provenance],
|
||||
"discovered_at": now,
|
||||
},
|
||||
},
|
||||
)
|
||||
if was_created:
|
||||
created += 1
|
||||
continue
|
||||
|
||||
updated_fields: list[str] = ["updated_at"]
|
||||
metadata = source.metadata if isinstance(source.metadata, dict) else {}
|
||||
evidence = metadata.get("discovery", [])
|
||||
if not isinstance(evidence, list):
|
||||
evidence = []
|
||||
if not any(item.get("url") == candidate.url for item in evidence if isinstance(item, dict)):
|
||||
evidence.append(provenance)
|
||||
metadata["discovery"] = evidence[-20:]
|
||||
metadata["discovered_at"] = now
|
||||
source.metadata = metadata
|
||||
updated_fields.append("metadata")
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
if not source.name:
|
||||
source.name = _candidate_name(candidate)
|
||||
updated_fields.append("name")
|
||||
if not source.base_url:
|
||||
source.base_url = candidate.url
|
||||
updated_fields.append("base_url")
|
||||
|
||||
if len(updated_fields) > 1:
|
||||
source.save(update_fields=sorted(set(updated_fields)))
|
||||
updated += 1
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
return (created, updated, skipped)
|
||||
|
||||
|
||||
def _candidate_name(candidate: SourceCandidate) -> str:
|
||||
label = candidate.label.strip()
|
||||
if label:
|
||||
return label
|
||||
return candidate.domain
|
||||
|
||||
|
||||
def _provenance_entry(candidate: SourceCandidate) -> dict[str, str | float]:
|
||||
return {
|
||||
"url": candidate.url,
|
||||
"source_type": candidate.source_type,
|
||||
"discovered_from": candidate.discovered_from,
|
||||
"confidence": candidate.confidence,
|
||||
"reason": candidate.reason,
|
||||
"label": candidate.label[:240],
|
||||
"seen_at": _utc_now(),
|
||||
}
|
||||
|
||||
|
||||
def _discover_html(
|
||||
html: str,
|
||||
*,
|
||||
base_url: str,
|
||||
include_jsonld: bool,
|
||||
include_feed_links: bool,
|
||||
) -> list[SourceCandidate]:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
base_domain = _hostname(base_url)
|
||||
candidates: list[SourceCandidate] = []
|
||||
|
||||
for anchor in soup.find_all("a", href=True):
|
||||
label = " ".join(anchor.get_text(" ", strip=True).split())
|
||||
candidate = _build_candidate(
|
||||
raw_url=urljoin(base_url, str(anchor["href"])),
|
||||
base_domain=base_domain,
|
||||
source_type=Source.Type.EMPLOYER,
|
||||
label=label,
|
||||
reason="career-link",
|
||||
discovered_from="html",
|
||||
confidence=0.9,
|
||||
allow_off_domain=False,
|
||||
)
|
||||
if not candidate:
|
||||
continue
|
||||
if not CAREER_PATTERN.search(label) and not CAREER_PATTERN.search(urlsplit(candidate.url).path):
|
||||
continue
|
||||
candidates.append(candidate)
|
||||
|
||||
if include_jsonld:
|
||||
for js in soup.find_all("script", type=re.compile(r"application/ld\+json", re.I)):
|
||||
raw = js.get_text("", strip=True)
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
for url in _collect_jsonld_urls(parsed):
|
||||
candidate = _build_candidate(
|
||||
raw_url=urljoin(base_url, url),
|
||||
base_domain=base_domain,
|
||||
source_type=Source.Type.EMPLOYER,
|
||||
label="JSON-LD job",
|
||||
reason="jsonld",
|
||||
discovered_from="html-jsonld",
|
||||
confidence=0.95,
|
||||
allow_off_domain=False,
|
||||
)
|
||||
if candidate:
|
||||
candidates.append(candidate)
|
||||
|
||||
if include_feed_links:
|
||||
for link in soup.find_all("link", href=True):
|
||||
rel = {str(item).lower() for item in (link.get("rel") or [])}
|
||||
if "alternate" not in rel:
|
||||
continue
|
||||
link_type = str(link.get("type") or "")
|
||||
if not FEED_TYPE_PATTERN.search(link_type):
|
||||
continue
|
||||
candidate = _build_candidate(
|
||||
raw_url=urljoin(base_url, str(link["href"])),
|
||||
base_domain=base_domain,
|
||||
source_type=Source.Type.RSS,
|
||||
label=(str(link.get("title") or "Feedlink")).strip()[:120],
|
||||
reason="feed",
|
||||
discovered_from="html",
|
||||
confidence=0.86,
|
||||
allow_off_domain=False,
|
||||
)
|
||||
if candidate:
|
||||
candidates.append(candidate)
|
||||
|
||||
return _dedupe(candidates)
|
||||
|
||||
|
||||
def _collect_jsonld_urls(payload) -> list[str]:
|
||||
urls: list[str] = []
|
||||
|
||||
def walk(node):
|
||||
if isinstance(node, dict):
|
||||
candidate_type = str(node.get("@type", "")).lower()
|
||||
if candidate_type == "jobposting":
|
||||
for key in ("url", "applyUrl", "application", "applicationurl"):
|
||||
value = node.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
urls.append(value)
|
||||
for key in ("@graph", "itemListElement", "item", "jobs", "jobPosting"):
|
||||
nested = node.get(key)
|
||||
if nested is not None:
|
||||
walk(nested)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
walk(payload)
|
||||
return urls
|
||||
|
||||
|
||||
def _build_candidate(
|
||||
*,
|
||||
raw_url: str,
|
||||
base_domain: str,
|
||||
source_type: str,
|
||||
label: str,
|
||||
reason: str,
|
||||
discovered_from: str,
|
||||
confidence: float,
|
||||
allow_off_domain: bool,
|
||||
) -> SourceCandidate | None:
|
||||
canonical = canonicalize_url(raw_url)
|
||||
if not canonical:
|
||||
return None
|
||||
if not _url_is_http(canonical):
|
||||
return None
|
||||
hostname = _hostname(canonical)
|
||||
if not hostname:
|
||||
return None
|
||||
if is_denied_domain(hostname):
|
||||
return None
|
||||
if _is_private_host(hostname):
|
||||
return None
|
||||
if (not allow_off_domain) and base_domain and not domain_matches(hostname, base_domain):
|
||||
return None
|
||||
return SourceCandidate(
|
||||
url=canonical,
|
||||
domain=hostname,
|
||||
source_type=source_type,
|
||||
label=label,
|
||||
confidence=confidence,
|
||||
reason=reason,
|
||||
discovered_from=discovered_from,
|
||||
)
|
||||
|
||||
|
||||
def _url_is_http(url: str) -> bool:
|
||||
return urlsplit(url).scheme.lower() in {"http", "https"}
|
||||
|
||||
|
||||
def _hostname(value: str) -> str:
|
||||
return (urlsplit(value).hostname or "").lower()
|
||||
|
||||
|
||||
def _is_private_host(hostname: str) -> bool:
|
||||
host = hostname.lower().rstrip(".")
|
||||
if host in {"localhost", "127.0.0.1", "::1"} or host.endswith((".local", ".localhost")):
|
||||
return True
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
return False
|
||||
return not ip.is_global
|
||||
|
||||
|
||||
def _dedupe(candidates: list[SourceCandidate]) -> list[SourceCandidate]:
|
||||
seen: set[tuple[str, str]] = set()
|
||||
result: list[SourceCandidate] = []
|
||||
for candidate in sorted(candidates, key=lambda item: item.confidence, reverse=True):
|
||||
key = (candidate.url, candidate.source_type)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(candidate)
|
||||
return result
|
||||
|
||||
|
||||
def _local_tag(tag_name: str) -> str:
|
||||
return tag_name.rsplit("}", 1)[-1].lower()
|
||||
|
||||
|
||||
def _to_domain_root_url(url: str) -> str:
|
||||
parsed = urlsplit(url)
|
||||
if not parsed.scheme or not parsed.hostname:
|
||||
return url
|
||||
return f"{parsed.scheme}://{parsed.hostname}"
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
Reference in New Issue
Block a user