349 lines
13 KiB
Python
349 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import html
|
|
from dataclasses import dataclass
|
|
from datetime import timedelta
|
|
from urllib.parse import urlsplit
|
|
from uuid import uuid4
|
|
|
|
import bleach
|
|
from django.conf import settings
|
|
from django.db import transaction
|
|
from django.db.models import QuerySet
|
|
from django.utils import timezone
|
|
|
|
from apps.jobs.models import JobSourceAlias
|
|
from apps.jobs.services.pipeline import process_raw_document
|
|
from apps.sources.models import RawDocument, Source, SourcePolicyReview, SourceRun
|
|
from apps.sources.services.canonicalize import canonicalize_url
|
|
from apps.sources.services.fetcher import (
|
|
FetchedDocument,
|
|
FetchError,
|
|
FetchTimeoutError,
|
|
PolicyBlockedError,
|
|
RateLimitedError,
|
|
fetch_url,
|
|
)
|
|
from apps.sources.services.policy import assess_url, create_policy_review
|
|
|
|
|
|
class ManualImportError(RuntimeError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ManualImportSummary:
|
|
source_id: int
|
|
source_name: str
|
|
source_url: str
|
|
mode: str
|
|
extracted_count: int
|
|
created_count: int
|
|
duplicate_count: int
|
|
warnings: list[str]
|
|
jobs: list[dict[str, str]]
|
|
|
|
def to_session_payload(self) -> dict[str, object]:
|
|
return {
|
|
"source_id": self.source_id,
|
|
"source_name": self.source_name,
|
|
"source_url": self.source_url,
|
|
"mode": self.mode,
|
|
"extracted_count": self.extracted_count,
|
|
"created_count": self.created_count,
|
|
"duplicate_count": self.duplicate_count,
|
|
"warnings": self.warnings,
|
|
"jobs": self.jobs,
|
|
}
|
|
|
|
|
|
def _sanitize_pasted_text(value: str) -> str:
|
|
text = bleach.clean(value or "", tags=[], attributes={}, strip=True).strip()
|
|
if not text:
|
|
raise ManualImportError("Het geplakte tekstveld bevat geen bruikbare inhoud.")
|
|
if len(text.encode("utf-8")) > settings.MANUAL_IMPORT_PASTE_MAX_BYTES:
|
|
raise ManualImportError(
|
|
"Het tekstveld is te groot voor veilige import. Verwijder overtollige tekst."
|
|
)
|
|
return text
|
|
|
|
|
|
def _kind_for(content_type: str) -> str:
|
|
content_type = (content_type or "").lower()
|
|
if "html" in content_type:
|
|
return RawDocument.Kind.HTML
|
|
if "xml" in content_type or "rss" in content_type or "atom" in content_type:
|
|
return RawDocument.Kind.XML
|
|
if "json" in content_type:
|
|
return RawDocument.Kind.JSON
|
|
return RawDocument.Kind.TEXT
|
|
|
|
|
|
def _mode_source_name(domain: str, *, mode: str) -> str:
|
|
if mode == "paste":
|
|
return f"Handmatige tekstimport {domain}"
|
|
return f"Handmatige URL-import {domain}"
|
|
|
|
|
|
def _ensure_manual_review(source: Source, actor) -> None:
|
|
review = source.latest_policy_review
|
|
if review and not review.is_expired and review.decision == SourcePolicyReview.Decision.ALLOW:
|
|
return
|
|
create_policy_review(
|
|
source,
|
|
actor=actor if actor and getattr(actor, "pk", None) else None,
|
|
decision=SourcePolicyReview.Decision.ALLOW,
|
|
reason="Handmatige import uitgevoerd.",
|
|
scope=SourcePolicyReview.Scope.SOURCE,
|
|
)
|
|
|
|
|
|
def _ensure_manual_source(*, domain: str, source_url: str, actor, mode: str) -> Source:
|
|
defaults = {
|
|
"name": _mode_source_name(domain, mode=mode),
|
|
"base_url": source_url,
|
|
"status": Source.Status.CANDIDATE,
|
|
"policy": Source.Policy.ALLOW,
|
|
}
|
|
source, created = Source.objects.get_or_create(
|
|
domain=domain,
|
|
source_type=Source.Type.MANUAL,
|
|
defaults=defaults,
|
|
)
|
|
if not created:
|
|
source.name = _mode_source_name(domain, mode=mode)
|
|
source.base_url = source_url
|
|
source.status = Source.Status.CANDIDATE
|
|
source.policy = Source.Policy.ALLOW
|
|
source.save(update_fields=["name", "base_url", "status", "policy", "updated_at"])
|
|
_ensure_manual_review(source, actor=actor)
|
|
return source
|
|
|
|
|
|
def _create_raw_document(
|
|
source: Source,
|
|
*,
|
|
source_run: SourceRun,
|
|
requested_url: str,
|
|
final_url: str,
|
|
content_type: str,
|
|
content: bytes,
|
|
) -> RawDocument:
|
|
return RawDocument.objects.create(
|
|
source=source,
|
|
source_run=source_run,
|
|
url=requested_url,
|
|
final_url=final_url,
|
|
kind=_kind_for(content_type),
|
|
content_type=content_type[:200],
|
|
http_status=None,
|
|
response_headers={},
|
|
content_hash=hashlib.sha256(content).hexdigest(),
|
|
body_text=content.decode("utf-8", errors="replace"),
|
|
byte_length=len(content),
|
|
retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
|
|
)
|
|
|
|
|
|
def _build_jobs_from_document(document: RawDocument) -> list[dict[str, str]]:
|
|
alias_qs: QuerySet[JobSourceAlias] = JobSourceAlias.objects.select_related("job").filter(
|
|
raw_document=document
|
|
)
|
|
jobs: list[dict[str, str]] = []
|
|
for alias in alias_qs:
|
|
title = alias.source_title or (alias.job.original_title if alias.job else "Vacature")
|
|
employer = alias.source_employer or "Onbekende werkgever"
|
|
jobs.append(
|
|
{
|
|
"id": str(alias.job_id),
|
|
"title": title,
|
|
"employer": employer,
|
|
}
|
|
)
|
|
return jobs
|
|
|
|
|
|
def _run_pipeline(document: RawDocument, *, source_run: SourceRun) -> ManualImportSummary:
|
|
metrics = process_raw_document(document)
|
|
source = document.source
|
|
source_name = source.name if source else ""
|
|
warnings: list[str] = list(metrics.get("warnings", []))
|
|
len(warnings)
|
|
source_run.finish(
|
|
SourceRun.Status.SUCCESS,
|
|
http_status=document.source_run.http_status if document.source_run else None,
|
|
extracted_count=int(metrics["extracted"]),
|
|
created_count=int(metrics["created"]),
|
|
updated_count=int(metrics["updated"]),
|
|
duplicate_count=int(metrics["duplicates"]),
|
|
metrics={"parser": metrics["parser"], "warnings": warnings},
|
|
)
|
|
jobs = _build_jobs_from_document(document)
|
|
if metrics["created"] == 0 and metrics["updated"] == 0 and not warnings:
|
|
warnings.append("De bron leverde geen herkenbare vacaturedata op.")
|
|
if not warnings:
|
|
# keep stable, machine-readable payload shape
|
|
warnings = []
|
|
return ManualImportSummary(
|
|
source_id=source.pk,
|
|
source_name=source_name,
|
|
source_url=document.final_url,
|
|
mode="url",
|
|
extracted_count=int(metrics["extracted"]),
|
|
created_count=int(metrics["created"]),
|
|
duplicate_count=int(metrics["duplicates"]),
|
|
warnings=warnings,
|
|
jobs=jobs,
|
|
)
|
|
|
|
|
|
def _build_synthetic_paste_payload(raw_text: str) -> tuple[str, bytes, str]:
|
|
lines = [html.escape(line.strip()) for line in raw_text.splitlines() if line.strip()]
|
|
title = lines[0] if lines else "Handmatige vacature"
|
|
body = "".join(f"<p>{line}</p>" for line in lines)
|
|
source_domain = f"manual-{uuid4().hex[:16]}"
|
|
synthetic_url = f"https://{source_domain}.vacature.local/"
|
|
html_content = (
|
|
f"<html><body><main><h1>{title}</h1>{body}</main>"
|
|
"<footer>Handmatige import; geen externe fetch</footer></body></html>"
|
|
)
|
|
return synthetic_url, html_content.encode("utf-8"), source_domain
|
|
|
|
|
|
def import_manual_source(
|
|
*, actor, source_url: str | None = None, pasted_text: str | None = None
|
|
) -> ManualImportSummary:
|
|
source_url = (source_url or "").strip()
|
|
pasted_text = (pasted_text or "").strip()
|
|
if not source_url and not pasted_text:
|
|
raise ManualImportError("Vul een URL of tekst in.")
|
|
|
|
with transaction.atomic():
|
|
if source_url:
|
|
normalized = canonicalize_url(source_url)
|
|
parsed = urlsplit(normalized)
|
|
domain = (parsed.hostname or "").lower()
|
|
if not domain:
|
|
raise ManualImportError("De bron-URL bevat geen geldig domein.")
|
|
decision = assess_url(normalized)
|
|
if not decision.allowed:
|
|
raise ManualImportError(f"Import geblokkeerd: {decision.reason}")
|
|
source = _ensure_manual_source(
|
|
domain=domain,
|
|
source_url=normalized,
|
|
actor=actor,
|
|
mode="url",
|
|
)
|
|
source_run = SourceRun.objects.create(source=source)
|
|
try:
|
|
fetched: FetchedDocument = fetch_url(normalized, source=source)
|
|
except PolicyBlockedError as exc:
|
|
source.status = Source.Status.QUARANTINED
|
|
source.policy = Source.Policy.DENY
|
|
source.policy_reason = str(exc)[:1000]
|
|
source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
|
|
source_run.finish(
|
|
SourceRun.Status.SKIPPED,
|
|
error_category="policy",
|
|
error_message=str(exc)[:1000],
|
|
)
|
|
raise ManualImportError(f"Import geblokkeerd: {exc}") from exc
|
|
except RateLimitedError as exc:
|
|
source_run.finish(
|
|
SourceRun.Status.FAILED,
|
|
error_category="rate_limited",
|
|
error_message=str(exc)[:1000],
|
|
)
|
|
raise ManualImportError(f"Rate limiting tijdens import: {exc}") from exc
|
|
except FetchTimeoutError as exc:
|
|
source_run.finish(
|
|
SourceRun.Status.FAILED,
|
|
error_category="timeout",
|
|
error_message=str(exc)[:1000],
|
|
)
|
|
raise ManualImportError(f"Time-out tijdens import: {exc}") from exc
|
|
except FetchError as exc:
|
|
source_run.finish(
|
|
SourceRun.Status.FAILED,
|
|
error_category="fetch",
|
|
error_message=str(exc)[:1000],
|
|
)
|
|
raise ManualImportError(f"Fetch mislukt: {exc}") from exc
|
|
|
|
if fetched.status_code == 304:
|
|
source_run.finish(
|
|
SourceRun.Status.SUCCESS,
|
|
http_status=304,
|
|
extracted_count=0,
|
|
created_count=0,
|
|
updated_count=0,
|
|
duplicate_count=0,
|
|
metrics={"parser": "not-modified", "warnings": []},
|
|
)
|
|
return ManualImportSummary(
|
|
source_id=source.pk,
|
|
source_name=source.name,
|
|
source_url=source.base_url,
|
|
mode="url",
|
|
extracted_count=0,
|
|
created_count=0,
|
|
duplicate_count=0,
|
|
warnings=["Geen inhoudsverandering (304)."],
|
|
jobs=[],
|
|
)
|
|
|
|
document = _create_raw_document(
|
|
source,
|
|
source_run=source_run,
|
|
requested_url=fetched.requested_url,
|
|
final_url=fetched.final_url,
|
|
content_type=fetched.headers.get("content-type", ""),
|
|
content=fetched.content,
|
|
)
|
|
source.etag = fetched.headers.get("etag", source.etag)
|
|
source.last_modified = fetched.headers.get("last-modified", source.last_modified)
|
|
source.save(update_fields=["etag", "last_modified", "updated_at"])
|
|
summary = _run_pipeline(document, source_run=source_run)
|
|
return ManualImportSummary(
|
|
source_id=summary.source_id,
|
|
source_name=summary.source_name,
|
|
source_url=summary.source_url,
|
|
mode="url",
|
|
extracted_count=summary.extracted_count,
|
|
created_count=summary.created_count,
|
|
duplicate_count=summary.duplicate_count,
|
|
warnings=summary.warnings,
|
|
jobs=summary.jobs,
|
|
)
|
|
|
|
text = _sanitize_pasted_text(pasted_text)
|
|
synthetic_url, payload, synthetic_domain = _build_synthetic_paste_payload(text)
|
|
source = _ensure_manual_source(
|
|
domain=synthetic_domain,
|
|
source_url=synthetic_url,
|
|
actor=actor,
|
|
mode="paste",
|
|
)
|
|
source_run = SourceRun.objects.create(source=source)
|
|
document = _create_raw_document(
|
|
source,
|
|
source_run=source_run,
|
|
requested_url=synthetic_url,
|
|
final_url=synthetic_url,
|
|
content_type="text/html; charset=utf-8",
|
|
content=payload,
|
|
)
|
|
summary = _run_pipeline(document, source_run=source_run)
|
|
return ManualImportSummary(
|
|
source_id=summary.source_id,
|
|
source_name=source.name,
|
|
source_url=synthetic_url,
|
|
mode="paste",
|
|
extracted_count=summary.extracted_count,
|
|
created_count=summary.created_count,
|
|
duplicate_count=summary.duplicate_count,
|
|
warnings=summary.warnings,
|
|
jobs=summary.jobs,
|
|
)
|