Jens
@@ -0,0 +1,355 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import html
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from urllib.parse import urlsplit
|
||||
from uuid import uuid4
|
||||
|
||||
import bleach
|
||||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
from django.db.models import QuerySet
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.jobs.models import JobSourceAlias
|
||||
from apps.jobs.services.pipeline import process_raw_document
|
||||
from apps.sources.models import RawDocument, Source, SourcePolicyReview, SourceRun
|
||||
from apps.sources.services.canonicalize import canonicalize_url
|
||||
from apps.sources.services.fetcher import (
|
||||
FetchError,
|
||||
FetchTimeoutError,
|
||||
FetchedDocument,
|
||||
PolicyBlockedError,
|
||||
RateLimitedError,
|
||||
fetch_url,
|
||||
)
|
||||
from apps.sources.services.policy import assess_url, create_policy_review
|
||||
|
||||
|
||||
class ManualImportError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ManualImportSummary:
|
||||
source_id: int
|
||||
source_name: str
|
||||
source_url: str
|
||||
mode: str
|
||||
extracted_count: int
|
||||
created_count: int
|
||||
duplicate_count: int
|
||||
warnings: list[str]
|
||||
jobs: list[dict[str, str]]
|
||||
|
||||
def to_session_payload(self) -> dict[str, object]:
|
||||
return {
|
||||
"source_id": self.source_id,
|
||||
"source_name": self.source_name,
|
||||
"source_url": self.source_url,
|
||||
"mode": self.mode,
|
||||
"extracted_count": self.extracted_count,
|
||||
"created_count": self.created_count,
|
||||
"duplicate_count": self.duplicate_count,
|
||||
"warnings": self.warnings,
|
||||
"jobs": self.jobs,
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_pasted_text(value: str) -> str:
|
||||
text = bleach.clean(value or "", tags=[], attributes={}, strip=True).strip()
|
||||
if not text:
|
||||
raise ManualImportError("Het geplakte tekstveld bevat geen bruikbare inhoud.")
|
||||
if len(text.encode("utf-8")) > settings.MANUAL_IMPORT_PASTE_MAX_BYTES:
|
||||
raise ManualImportError(
|
||||
"Het tekstveld is te groot voor veilige import. Verwijder overtollige tekst."
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def _kind_for(content_type: str) -> str:
|
||||
content_type = (content_type or "").lower()
|
||||
if "html" in content_type:
|
||||
return RawDocument.Kind.HTML
|
||||
if "xml" in content_type or "rss" in content_type or "atom" in content_type:
|
||||
return RawDocument.Kind.XML
|
||||
if "json" in content_type:
|
||||
return RawDocument.Kind.JSON
|
||||
return RawDocument.Kind.TEXT
|
||||
|
||||
|
||||
def _mode_source_name(domain: str, *, mode: str) -> str:
|
||||
if mode == "paste":
|
||||
return f"Handmatige tekstimport {domain}"
|
||||
return f"Handmatige URL-import {domain}"
|
||||
|
||||
|
||||
def _ensure_manual_review(source: Source, actor) -> None:
|
||||
review = source.latest_policy_review
|
||||
if (
|
||||
review
|
||||
and not review.is_expired
|
||||
and review.decision == SourcePolicyReview.Decision.ALLOW
|
||||
):
|
||||
return
|
||||
create_policy_review(
|
||||
source,
|
||||
actor=actor if actor and getattr(actor, "pk", None) else None,
|
||||
decision=SourcePolicyReview.Decision.ALLOW,
|
||||
reason="Handmatige import uitgevoerd.",
|
||||
scope=SourcePolicyReview.Scope.SOURCE,
|
||||
)
|
||||
|
||||
|
||||
def _ensure_manual_source(*, domain: str, source_url: str, actor, mode: str) -> Source:
|
||||
defaults = {
|
||||
"name": _mode_source_name(domain, mode=mode),
|
||||
"base_url": source_url,
|
||||
"status": Source.Status.CANDIDATE,
|
||||
"policy": Source.Policy.ALLOW,
|
||||
}
|
||||
source, created = Source.objects.get_or_create(
|
||||
domain=domain,
|
||||
source_type=Source.Type.MANUAL,
|
||||
defaults=defaults,
|
||||
)
|
||||
if not created:
|
||||
source.name = _mode_source_name(domain, mode=mode)
|
||||
source.base_url = source_url
|
||||
source.status = Source.Status.CANDIDATE
|
||||
source.policy = Source.Policy.ALLOW
|
||||
source.save(
|
||||
update_fields=["name", "base_url", "status", "policy", "updated_at"]
|
||||
)
|
||||
_ensure_manual_review(source, actor=actor)
|
||||
return source
|
||||
|
||||
|
||||
def _create_raw_document(
|
||||
source: Source,
|
||||
*,
|
||||
source_run: SourceRun,
|
||||
requested_url: str,
|
||||
final_url: str,
|
||||
content_type: str,
|
||||
content: bytes,
|
||||
) -> RawDocument:
|
||||
return RawDocument.objects.create(
|
||||
source=source,
|
||||
source_run=source_run,
|
||||
url=requested_url,
|
||||
final_url=final_url,
|
||||
kind=_kind_for(content_type),
|
||||
content_type=content_type[:200],
|
||||
http_status=None,
|
||||
response_headers={},
|
||||
content_hash=hashlib.sha256(content).hexdigest(),
|
||||
body_text=content.decode("utf-8", errors="replace"),
|
||||
byte_length=len(content),
|
||||
retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
|
||||
)
|
||||
|
||||
|
||||
def _build_jobs_from_document(document: RawDocument) -> list[dict[str, str]]:
|
||||
alias_qs: QuerySet[JobSourceAlias] = JobSourceAlias.objects.select_related("job").filter(
|
||||
raw_document=document
|
||||
)
|
||||
jobs: list[dict[str, str]] = []
|
||||
for alias in alias_qs:
|
||||
title = alias.source_title or (alias.job.original_title if alias.job else "Vacature")
|
||||
employer = alias.source_employer or "Onbekende werkgever"
|
||||
jobs.append(
|
||||
{
|
||||
"id": str(alias.job_id),
|
||||
"title": title,
|
||||
"employer": employer,
|
||||
}
|
||||
)
|
||||
return jobs
|
||||
|
||||
|
||||
def _run_pipeline(document: RawDocument, *, source_run: SourceRun) -> ManualImportSummary:
|
||||
metrics = process_raw_document(document)
|
||||
source = document.source
|
||||
source_name = source.name if source else ""
|
||||
warnings: list[str] = list(metrics.get("warnings", []))
|
||||
warnings_count = len(warnings)
|
||||
source_run.finish(
|
||||
SourceRun.Status.SUCCESS,
|
||||
http_status=document.source_run.http_status if document.source_run else None,
|
||||
extracted_count=int(metrics["extracted"]),
|
||||
created_count=int(metrics["created"]),
|
||||
updated_count=int(metrics["updated"]),
|
||||
duplicate_count=int(metrics["duplicates"]),
|
||||
metrics={"parser": metrics["parser"], "warnings": warnings},
|
||||
)
|
||||
jobs = _build_jobs_from_document(document)
|
||||
if metrics["created"] == 0 and metrics["updated"] == 0:
|
||||
if not warnings:
|
||||
warnings.append("De bron leverde geen herkenbare vacaturedata op.")
|
||||
if not warnings:
|
||||
# keep stable, machine-readable payload shape
|
||||
warnings = []
|
||||
return ManualImportSummary(
|
||||
source_id=source.pk,
|
||||
source_name=source_name,
|
||||
source_url=document.final_url,
|
||||
mode="url",
|
||||
extracted_count=int(metrics["extracted"]),
|
||||
created_count=int(metrics["created"]),
|
||||
duplicate_count=int(metrics["duplicates"]),
|
||||
warnings=warnings,
|
||||
jobs=jobs,
|
||||
)
|
||||
|
||||
|
||||
def _build_synthetic_paste_payload(raw_text: str) -> tuple[str, bytes, str]:
|
||||
lines = [html.escape(line.strip()) for line in raw_text.splitlines() if line.strip()]
|
||||
title = lines[0] if lines else "Handmatige vacature"
|
||||
body = "".join(f"<p>{line}</p>" for line in lines)
|
||||
source_domain = f"manual-{uuid4().hex[:16]}"
|
||||
synthetic_url = f"https://{source_domain}.vacature.local/"
|
||||
html_content = (
|
||||
f"<html><body><main><h1>{title}</h1>{body}</main>"
|
||||
"<footer>Handmatige import; geen externe fetch</footer></body></html>"
|
||||
)
|
||||
return synthetic_url, html_content.encode("utf-8"), source_domain
|
||||
|
||||
|
||||
def import_manual_source(
|
||||
*, actor, source_url: str | None = None, pasted_text: str | None = None
|
||||
) -> ManualImportSummary:
|
||||
source_url = (source_url or "").strip()
|
||||
pasted_text = (pasted_text or "").strip()
|
||||
if not source_url and not pasted_text:
|
||||
raise ManualImportError("Vul een URL of tekst in.")
|
||||
|
||||
with transaction.atomic():
|
||||
if source_url:
|
||||
normalized = canonicalize_url(source_url)
|
||||
parsed = urlsplit(normalized)
|
||||
domain = (parsed.hostname or "").lower()
|
||||
if not domain:
|
||||
raise ManualImportError("De bron-URL bevat geen geldig domein.")
|
||||
decision = assess_url(normalized)
|
||||
if not decision.allowed:
|
||||
raise ManualImportError(f"Import geblokkeerd: {decision.reason}")
|
||||
source = _ensure_manual_source(
|
||||
domain=domain,
|
||||
source_url=normalized,
|
||||
actor=actor,
|
||||
mode="url",
|
||||
)
|
||||
source_run = SourceRun.objects.create(source=source)
|
||||
try:
|
||||
fetched: FetchedDocument = fetch_url(normalized, source=source)
|
||||
except PolicyBlockedError as exc:
|
||||
source.status = Source.Status.QUARANTINED
|
||||
source.policy = Source.Policy.DENY
|
||||
source.policy_reason = str(exc)[:1000]
|
||||
source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
|
||||
source_run.finish(
|
||||
SourceRun.Status.SKIPPED,
|
||||
error_category="policy",
|
||||
error_message=str(exc)[:1000],
|
||||
)
|
||||
raise ManualImportError(f"Import geblokkeerd: {exc}") from exc
|
||||
except RateLimitedError as exc:
|
||||
source_run.finish(
|
||||
SourceRun.Status.FAILED,
|
||||
error_category="rate_limited",
|
||||
error_message=str(exc)[:1000],
|
||||
)
|
||||
raise ManualImportError(f"Rate limiting tijdens import: {exc}") from exc
|
||||
except FetchTimeoutError as exc:
|
||||
source_run.finish(
|
||||
SourceRun.Status.FAILED,
|
||||
error_category="timeout",
|
||||
error_message=str(exc)[:1000],
|
||||
)
|
||||
raise ManualImportError(f"Time-out tijdens import: {exc}") from exc
|
||||
except FetchError as exc:
|
||||
source_run.finish(
|
||||
SourceRun.Status.FAILED,
|
||||
error_category="fetch",
|
||||
error_message=str(exc)[:1000],
|
||||
)
|
||||
raise ManualImportError(f"Fetch mislukt: {exc}") from exc
|
||||
|
||||
if fetched.status_code == 304:
|
||||
source_run.finish(
|
||||
SourceRun.Status.SUCCESS,
|
||||
http_status=304,
|
||||
extracted_count=0,
|
||||
created_count=0,
|
||||
updated_count=0,
|
||||
duplicate_count=0,
|
||||
metrics={"parser": "not-modified", "warnings": []},
|
||||
)
|
||||
return ManualImportSummary(
|
||||
source_id=source.pk,
|
||||
source_name=source.name,
|
||||
source_url=source.base_url,
|
||||
mode="url",
|
||||
extracted_count=0,
|
||||
created_count=0,
|
||||
duplicate_count=0,
|
||||
warnings=["Geen inhoudsverandering (304)."],
|
||||
jobs=[],
|
||||
)
|
||||
|
||||
document = _create_raw_document(
|
||||
source,
|
||||
source_run=source_run,
|
||||
requested_url=fetched.requested_url,
|
||||
final_url=fetched.final_url,
|
||||
content_type=fetched.headers.get("content-type", ""),
|
||||
content=fetched.content,
|
||||
)
|
||||
source.etag = fetched.headers.get("etag", source.etag)
|
||||
source.last_modified = fetched.headers.get("last-modified", source.last_modified)
|
||||
source.save(update_fields=["etag", "last_modified", "updated_at"])
|
||||
summary = _run_pipeline(document, source_run=source_run)
|
||||
return ManualImportSummary(
|
||||
source_id=summary.source_id,
|
||||
source_name=summary.source_name,
|
||||
source_url=summary.source_url,
|
||||
mode="url",
|
||||
extracted_count=summary.extracted_count,
|
||||
created_count=summary.created_count,
|
||||
duplicate_count=summary.duplicate_count,
|
||||
warnings=summary.warnings,
|
||||
jobs=summary.jobs,
|
||||
)
|
||||
|
||||
text = _sanitize_pasted_text(pasted_text)
|
||||
synthetic_url, payload, synthetic_domain = _build_synthetic_paste_payload(text)
|
||||
source = _ensure_manual_source(
|
||||
domain=synthetic_domain,
|
||||
source_url=synthetic_url,
|
||||
actor=actor,
|
||||
mode="paste",
|
||||
)
|
||||
source_run = SourceRun.objects.create(source=source)
|
||||
document = _create_raw_document(
|
||||
source,
|
||||
source_run=source_run,
|
||||
requested_url=synthetic_url,
|
||||
final_url=synthetic_url,
|
||||
content_type="text/html; charset=utf-8",
|
||||
content=payload,
|
||||
)
|
||||
summary = _run_pipeline(document, source_run=source_run)
|
||||
return ManualImportSummary(
|
||||
source_id=summary.source_id,
|
||||
source_name=source.name,
|
||||
source_url=synthetic_url,
|
||||
mode="paste",
|
||||
extracted_count=summary.extracted_count,
|
||||
created_count=summary.created_count,
|
||||
duplicate_count=summary.duplicate_count,
|
||||
warnings=summary.warnings,
|
||||
jobs=summary.jobs,
|
||||
)
|
||||
Reference in New Issue
Block a user