from __future__ import annotations import hashlib from datetime import timedelta from pathlib import Path from urllib.parse import urlsplit from django.conf import settings from django.core.management.base import BaseCommand, CommandError from django.utils import timezone from apps.jobs.services.pipeline import process_raw_document from apps.sources.models import RawDocument, Source, SourceRun class Command(BaseCommand): help = "Importeert een lokale HTML/XML-fixture via dezelfde pipeline als een live bron." def add_arguments(self, parser): parser.add_argument("path") parser.add_argument("--url", required=True) parser.add_argument("--source-name", default="Lokale fixture") parser.add_argument("--source-type", default=Source.Type.EMPLOYER) def handle(self, *args, **options): path = Path(options["path"]).resolve() if not path.is_file(): raise CommandError(f"Bestand bestaat niet: {path}") content = path.read_text(encoding="utf-8") hostname = (urlsplit(options["url"]).hostname or "").lower() if not hostname: raise CommandError("--url bevat geen geldig domein") source, _ = Source.objects.get_or_create( domain=hostname, source_type=options["source_type"], defaults={ "name": options["source_name"], "base_url": options["url"], "status": Source.Status.ACTIVE, "policy": Source.Policy.ALLOW, "parser_key": "auto", }, ) run = SourceRun.objects.create(source=source) suffix = path.suffix.lower() kind = RawDocument.Kind.XML if suffix in {".xml", ".rss"} else RawDocument.Kind.HTML document = RawDocument.objects.create( source=source, source_run=run, url=options["url"], final_url=options["url"], kind=kind, content_type="application/xml" if kind == RawDocument.Kind.XML else "text/html", content_hash=hashlib.sha256(content.encode()).hexdigest(), body_text=content, byte_length=len(content.encode()), retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS), ) metrics = process_raw_document(document) run.finish( SourceRun.Status.SUCCESS, extracted_count=int(metrics["extracted"]), created_count=int(metrics["created"]), updated_count=int(metrics["updated"]), duplicate_count=int(metrics["duplicates"]), metrics=metrics, ) self.stdout.write(self.style.SUCCESS(str(metrics)))