69 lines
2.7 KiB
Python
69 lines
2.7 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from datetime import timedelta
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
from django.conf import settings
|
|
from django.core.management.base import BaseCommand, CommandError
|
|
from django.utils import timezone
|
|
|
|
from apps.jobs.services.pipeline import process_raw_document
|
|
from apps.sources.models import RawDocument, Source, SourceRun
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "Importeert een lokale HTML/XML-fixture via dezelfde pipeline als een live bron."
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument("path")
|
|
parser.add_argument("--url", required=True)
|
|
parser.add_argument("--source-name", default="Lokale fixture")
|
|
parser.add_argument("--source-type", default=Source.Type.EMPLOYER)
|
|
|
|
def handle(self, *args, **options):
|
|
path = Path(options["path"]).resolve()
|
|
if not path.is_file():
|
|
raise CommandError(f"Bestand bestaat niet: {path}")
|
|
content = path.read_text(encoding="utf-8")
|
|
hostname = (urlsplit(options["url"]).hostname or "").lower()
|
|
if not hostname:
|
|
raise CommandError("--url bevat geen geldig domein")
|
|
source, _ = Source.objects.get_or_create(
|
|
base_url=options["url"],
|
|
source_type=options["source_type"],
|
|
defaults={
|
|
"domain": hostname,
|
|
"name": options["source_name"],
|
|
"status": Source.Status.ACTIVE,
|
|
"policy": Source.Policy.ALLOW,
|
|
"parser_key": "auto",
|
|
},
|
|
)
|
|
run = SourceRun.objects.create(source=source)
|
|
suffix = path.suffix.lower()
|
|
kind = RawDocument.Kind.XML if suffix in {".xml", ".rss"} else RawDocument.Kind.HTML
|
|
document = RawDocument.objects.create(
|
|
source=source,
|
|
source_run=run,
|
|
url=options["url"],
|
|
final_url=options["url"],
|
|
kind=kind,
|
|
content_type="application/xml" if kind == RawDocument.Kind.XML else "text/html",
|
|
content_hash=hashlib.sha256(content.encode()).hexdigest(),
|
|
body_text=content,
|
|
byte_length=len(content.encode()),
|
|
retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
|
|
)
|
|
metrics = process_raw_document(document)
|
|
run.finish(
|
|
SourceRun.Status.SUCCESS,
|
|
extracted_count=int(metrics["extracted"]),
|
|
created_count=int(metrics["created"]),
|
|
updated_count=int(metrics["updated"]),
|
|
duplicate_count=int(metrics["duplicates"]),
|
|
metrics=metrics,
|
|
)
|
|
self.stdout.write(self.style.SUCCESS(str(metrics)))
|