Initial deploy setup

apps/sources/management/commands/discover_sources.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from django.core.management.base import BaseCommand, CommandError
+
+from apps.sources.services.discovery import (
+    discover_from_email,
+    discover_from_feed,
+    discover_from_html,
+    discover_from_sitemap,
+    persist_discovery_candidates,
+)
+
+
+class Command(BaseCommand):
+    help = (
+        "Voert bronontdekking uit op lokaal aangeleverde content "
+        "en slaat alleen kandidaatregels op met policy review."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "mode",
+            choices=["html", "sitemap", "feed", "email"],
+            help="Bronmodus voor ontdekking",
+        )
+        parser.add_argument(
+            "--input",
+            required=True,
+            dest="input_path",
+            help="Pad naar inputbestand",
+        )
+        parser.add_argument(
+            "--base-url",
+            dest="base_url",
+            help="Base URL voor relativiteitsresolutie",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Geen database-write; toon alleen ontdekte kandidaten",
+        )
+
+    def handle(self, *args, **options):
+        mode = options["mode"]
+        input_path = Path(options["input_path"]).resolve()
+        base_url = (options["base_url"] or "").strip()
+        if mode in {"html", "sitemap", "feed"} and not base_url:
+            raise CommandError("--base-url is verplicht voor html/sitemap/feed")
+
+        if not input_path.is_file():
+            raise CommandError(f"Inputbestand niet gevonden: {input_path}")
+
+        if mode in {"html", "sitemap", "feed"}:
+            content = input_path.read_text(encoding="utf-8")
+        else:
+            content = input_path.read_bytes()
+
+        if mode == "html":
+            candidates = discover_from_html(content, base_url=base_url)
+        elif mode == "sitemap":
+            candidates = discover_from_sitemap(content, base_url=base_url)
+        elif mode == "feed":
+            candidates = discover_from_feed(content, base_url=base_url)
+        else:
+            candidates = discover_from_email(content)
+
+        self.stdout.write(f"Ontdekt {len(candidates)} kandidaten (mode={mode}).")
+        if not candidates:
+            self.stdout.write(self.style.WARNING("Geen bruikbare kandidaten gevonden."))
+            return
+
+        if options["dry_run"]:
+            self.stdout.write(self.style.WARNING("Dry-run modus: er wordt niet geschreven."))
+            for candidate in candidates:
+                self.stdout.write(f"- {candidate.source_type} {candidate.url} [{candidate.reason}]")
+            return
+
+        created, updated, skipped = persist_discovery_candidates(candidates)
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Persist result: {created} nieuw, {updated} bijgewerkt, {skipped} ongewijzigd."
+            )
+        )

apps/sources/management/commands/import_job_fixture.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import timedelta
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+
+from apps.jobs.services.pipeline import process_raw_document
+from apps.sources.models import RawDocument, Source, SourceRun
+
+
+class Command(BaseCommand):
+    help = "Importeert een lokale HTML/XML-fixture via dezelfde pipeline als een live bron."
+
+    def add_arguments(self, parser):
+        parser.add_argument("path")
+        parser.add_argument("--url", required=True)
+        parser.add_argument("--source-name", default="Lokale fixture")
+        parser.add_argument("--source-type", default=Source.Type.EMPLOYER)
+
+    def handle(self, *args, **options):
+        path = Path(options["path"]).resolve()
+        if not path.is_file():
+            raise CommandError(f"Bestand bestaat niet: {path}")
+        content = path.read_text(encoding="utf-8")
+        hostname = (urlsplit(options["url"]).hostname or "").lower()
+        if not hostname:
+            raise CommandError("--url bevat geen geldig domein")
+        source, _ = Source.objects.get_or_create(
+            domain=hostname,
+            source_type=options["source_type"],
+            defaults={
+                "name": options["source_name"],
+                "base_url": options["url"],
+                "status": Source.Status.ACTIVE,
+                "policy": Source.Policy.ALLOW,
+                "parser_key": "auto",
+            },
+        )
+        run = SourceRun.objects.create(source=source)
+        suffix = path.suffix.lower()
+        kind = RawDocument.Kind.XML if suffix in {".xml", ".rss"} else RawDocument.Kind.HTML
+        document = RawDocument.objects.create(
+            source=source,
+            source_run=run,
+            url=options["url"],
+            final_url=options["url"],
+            kind=kind,
+            content_type="application/xml" if kind == RawDocument.Kind.XML else "text/html",
+            content_hash=hashlib.sha256(content.encode()).hexdigest(),
+            body_text=content,
+            byte_length=len(content.encode()),
+            retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
+        )
+        metrics = process_raw_document(document)
+        run.finish(
+            SourceRun.Status.SUCCESS,
+            extracted_count=int(metrics["extracted"]),
+            created_count=int(metrics["created"]),
+            updated_count=int(metrics["updated"]),
+            duplicate_count=int(metrics["duplicates"]),
+            metrics=metrics,
+        )
+        self.stdout.write(self.style.SUCCESS(str(metrics)))

apps/sources/management/commands/seed_sources.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import yaml
+from django.core.management.base import BaseCommand, CommandError
+
+from apps.sources.models import Source
+from apps.sources.services.policy import is_denied_domain
+
+
+class Command(BaseCommand):
+    help = "Laadt gecontroleerde bronseeds uit YAML."
+
+    def add_arguments(self, parser):
+        parser.add_argument("path", nargs="?", default="config-data/seed_sources.yaml")
+
+    def handle(self, *args, **options):
+        path = Path(options["path"])
+        if not path.is_file():
+            raise CommandError(f"Seedbestand ontbreekt: {path}")
+        payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+        created = updated = 0
+        for item in payload.get("sources", []):
+            url = str(item.get("url") or "").strip()
+            domain = (urlsplit(url).hostname or "").lower()
+            if not domain:
+                self.stderr.write(f"Overgeslagen zonder domein: {item}")
+                continue
+            deny = is_denied_domain(domain)
+            source, was_created = Source.objects.update_or_create(
+                domain=domain,
+                source_type=item.get("type", Source.Type.EMPLOYER),
+                defaults={
+                    "name": item.get("name") or domain,
+                    "base_url": url,
+                    "status": Source.Status.PAUSED
+                    if deny
+                    else item.get("status", Source.Status.TRIAL),
+                    "policy": Source.Policy.DENY
+                    if deny
+                    else item.get("policy", Source.Policy.REVIEW),
+                    "policy_reason": "Standaard platformdenylist"
+                    if deny
+                    else item.get("policy_reason", ""),
+                    "parser_key": item.get("parser", "auto"),
+                    "crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)),
+                },
+            )
+            created += int(was_created)
+            updated += int(not was_created)
+        self.stdout.write(
+            self.style.SUCCESS(f"Bronnen: {created} aangemaakt, {updated} bijgewerkt")
+        )