from __future__ import annotations from pathlib import Path from urllib.parse import urlsplit import yaml from django.core.management.base import BaseCommand, CommandError from django.utils.dateparse import parse_datetime from apps.sources.models import Source, SourcePolicyReview from apps.sources.services.policy import is_denied_domain class Command(BaseCommand): help = "Laadt gecontroleerde bronseeds uit YAML." def add_arguments(self, parser): parser.add_argument("path", nargs="?", default="config-data/seed_sources.yaml") def handle(self, *args, **options): path = Path(options["path"]) if not path.is_file(): raise CommandError(f"Seedbestand ontbreekt: {path}") payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} created = updated = reviews_created = 0 if payload.get("disable_demo_sources") is True: demo_sources = Source.objects.filter(domain="jobs.example.org") for demo_source in demo_sources: demo_source.status = Source.Status.DISABLED demo_source.policy = Source.Policy.DENY demo_source.policy_reason = "Demobron verborgen na installatie van livebronnen" demo_source.metadata = {**demo_source.metadata, "demo": True} demo_source.save( update_fields=[ "status", "policy", "policy_reason", "metadata", "updated_at", ] ) retired = 0 for retired_url in payload.get("retire_source_urls", []): for retired_source in Source.objects.filter(base_url=str(retired_url).strip()): retired_source.status = Source.Status.DISABLED retired_source.next_run_at = None retired_source.metadata = { **retired_source.metadata, "retired": True, "retired_reason": "Buiten de ingestelde regionale dekking", "hidden_from_source_list": True, } retired_source.save( update_fields=["status", "next_run_at", "metadata", "updated_at"] ) retired += 1 for item in payload.get("sources", []): url = str(item.get("url") or "").strip() domain = (urlsplit(url).hostname or "").lower() if not domain: self.stderr.write(f"Overgeslagen zonder domein: {item}") continue deny = is_denied_domain(domain) source, was_created = Source.objects.update_or_create( base_url=url, source_type=item.get("type", Source.Type.EMPLOYER), defaults={ "domain": domain, "name": item.get("name") or domain, "status": Source.Status.PAUSED if deny else item.get("status", Source.Status.TRIAL), "policy": Source.Policy.DENY if deny else item.get("policy", Source.Policy.REVIEW), "policy_reason": "Standaard platformdenylist" if deny else item.get("policy_reason", ""), "parser_key": item.get("parser", "auto"), "crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)), "minimum_interval_seconds": int(item.get("minimum_interval_seconds", 30)), "allow_public_endpoint": bool(item.get("allow_public_endpoint", False)), "metadata": item.get("metadata") or {}, }, ) review = item.get("review") or {} if review: expires_at = parse_datetime(str(review.get("expires_at") or "")) if expires_at is None: raise CommandError( f"Bron {source.name}: review.expires_at moet een ISO-datetime zijn" ) decision = str(review.get("decision") or "") if decision not in SourcePolicyReview.Decision.values: raise CommandError(f"Bron {source.name}: ongeldige reviewbeslissing") _, review_created = SourcePolicyReview.objects.get_or_create( source=source, decision=decision, evidence_link=str(review.get("evidence_link") or ""), expires_at=expires_at, defaults={ "scope": SourcePolicyReview.Scope.SOURCE, "reason": str(review.get("reason") or "Bronseedreview"), "notes": str(review.get("notes") or ""), "metadata": review.get("metadata") or {}, }, ) reviews_created += int(review_created) created += int(was_created) updated += int(not was_created) self.stdout.write( self.style.SUCCESS( f"Bronnen: {created} aangemaakt, {updated} bijgewerkt, " f"{reviews_created} reviews geregistreerd, {retired} buiten regio uitgeschakeld" ) )