from __future__ import annotations from pathlib import Path from urllib.parse import urlsplit import yaml from django.core.management.base import BaseCommand, CommandError from apps.sources.models import Source from apps.sources.services.policy import is_denied_domain class Command(BaseCommand): help = "Laadt gecontroleerde bronseeds uit YAML." def add_arguments(self, parser): parser.add_argument("path", nargs="?", default="config-data/seed_sources.yaml") def handle(self, *args, **options): path = Path(options["path"]) if not path.is_file(): raise CommandError(f"Seedbestand ontbreekt: {path}") payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} created = updated = 0 for item in payload.get("sources", []): url = str(item.get("url") or "").strip() domain = (urlsplit(url).hostname or "").lower() if not domain: self.stderr.write(f"Overgeslagen zonder domein: {item}") continue deny = is_denied_domain(domain) source, was_created = Source.objects.update_or_create( domain=domain, source_type=item.get("type", Source.Type.EMPLOYER), defaults={ "name": item.get("name") or domain, "base_url": url, "status": Source.Status.PAUSED if deny else item.get("status", Source.Status.TRIAL), "policy": Source.Policy.DENY if deny else item.get("policy", Source.Policy.REVIEW), "policy_reason": "Standaard platformdenylist" if deny else item.get("policy_reason", ""), "parser_key": item.get("parser", "auto"), "crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)), }, ) created += int(was_created) updated += int(not was_created) self.stdout.write( self.style.SUCCESS(f"Bronnen: {created} aangemaakt, {updated} bijgewerkt") )