56 lines
2.2 KiB
Python
56 lines
2.2 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
import yaml
|
|
from django.core.management.base import BaseCommand, CommandError
|
|
|
|
from apps.sources.models import Source
|
|
from apps.sources.services.policy import is_denied_domain
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "Laadt gecontroleerde bronseeds uit YAML."
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument("path", nargs="?", default="config-data/seed_sources.yaml")
|
|
|
|
def handle(self, *args, **options):
|
|
path = Path(options["path"])
|
|
if not path.is_file():
|
|
raise CommandError(f"Seedbestand ontbreekt: {path}")
|
|
payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
|
created = updated = 0
|
|
for item in payload.get("sources", []):
|
|
url = str(item.get("url") or "").strip()
|
|
domain = (urlsplit(url).hostname or "").lower()
|
|
if not domain:
|
|
self.stderr.write(f"Overgeslagen zonder domein: {item}")
|
|
continue
|
|
deny = is_denied_domain(domain)
|
|
source, was_created = Source.objects.update_or_create(
|
|
domain=domain,
|
|
source_type=item.get("type", Source.Type.EMPLOYER),
|
|
defaults={
|
|
"name": item.get("name") or domain,
|
|
"base_url": url,
|
|
"status": Source.Status.PAUSED
|
|
if deny
|
|
else item.get("status", Source.Status.TRIAL),
|
|
"policy": Source.Policy.DENY
|
|
if deny
|
|
else item.get("policy", Source.Policy.REVIEW),
|
|
"policy_reason": "Standaard platformdenylist"
|
|
if deny
|
|
else item.get("policy_reason", ""),
|
|
"parser_key": item.get("parser", "auto"),
|
|
"crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)),
|
|
},
|
|
)
|
|
created += int(was_created)
|
|
updated += int(not was_created)
|
|
self.stdout.write(
|
|
self.style.SUCCESS(f"Bronnen: {created} aangemaakt, {updated} bijgewerkt")
|
|
)
|