117 lines
5.3 KiB
Python
117 lines
5.3 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
import yaml
|
|
from django.core.management.base import BaseCommand, CommandError
|
|
from django.utils.dateparse import parse_datetime
|
|
|
|
from apps.sources.models import Source, SourcePolicyReview
|
|
from apps.sources.services.policy import is_denied_domain
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "Laadt gecontroleerde bronseeds uit YAML."
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument("path", nargs="?", default="config-data/seed_sources.yaml")
|
|
|
|
def handle(self, *args, **options):
|
|
path = Path(options["path"])
|
|
if not path.is_file():
|
|
raise CommandError(f"Seedbestand ontbreekt: {path}")
|
|
payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
|
created = updated = reviews_created = 0
|
|
if payload.get("disable_demo_sources") is True:
|
|
demo_sources = Source.objects.filter(domain="jobs.example.org")
|
|
for demo_source in demo_sources:
|
|
demo_source.status = Source.Status.DISABLED
|
|
demo_source.policy = Source.Policy.DENY
|
|
demo_source.policy_reason = "Demobron verborgen na installatie van livebronnen"
|
|
demo_source.metadata = {**demo_source.metadata, "demo": True}
|
|
demo_source.save(
|
|
update_fields=[
|
|
"status",
|
|
"policy",
|
|
"policy_reason",
|
|
"metadata",
|
|
"updated_at",
|
|
]
|
|
)
|
|
retired = 0
|
|
for retired_url in payload.get("retire_source_urls", []):
|
|
for retired_source in Source.objects.filter(base_url=str(retired_url).strip()):
|
|
retired_source.status = Source.Status.DISABLED
|
|
retired_source.next_run_at = None
|
|
retired_source.metadata = {
|
|
**retired_source.metadata,
|
|
"retired": True,
|
|
"retired_reason": "Buiten de ingestelde regionale dekking",
|
|
"hidden_from_source_list": True,
|
|
}
|
|
retired_source.save(
|
|
update_fields=["status", "next_run_at", "metadata", "updated_at"]
|
|
)
|
|
retired += 1
|
|
for item in payload.get("sources", []):
|
|
url = str(item.get("url") or "").strip()
|
|
domain = (urlsplit(url).hostname or "").lower()
|
|
if not domain:
|
|
self.stderr.write(f"Overgeslagen zonder domein: {item}")
|
|
continue
|
|
deny = is_denied_domain(domain)
|
|
source, was_created = Source.objects.update_or_create(
|
|
base_url=url,
|
|
source_type=item.get("type", Source.Type.EMPLOYER),
|
|
defaults={
|
|
"domain": domain,
|
|
"name": item.get("name") or domain,
|
|
"status": Source.Status.PAUSED
|
|
if deny
|
|
else item.get("status", Source.Status.TRIAL),
|
|
"policy": Source.Policy.DENY
|
|
if deny
|
|
else item.get("policy", Source.Policy.REVIEW),
|
|
"policy_reason": "Standaard platformdenylist"
|
|
if deny
|
|
else item.get("policy_reason", ""),
|
|
"parser_key": item.get("parser", "auto"),
|
|
"crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)),
|
|
"minimum_interval_seconds": int(item.get("minimum_interval_seconds", 30)),
|
|
"allow_public_endpoint": bool(item.get("allow_public_endpoint", False)),
|
|
"metadata": item.get("metadata") or {},
|
|
},
|
|
)
|
|
review = item.get("review") or {}
|
|
if review:
|
|
expires_at = parse_datetime(str(review.get("expires_at") or ""))
|
|
if expires_at is None:
|
|
raise CommandError(
|
|
f"Bron {source.name}: review.expires_at moet een ISO-datetime zijn"
|
|
)
|
|
decision = str(review.get("decision") or "")
|
|
if decision not in SourcePolicyReview.Decision.values:
|
|
raise CommandError(f"Bron {source.name}: ongeldige reviewbeslissing")
|
|
_, review_created = SourcePolicyReview.objects.get_or_create(
|
|
source=source,
|
|
decision=decision,
|
|
evidence_link=str(review.get("evidence_link") or ""),
|
|
expires_at=expires_at,
|
|
defaults={
|
|
"scope": SourcePolicyReview.Scope.SOURCE,
|
|
"reason": str(review.get("reason") or "Bronseedreview"),
|
|
"notes": str(review.get("notes") or ""),
|
|
"metadata": review.get("metadata") or {},
|
|
},
|
|
)
|
|
reviews_created += int(review_created)
|
|
created += int(was_created)
|
|
updated += int(not was_created)
|
|
self.stdout.write(
|
|
self.style.SUCCESS(
|
|
f"Bronnen: {created} aangemaakt, {updated} bijgewerkt, "
|
|
f"{reviews_created} reviews geregistreerd, {retired} buiten regio uitgeschakeld"
|
|
)
|
|
)
|