from __future__ import annotations from dataclasses import dataclass from datetime import datetime, timedelta from urllib.parse import urlsplit from django.conf import settings from django.utils import timezone from apps.sources.models import Source, SourcePolicyReview from .canonicalize import domain_matches from .robots import assess_robots DEFAULT_DENYLIST = { "linkedin.com", "indeed.com", "indeed.be", "stepstone.be", "jobat.be", "vdab.be", "ictjob.be", "careerjet.be", "randstad.be", "roberthalf.com", } @dataclass(frozen=True) class PolicyDecision: allowed: bool status: str reason: str def _review_expiry(now: datetime | None = None) -> datetime: return (now or timezone.now()) + timedelta(days=getattr(settings, "SOURCE_REVIEW_TTL_DAYS", 90)) def is_denied_domain(hostname: str, denylist: set[str] | None = None) -> bool: denylist = denylist or DEFAULT_DENYLIST return any(domain_matches(hostname, domain) for domain in denylist) def create_policy_review( source: Source, *, actor, decision: SourcePolicyReview.Decision, reason: str, scope: SourcePolicyReview.Scope = SourcePolicyReview.Scope.SOURCE, notes: str = "", evidence_link: str = "", expires_at: datetime | None = None, metadata: dict | None = None, ) -> SourcePolicyReview: return SourcePolicyReview.objects.create( source=source, actor=actor if actor and getattr(actor, "pk", None) else None, decision=decision, scope=scope, reason=reason, notes=notes, evidence_link=evidence_link, expires_at=expires_at or _review_expiry(), metadata=metadata or {}, ) def _active_review(source: Source) -> SourcePolicyReview | None: review = source.latest_policy_review if review and not review.is_expired: return review return None def _check_review_gate(source: Source) -> PolicyDecision | None: review = _active_review(source) if source.policy == Source.Policy.REVIEW and source.status in { Source.Status.CANDIDATE, Source.Status.TRIAL, }: if not review: return PolicyDecision(False, Source.Policy.REVIEW, "Review vereist") if review.decision == SourcePolicyReview.Decision.DENY: return PolicyDecision( False, Source.Policy.DENY, review.reason or "Review blokkeert bron" ) if review.decision == SourcePolicyReview.Decision.PAUSE: return PolicyDecision( False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze" ) return None if source.policy == Source.Policy.ALLOW: if not review: return PolicyDecision( False, Source.Policy.REVIEW, "Review ontbreekt voor actief beleid" ) if review.decision == SourcePolicyReview.Decision.DENY: return PolicyDecision( False, Source.Policy.DENY, review.reason or "Review blokkeert bron" ) if review.decision == SourcePolicyReview.Decision.PAUSE: return PolicyDecision( False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze" ) return None def assess_url(url: str, *, source: Source | None = None) -> PolicyDecision: hostname = (urlsplit(url).hostname or "").lower() if not hostname: return PolicyDecision(False, Source.Policy.DENY, "URL zonder hostname") if is_denied_domain(hostname): return PolicyDecision(False, Source.Policy.DENY, "Platformdomein staat op de denylist") if source: if source.status in {Source.Status.DISABLED, Source.Status.PAUSED}: return PolicyDecision(False, Source.Policy.REVIEW, f"Bronstatus: {source.status}") if source.policy == Source.Policy.DENY: return PolicyDecision( False, Source.Policy.DENY, source.policy_reason or "Bron geblokkeerd" ) review_decision = _check_review_gate(source) if review_decision is not None: return review_decision robots = assess_robots(url, source=source) if not robots.allowed: return PolicyDecision(False, Source.Policy.REVIEW, robots.reason) return PolicyDecision(True, Source.Policy.ALLOW, "Toegestane publieke bron")