Initial deploy setup

2026-07-21 14:00:00 +02:00
commit b8091e59bd
285 changed files with 27854 additions and 0 deletions
@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from urllib.parse import urlsplit
+
+from django.conf import settings
+from django.utils import timezone
+
+from apps.sources.models import Source, SourcePolicyReview
+
+from .canonicalize import domain_matches
+from .robots import assess_robots
+
+DEFAULT_DENYLIST = {
+    "linkedin.com",
+    "indeed.com",
+    "indeed.be",
+    "stepstone.be",
+    "jobat.be",
+    "vdab.be",
+}
+
+
+@dataclass(frozen=True)
+class PolicyDecision:
+    allowed: bool
+    status: str
+    reason: str
+
+
+def _review_expiry(now: datetime | None = None) -> datetime:
+    return (now or timezone.now()) + timedelta(days=getattr(settings, "SOURCE_REVIEW_TTL_DAYS", 90))
+
+
+def is_denied_domain(hostname: str, denylist: set[str] | None = None) -> bool:
+    denylist = denylist or DEFAULT_DENYLIST
+    return any(domain_matches(hostname, domain) for domain in denylist)
+
+
+def create_policy_review(
+    source: Source,
+    *,
+    actor,
+    decision: SourcePolicyReview.Decision,
+    reason: str,
+    scope: SourcePolicyReview.Scope = SourcePolicyReview.Scope.SOURCE,
+    notes: str = "",
+    evidence_link: str = "",
+    expires_at: datetime | None = None,
+    metadata: dict | None = None,
+) -> SourcePolicyReview:
+    return SourcePolicyReview.objects.create(
+        source=source,
+        actor=actor if actor and getattr(actor, "pk", None) else None,
+        decision=decision,
+        scope=scope,
+        reason=reason,
+        notes=notes,
+        evidence_link=evidence_link,
+        expires_at=expires_at or _review_expiry(),
+        metadata=metadata or {},
+    )
+
+
+def _active_review(source: Source) -> SourcePolicyReview | None:
+    review = source.latest_policy_review
+    if review and not review.is_expired:
+        return review
+    return None
+
+
+def _check_review_gate(source: Source) -> PolicyDecision | None:
+    review = _active_review(source)
+    if source.policy == Source.Policy.REVIEW and source.status in {
+        Source.Status.CANDIDATE,
+        Source.Status.TRIAL,
+    }:
+        if not review:
+            return PolicyDecision(False, Source.Policy.REVIEW, "Review vereist")
+        if review.decision == SourcePolicyReview.Decision.DENY:
+            return PolicyDecision(False, Source.Policy.DENY, review.reason or "Review blokkeert bron")
+        if review.decision == SourcePolicyReview.Decision.PAUSE:
+            return PolicyDecision(False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze")
+        return None
+
+    if source.policy == Source.Policy.ALLOW:
+        if not review:
+            return PolicyDecision(False, Source.Policy.REVIEW, "Review ontbreekt voor actief beleid")
+        if review.decision == SourcePolicyReview.Decision.DENY:
+            return PolicyDecision(False, Source.Policy.DENY, review.reason or "Review blokkeert bron")
+        if review.decision == SourcePolicyReview.Decision.PAUSE:
+            return PolicyDecision(False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze")
+    return None
+
+
+def assess_url(url: str, *, source: Source | None = None) -> PolicyDecision:
+    hostname = (urlsplit(url).hostname or "").lower()
+    if not hostname:
+        return PolicyDecision(False, Source.Policy.DENY, "URL zonder hostname")
+    if is_denied_domain(hostname):
+        return PolicyDecision(False, Source.Policy.DENY, "Platformdomein staat op de denylist")
+    if source:
+        if source.status in {Source.Status.DISABLED, Source.Status.PAUSED}:
+            return PolicyDecision(False, Source.Policy.REVIEW, f"Bronstatus: {source.status}")
+        if source.policy == Source.Policy.DENY:
+            return PolicyDecision(
+                False, Source.Policy.DENY, source.policy_reason or "Bron geblokkeerd"
+            )
+        review_decision = _check_review_gate(source)
+        if review_decision is not None:
+            return review_decision
+
+    robots = assess_robots(url, source=source)
+    if not robots.allowed:
+        return PolicyDecision(False, Source.Policy.REVIEW, robots.reason)
+
+    return PolicyDecision(True, Source.Policy.ALLOW, "Toegestane publieke bron")