133 lines
4.3 KiB
Python
133 lines
4.3 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta
|
|
from urllib.parse import urlsplit
|
|
|
|
from django.conf import settings
|
|
from django.utils import timezone
|
|
|
|
from apps.sources.models import Source, SourcePolicyReview
|
|
|
|
from .canonicalize import domain_matches
|
|
from .robots import assess_robots
|
|
|
|
DEFAULT_DENYLIST = {
|
|
"linkedin.com",
|
|
"indeed.com",
|
|
"indeed.be",
|
|
"stepstone.be",
|
|
"jobat.be",
|
|
"vdab.be",
|
|
"ictjob.be",
|
|
"careerjet.be",
|
|
"randstad.be",
|
|
"roberthalf.com",
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PolicyDecision:
|
|
allowed: bool
|
|
status: str
|
|
reason: str
|
|
|
|
|
|
def _review_expiry(now: datetime | None = None) -> datetime:
|
|
return (now or timezone.now()) + timedelta(days=getattr(settings, "SOURCE_REVIEW_TTL_DAYS", 90))
|
|
|
|
|
|
def is_denied_domain(hostname: str, denylist: set[str] | None = None) -> bool:
|
|
denylist = denylist or DEFAULT_DENYLIST
|
|
return any(domain_matches(hostname, domain) for domain in denylist)
|
|
|
|
|
|
def create_policy_review(
|
|
source: Source,
|
|
*,
|
|
actor,
|
|
decision: SourcePolicyReview.Decision,
|
|
reason: str,
|
|
scope: SourcePolicyReview.Scope = SourcePolicyReview.Scope.SOURCE,
|
|
notes: str = "",
|
|
evidence_link: str = "",
|
|
expires_at: datetime | None = None,
|
|
metadata: dict | None = None,
|
|
) -> SourcePolicyReview:
|
|
return SourcePolicyReview.objects.create(
|
|
source=source,
|
|
actor=actor if actor and getattr(actor, "pk", None) else None,
|
|
decision=decision,
|
|
scope=scope,
|
|
reason=reason,
|
|
notes=notes,
|
|
evidence_link=evidence_link,
|
|
expires_at=expires_at or _review_expiry(),
|
|
metadata=metadata or {},
|
|
)
|
|
|
|
|
|
def _active_review(source: Source) -> SourcePolicyReview | None:
|
|
review = source.latest_policy_review
|
|
if review and not review.is_expired:
|
|
return review
|
|
return None
|
|
|
|
|
|
def _check_review_gate(source: Source) -> PolicyDecision | None:
|
|
review = _active_review(source)
|
|
if source.policy == Source.Policy.REVIEW and source.status in {
|
|
Source.Status.CANDIDATE,
|
|
Source.Status.TRIAL,
|
|
}:
|
|
if not review:
|
|
return PolicyDecision(False, Source.Policy.REVIEW, "Review vereist")
|
|
if review.decision == SourcePolicyReview.Decision.DENY:
|
|
return PolicyDecision(
|
|
False, Source.Policy.DENY, review.reason or "Review blokkeert bron"
|
|
)
|
|
if review.decision == SourcePolicyReview.Decision.PAUSE:
|
|
return PolicyDecision(
|
|
False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze"
|
|
)
|
|
return None
|
|
|
|
if source.policy == Source.Policy.ALLOW:
|
|
if not review:
|
|
return PolicyDecision(
|
|
False, Source.Policy.REVIEW, "Review ontbreekt voor actief beleid"
|
|
)
|
|
if review.decision == SourcePolicyReview.Decision.DENY:
|
|
return PolicyDecision(
|
|
False, Source.Policy.DENY, review.reason or "Review blokkeert bron"
|
|
)
|
|
if review.decision == SourcePolicyReview.Decision.PAUSE:
|
|
return PolicyDecision(
|
|
False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze"
|
|
)
|
|
return None
|
|
|
|
|
|
def assess_url(url: str, *, source: Source | None = None) -> PolicyDecision:
|
|
hostname = (urlsplit(url).hostname or "").lower()
|
|
if not hostname:
|
|
return PolicyDecision(False, Source.Policy.DENY, "URL zonder hostname")
|
|
if is_denied_domain(hostname):
|
|
return PolicyDecision(False, Source.Policy.DENY, "Platformdomein staat op de denylist")
|
|
if source:
|
|
if source.status in {Source.Status.DISABLED, Source.Status.PAUSED}:
|
|
return PolicyDecision(False, Source.Policy.REVIEW, f"Bronstatus: {source.status}")
|
|
if source.policy == Source.Policy.DENY:
|
|
return PolicyDecision(
|
|
False, Source.Policy.DENY, source.policy_reason or "Bron geblokkeerd"
|
|
)
|
|
review_decision = _check_review_gate(source)
|
|
if review_decision is not None:
|
|
return review_decision
|
|
|
|
robots = assess_robots(url, source=source)
|
|
if not robots.allowed:
|
|
return PolicyDecision(False, Source.Policy.REVIEW, robots.reason)
|
|
|
|
return PolicyDecision(True, Source.Policy.ALLOW, "Toegestane publieke bron")
|