336 lines
13 KiB
Python
336 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import timedelta
|
|
from urllib.parse import urlparse
|
|
|
|
from django.conf import settings
|
|
from django.core.exceptions import ValidationError
|
|
from django.db import models
|
|
from django.utils import timezone
|
|
|
|
from apps.core.models import TimeStampedModel
|
|
|
|
|
|
class Source(TimeStampedModel):
|
|
class Type(models.TextChoices):
|
|
EMPLOYER = "employer", "Werkgeverspagina"
|
|
ATS = "ats", "Publieke ATS-pagina"
|
|
RSS = "rss", "RSS/Atom"
|
|
SITEMAP = "sitemap", "Sitemap"
|
|
EMAIL = "email", "Vacaturemail"
|
|
MANUAL = "manual", "Handmatige import"
|
|
|
|
class Status(models.TextChoices):
|
|
CANDIDATE = "candidate", "Kandidaat"
|
|
TRIAL = "trial", "Proefrun"
|
|
ACTIVE = "active", "Actief"
|
|
QUARANTINED = "quarantined", "Quarantaine"
|
|
PAUSED = "paused", "Gepauzeerd"
|
|
DISABLED = "disabled", "Uitgeschakeld"
|
|
|
|
class Policy(models.TextChoices):
|
|
ALLOW = "allow", "Toestaan"
|
|
REVIEW = "review", "Te beoordelen"
|
|
DENY = "deny", "Blokkeren"
|
|
|
|
name = models.CharField(max_length=200)
|
|
source_type = models.CharField(max_length=24, choices=Type.choices)
|
|
base_url = models.URLField(max_length=1000, blank=True)
|
|
domain = models.CharField(max_length=255, db_index=True)
|
|
status = models.CharField(max_length=24, choices=Status.choices, default=Status.CANDIDATE)
|
|
policy = models.CharField(max_length=16, choices=Policy.choices, default=Policy.REVIEW)
|
|
policy_reason = models.CharField(max_length=500, blank=True)
|
|
parser_key = models.CharField(max_length=120, default="auto")
|
|
strict_mode = models.BooleanField(default=True)
|
|
allow_public_endpoint = models.BooleanField(default=False)
|
|
honor_robots = models.BooleanField(default=True)
|
|
|
|
crawl_interval_minutes = models.PositiveIntegerField(default=720)
|
|
minimum_interval_seconds = models.PositiveIntegerField(default=30)
|
|
max_concurrency = models.PositiveSmallIntegerField(default=1)
|
|
next_run_at = models.DateTimeField(null=True, blank=True, db_index=True)
|
|
last_success_at = models.DateTimeField(null=True, blank=True)
|
|
last_failure_at = models.DateTimeField(null=True, blank=True)
|
|
failure_count = models.PositiveIntegerField(default=0)
|
|
etag = models.CharField(max_length=500, blank=True)
|
|
last_modified = models.CharField(max_length=500, blank=True)
|
|
robots_checked_at = models.DateTimeField(null=True, blank=True)
|
|
terms_checked_at = models.DateTimeField(null=True, blank=True)
|
|
metadata = models.JSONField(default=dict, blank=True)
|
|
|
|
class Meta:
|
|
ordering = ["name"]
|
|
constraints = [
|
|
models.UniqueConstraint(
|
|
fields=["domain", "source_type"], name="unique_domain_source_type"
|
|
)
|
|
]
|
|
|
|
def __str__(self) -> str:
|
|
return self.name
|
|
|
|
def clean(self) -> None:
|
|
if self.base_url:
|
|
parsed = urlparse(self.base_url)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
raise ValidationError({"base_url": "Alleen geldige http(s)-URLs zijn toegestaan."})
|
|
if self.domain and parsed.hostname.lower().rstrip(".") != self.domain.lower().rstrip(
|
|
"."
|
|
):
|
|
raise ValidationError({"domain": "Domein moet overeenkomen met de base URL."})
|
|
if self.policy == self.Policy.DENY and self.status == self.Status.ACTIVE:
|
|
raise ValidationError("Een geblokkeerde bron kan niet actief zijn.")
|
|
|
|
@property
|
|
def discovery_evidence(self) -> list[dict[str, str | float]]:
|
|
raw = self.metadata if isinstance(self.metadata, dict) else {}
|
|
entries = raw.get("discovery", [])
|
|
if not isinstance(entries, list):
|
|
return []
|
|
return [entry for entry in entries if isinstance(entry, dict)]
|
|
|
|
@property
|
|
def latest_policy_review(self) -> SourcePolicyReview | None:
|
|
return self.policy_reviews.order_by("-created_at").first()
|
|
|
|
@property
|
|
def policy_review_state(self) -> str:
|
|
review = self.latest_policy_review
|
|
if not review:
|
|
return "missing"
|
|
if review.is_expired:
|
|
return "expired"
|
|
return "active"
|
|
|
|
@property
|
|
def policy_review_summary(self) -> str:
|
|
review = self.latest_policy_review
|
|
if not review:
|
|
return "Geen review geregistreerd"
|
|
if review.is_expired:
|
|
return (
|
|
f"Review verlopen op {review.expires_at:%Y-%m-%d}"
|
|
if review.expires_at
|
|
else "Review verlopen"
|
|
)
|
|
if review.expires_at:
|
|
return f"{review.get_decision_display()} geldig tot {review.expires_at:%Y-%m-%d}"
|
|
return f"{review.get_decision_display()} zonder vervaldatum"
|
|
|
|
@property
|
|
def requires_terms_review(self) -> bool:
|
|
return self.policy == Source.Policy.ALLOW and self.policy_review_state in {
|
|
"missing",
|
|
"expired",
|
|
}
|
|
|
|
def schedule_after_success(self, *, now=None, jitter_seconds: int = 0) -> None:
|
|
now = now or timezone.now()
|
|
self.last_success_at = now
|
|
self.failure_count = 0
|
|
self.next_run_at = (
|
|
now
|
|
+ timedelta(minutes=self.crawl_interval_minutes)
|
|
+ timedelta(seconds=max(0, jitter_seconds))
|
|
)
|
|
self.save(update_fields=["last_success_at", "failure_count", "next_run_at", "updated_at"])
|
|
|
|
def schedule_after_failure(
|
|
self,
|
|
*,
|
|
now=None,
|
|
backoff_minutes: int | None = None,
|
|
backoff_seconds: int | None = None,
|
|
) -> None:
|
|
now = now or timezone.now()
|
|
self.last_failure_at = now
|
|
self.failure_count += 1
|
|
if backoff_seconds is None:
|
|
backoff = backoff_minutes or min(24 * 60, max(15, 2 ** min(self.failure_count, 10)))
|
|
backoff_seconds = backoff * 60
|
|
self.next_run_at = now + timedelta(seconds=max(0, backoff_seconds))
|
|
self.save(
|
|
update_fields=[
|
|
"last_failure_at",
|
|
"failure_count",
|
|
"next_run_at",
|
|
"updated_at",
|
|
]
|
|
)
|
|
|
|
|
|
class SourceRun(TimeStampedModel):
|
|
class Status(models.TextChoices):
|
|
RUNNING = "running", "Bezig"
|
|
SUCCESS = "success", "Geslaagd"
|
|
PARTIAL = "partial", "Gedeeltelijk"
|
|
FAILED = "failed", "Mislukt"
|
|
SKIPPED = "skipped", "Overgeslagen"
|
|
|
|
source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="runs")
|
|
status = models.CharField(max_length=16, choices=Status.choices, default=Status.RUNNING)
|
|
started_at = models.DateTimeField(default=timezone.now)
|
|
finished_at = models.DateTimeField(null=True, blank=True)
|
|
http_status = models.PositiveSmallIntegerField(null=True, blank=True)
|
|
discovered_count = models.PositiveIntegerField(default=0)
|
|
extracted_count = models.PositiveIntegerField(default=0)
|
|
created_count = models.PositiveIntegerField(default=0)
|
|
updated_count = models.PositiveIntegerField(default=0)
|
|
duplicate_count = models.PositiveIntegerField(default=0)
|
|
error_category = models.CharField(max_length=80, blank=True)
|
|
error_message = models.CharField(max_length=1000, blank=True)
|
|
metrics = models.JSONField(default=dict, blank=True)
|
|
|
|
class Meta:
|
|
ordering = ["-started_at"]
|
|
|
|
def finish(self, status: str, **metrics) -> None:
|
|
self.status = status
|
|
self.finished_at = timezone.now()
|
|
for key, value in metrics.items():
|
|
if hasattr(self, key):
|
|
setattr(self, key, value)
|
|
self.save()
|
|
|
|
|
|
class SourceLease(TimeStampedModel):
|
|
source = models.OneToOneField(Source, on_delete=models.CASCADE, related_name="lease")
|
|
token = models.CharField(max_length=64, default="")
|
|
worker_id = models.CharField(max_length=128, blank=True)
|
|
expires_at = models.DateTimeField(db_index=True)
|
|
|
|
class Meta:
|
|
ordering = ["source"]
|
|
|
|
def __str__(self) -> str:
|
|
return f"{self.source_id}:{self.token}"
|
|
|
|
@property
|
|
def is_expired(self) -> bool:
|
|
return self.expires_at <= timezone.now()
|
|
|
|
|
|
class SourceOriginState(TimeStampedModel):
|
|
domain = models.CharField(max_length=255, unique=True)
|
|
next_allowed_at = models.DateTimeField()
|
|
|
|
class Meta:
|
|
ordering = ["domain"]
|
|
|
|
def __str__(self) -> str:
|
|
return self.domain
|
|
|
|
|
|
class SourcePolicyReview(TimeStampedModel):
|
|
class Scope(models.TextChoices):
|
|
SOURCE = "source", "Bronspecifiek"
|
|
DOMAIN = "domain", "Domeinbrede controle"
|
|
|
|
class Decision(models.TextChoices):
|
|
ALLOW = "allow", "Toestaan"
|
|
TRIAL = "trial", "Proefrun"
|
|
PAUSE = "pause", "Pauzeren"
|
|
DENY = "deny", "Blokkeren"
|
|
|
|
source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="policy_reviews")
|
|
actor = models.ForeignKey(
|
|
settings.AUTH_USER_MODEL,
|
|
on_delete=models.SET_NULL,
|
|
null=True,
|
|
blank=True,
|
|
related_name="policy_reviews",
|
|
)
|
|
scope = models.CharField(max_length=20, choices=Scope.choices, default=Scope.SOURCE)
|
|
decision = models.CharField(max_length=16, choices=Decision.choices)
|
|
reason = models.CharField(max_length=500)
|
|
evidence_link = models.URLField(max_length=500, blank=True)
|
|
notes = models.TextField(blank=True)
|
|
expires_at = models.DateTimeField(null=True, blank=True)
|
|
metadata = models.JSONField(default=dict, blank=True)
|
|
|
|
class Meta:
|
|
ordering = ["-created_at"]
|
|
|
|
@property
|
|
def is_expired(self) -> bool:
|
|
if not self.expires_at:
|
|
return False
|
|
return self.expires_at <= timezone.now()
|
|
|
|
|
|
class SourceRobotsCache(TimeStampedModel):
|
|
origin = models.CharField(max_length=255, unique=True)
|
|
expires_at = models.DateTimeField()
|
|
etag = models.CharField(max_length=255, blank=True)
|
|
last_modified = models.CharField(max_length=255, blank=True)
|
|
allow_rules = models.JSONField(default=dict, blank=True)
|
|
disallow_rules = models.JSONField(default=dict, blank=True)
|
|
error = models.CharField(max_length=500, blank=True)
|
|
byte_length = models.PositiveIntegerField(default=0)
|
|
|
|
class Meta:
|
|
ordering = ["origin"]
|
|
|
|
@property
|
|
def is_fresh(self) -> bool:
|
|
return self.expires_at > timezone.now()
|
|
|
|
|
|
class RawDocument(TimeStampedModel):
|
|
class Kind(models.TextChoices):
|
|
HTML = "html", "HTML"
|
|
XML = "xml", "XML"
|
|
EMAIL = "email", "E-mail"
|
|
JSON = "json", "JSON"
|
|
TEXT = "text", "Tekst"
|
|
|
|
source = models.ForeignKey(
|
|
Source, on_delete=models.SET_NULL, null=True, related_name="documents"
|
|
)
|
|
source_run = models.ForeignKey(
|
|
SourceRun, on_delete=models.SET_NULL, null=True, blank=True, related_name="documents"
|
|
)
|
|
url = models.URLField(max_length=2000, blank=True)
|
|
final_url = models.URLField(max_length=2000, blank=True)
|
|
kind = models.CharField(max_length=16, choices=Kind.choices)
|
|
content_type = models.CharField(max_length=200, blank=True)
|
|
http_status = models.PositiveSmallIntegerField(null=True, blank=True)
|
|
response_headers = models.JSONField(default=dict, blank=True)
|
|
content_hash = models.CharField(max_length=64, db_index=True)
|
|
body_text = models.TextField(blank=True)
|
|
byte_length = models.PositiveIntegerField(default=0)
|
|
parser_key = models.CharField(max_length=120, blank=True)
|
|
parser_version = models.CharField(max_length=80, blank=True)
|
|
extraction_confidence = models.DecimalField(max_digits=4, decimal_places=3, default=0)
|
|
retain_until = models.DateTimeField(null=True, blank=True)
|
|
quarantined = models.BooleanField(default=False)
|
|
quarantine_reason = models.CharField(max_length=500, blank=True)
|
|
metadata = models.JSONField(default=dict, blank=True)
|
|
|
|
class Meta:
|
|
ordering = ["-created_at"]
|
|
indexes = [models.Index(fields=["source", "content_hash"])]
|
|
|
|
def __str__(self) -> str:
|
|
return self.final_url or self.url or f"Document {self.pk}"
|
|
|
|
|
|
class EmailMessageRecord(TimeStampedModel):
|
|
message_id = models.CharField(max_length=998, unique=True)
|
|
mailbox = models.CharField(max_length=255, default="INBOX")
|
|
sender = models.CharField(max_length=500, blank=True)
|
|
subject = models.CharField(max_length=998, blank=True)
|
|
received_at = models.DateTimeField(null=True, blank=True)
|
|
raw_document = models.OneToOneField(
|
|
RawDocument, on_delete=models.SET_NULL, null=True, blank=True, related_name="email_record"
|
|
)
|
|
links = models.JSONField(default=list, blank=True)
|
|
processed = models.BooleanField(default=False)
|
|
error_message = models.CharField(max_length=1000, blank=True)
|
|
|
|
class Meta:
|
|
ordering = ["-received_at", "-created_at"]
|
|
|
|
def __str__(self) -> str:
|
|
return self.subject or self.message_id
|