Initial deploy setup

apps/sources/models.py

@@ -0,0 +1,326 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from uuid import uuid4
+from urllib.parse import urlparse
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.utils import timezone
+
+from apps.core.models import TimeStampedModel
+
+
+class Source(TimeStampedModel):
+    class Type(models.TextChoices):
+        EMPLOYER = "employer", "Werkgeverspagina"
+        ATS = "ats", "Publieke ATS-pagina"
+        RSS = "rss", "RSS/Atom"
+        SITEMAP = "sitemap", "Sitemap"
+        EMAIL = "email", "Vacaturemail"
+        MANUAL = "manual", "Handmatige import"
+
+    class Status(models.TextChoices):
+        CANDIDATE = "candidate", "Kandidaat"
+        TRIAL = "trial", "Proefrun"
+        ACTIVE = "active", "Actief"
+        QUARANTINED = "quarantined", "Quarantaine"
+        PAUSED = "paused", "Gepauzeerd"
+        DISABLED = "disabled", "Uitgeschakeld"
+
+    class Policy(models.TextChoices):
+        ALLOW = "allow", "Toestaan"
+        REVIEW = "review", "Te beoordelen"
+        DENY = "deny", "Blokkeren"
+
+    name = models.CharField(max_length=200)
+    source_type = models.CharField(max_length=24, choices=Type.choices)
+    base_url = models.URLField(max_length=1000, blank=True)
+    domain = models.CharField(max_length=255, db_index=True)
+    status = models.CharField(max_length=24, choices=Status.choices, default=Status.CANDIDATE)
+    policy = models.CharField(max_length=16, choices=Policy.choices, default=Policy.REVIEW)
+    policy_reason = models.CharField(max_length=500, blank=True)
+    parser_key = models.CharField(max_length=120, default="auto")
+    strict_mode = models.BooleanField(default=True)
+    allow_public_endpoint = models.BooleanField(default=False)
+    honor_robots = models.BooleanField(default=True)
+
+    crawl_interval_minutes = models.PositiveIntegerField(default=720)
+    minimum_interval_seconds = models.PositiveIntegerField(default=30)
+    max_concurrency = models.PositiveSmallIntegerField(default=1)
+    next_run_at = models.DateTimeField(null=True, blank=True, db_index=True)
+    last_success_at = models.DateTimeField(null=True, blank=True)
+    last_failure_at = models.DateTimeField(null=True, blank=True)
+    failure_count = models.PositiveIntegerField(default=0)
+    etag = models.CharField(max_length=500, blank=True)
+    last_modified = models.CharField(max_length=500, blank=True)
+    robots_checked_at = models.DateTimeField(null=True, blank=True)
+    terms_checked_at = models.DateTimeField(null=True, blank=True)
+    metadata = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["name"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["domain", "source_type"], name="unique_domain_source_type"
+            )
+        ]
+
+    def __str__(self) -> str:
+        return self.name
+
+    def clean(self) -> None:
+        if self.base_url:
+            parsed = urlparse(self.base_url)
+            if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+                raise ValidationError({"base_url": "Alleen geldige http(s)-URLs zijn toegestaan."})
+            if self.domain and parsed.hostname.lower().rstrip(".") != self.domain.lower().rstrip(
+                "."
+            ):
+                raise ValidationError({"domain": "Domein moet overeenkomen met de base URL."})
+        if self.policy == self.Policy.DENY and self.status == self.Status.ACTIVE:
+            raise ValidationError("Een geblokkeerde bron kan niet actief zijn.")
+
+    @property
+    def discovery_evidence(self) -> list[dict[str, str | float]]:
+        raw = self.metadata if isinstance(self.metadata, dict) else {}
+        entries = raw.get("discovery", [])
+        if not isinstance(entries, list):
+            return []
+        return [entry for entry in entries if isinstance(entry, dict)]
+
+    @property
+    def latest_policy_review(self) -> "SourcePolicyReview | None":
+        return self.policy_reviews.order_by("-created_at").first()
+
+    @property
+    def policy_review_state(self) -> str:
+        review = self.latest_policy_review
+        if not review:
+            return "missing"
+        if review.is_expired:
+            return "expired"
+        return "active"
+
+    @property
+    def policy_review_summary(self) -> str:
+        review = self.latest_policy_review
+        if not review:
+            return "Geen review geregistreerd"
+        if review.is_expired:
+            return f"Review verlopen op {review.expires_at:%Y-%m-%d}" if review.expires_at else "Review verlopen"
+        if review.expires_at:
+            return f"{review.get_decision_display()} geldig tot {review.expires_at:%Y-%m-%d}"
+        return f"{review.get_decision_display()} zonder vervaldatum"
+
+    @property
+    def requires_terms_review(self) -> bool:
+        return self.policy == Source.Policy.ALLOW and self.policy_review_state in {"missing", "expired"}
+
+    def schedule_after_success(self, *, now=None, jitter_seconds: int = 0) -> None:
+        now = now or timezone.now()
+        self.last_success_at = now
+        self.failure_count = 0
+        self.next_run_at = now + timedelta(minutes=self.crawl_interval_minutes) + timedelta(
+            seconds=max(0, jitter_seconds)
+        )
+        self.save(update_fields=["last_success_at", "failure_count", "next_run_at", "updated_at"])
+
+    def schedule_after_failure(
+        self,
+        *, now=None,
+        backoff_minutes: int | None = None,
+        backoff_seconds: int | None = None,
+    ) -> None:
+        now = now or timezone.now()
+        self.last_failure_at = now
+        self.failure_count += 1
+        if backoff_seconds is None:
+            backoff = backoff_minutes or min(24 * 60, max(15, 2 ** min(self.failure_count, 10)))
+            backoff_seconds = backoff * 60
+        self.next_run_at = now + timedelta(seconds=max(0, backoff_seconds))
+        self.save(
+            update_fields=[
+                "last_failure_at",
+                "failure_count",
+                "next_run_at",
+                "updated_at",
+            ]
+        )
+
+
+class SourceRun(TimeStampedModel):
+    class Status(models.TextChoices):
+        RUNNING = "running", "Bezig"
+        SUCCESS = "success", "Geslaagd"
+        PARTIAL = "partial", "Gedeeltelijk"
+        FAILED = "failed", "Mislukt"
+        SKIPPED = "skipped", "Overgeslagen"
+
+    source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="runs")
+    status = models.CharField(max_length=16, choices=Status.choices, default=Status.RUNNING)
+    started_at = models.DateTimeField(default=timezone.now)
+    finished_at = models.DateTimeField(null=True, blank=True)
+    http_status = models.PositiveSmallIntegerField(null=True, blank=True)
+    discovered_count = models.PositiveIntegerField(default=0)
+    extracted_count = models.PositiveIntegerField(default=0)
+    created_count = models.PositiveIntegerField(default=0)
+    updated_count = models.PositiveIntegerField(default=0)
+    duplicate_count = models.PositiveIntegerField(default=0)
+    error_category = models.CharField(max_length=80, blank=True)
+    error_message = models.CharField(max_length=1000, blank=True)
+    metrics = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["-started_at"]
+
+    def finish(self, status: str, **metrics) -> None:
+        self.status = status
+        self.finished_at = timezone.now()
+        for key, value in metrics.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+        self.save()
+
+
+class SourceLease(TimeStampedModel):
+    source = models.OneToOneField(Source, on_delete=models.CASCADE, related_name="lease")
+    token = models.CharField(max_length=64, default=lambda: str(uuid4()))
+    worker_id = models.CharField(max_length=128, blank=True)
+    expires_at = models.DateTimeField(db_index=True)
+
+    class Meta:
+        ordering = ["source"]
+
+    def __str__(self) -> str:
+        return f"{self.source_id}:{self.token}"
+
+    @property
+    def is_expired(self) -> bool:
+        return self.expires_at <= timezone.now()
+
+
+class SourceOriginState(TimeStampedModel):
+    domain = models.CharField(max_length=255, unique=True)
+    next_allowed_at = models.DateTimeField()
+
+    class Meta:
+        ordering = ["domain"]
+
+    def __str__(self) -> str:
+        return self.domain
+
+
+class SourcePolicyReview(TimeStampedModel):
+    class Scope(models.TextChoices):
+        SOURCE = "source", "Bronspecifiek"
+        DOMAIN = "domain", "Domeinbrede controle"
+
+    class Decision(models.TextChoices):
+        ALLOW = "allow", "Toestaan"
+        TRIAL = "trial", "Proefrun"
+        PAUSE = "pause", "Pauzeren"
+        DENY = "deny", "Blokkeren"
+
+    source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="policy_reviews")
+    actor = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="policy_reviews",
+    )
+    scope = models.CharField(max_length=20, choices=Scope.choices, default=Scope.SOURCE)
+    decision = models.CharField(max_length=16, choices=Decision.choices)
+    reason = models.CharField(max_length=500)
+    evidence_link = models.URLField(max_length=500, blank=True)
+    notes = models.TextField(blank=True)
+    expires_at = models.DateTimeField(null=True, blank=True)
+    metadata = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["-created_at"]
+
+    @property
+    def is_expired(self) -> bool:
+        if not self.expires_at:
+            return False
+        return self.expires_at <= timezone.now()
+
+
+class SourceRobotsCache(TimeStampedModel):
+    origin = models.CharField(max_length=255, unique=True)
+    expires_at = models.DateTimeField()
+    etag = models.CharField(max_length=255, blank=True)
+    last_modified = models.CharField(max_length=255, blank=True)
+    allow_rules = models.JSONField(default=dict, blank=True)
+    disallow_rules = models.JSONField(default=dict, blank=True)
+    error = models.CharField(max_length=500, blank=True)
+    byte_length = models.PositiveIntegerField(default=0)
+
+    class Meta:
+        ordering = ["origin"]
+
+    @property
+    def is_fresh(self) -> bool:
+        return self.expires_at > timezone.now()
+
+
+class RawDocument(TimeStampedModel):
+    class Kind(models.TextChoices):
+        HTML = "html", "HTML"
+        XML = "xml", "XML"
+        EMAIL = "email", "E-mail"
+        JSON = "json", "JSON"
+        TEXT = "text", "Tekst"
+
+    source = models.ForeignKey(
+        Source, on_delete=models.SET_NULL, null=True, related_name="documents"
+    )
+    source_run = models.ForeignKey(
+        SourceRun, on_delete=models.SET_NULL, null=True, blank=True, related_name="documents"
+    )
+    url = models.URLField(max_length=2000, blank=True)
+    final_url = models.URLField(max_length=2000, blank=True)
+    kind = models.CharField(max_length=16, choices=Kind.choices)
+    content_type = models.CharField(max_length=200, blank=True)
+    http_status = models.PositiveSmallIntegerField(null=True, blank=True)
+    response_headers = models.JSONField(default=dict, blank=True)
+    content_hash = models.CharField(max_length=64, db_index=True)
+    body_text = models.TextField(blank=True)
+    byte_length = models.PositiveIntegerField(default=0)
+    parser_key = models.CharField(max_length=120, blank=True)
+    parser_version = models.CharField(max_length=80, blank=True)
+    extraction_confidence = models.DecimalField(max_digits=4, decimal_places=3, default=0)
+    retain_until = models.DateTimeField(null=True, blank=True)
+    quarantined = models.BooleanField(default=False)
+    quarantine_reason = models.CharField(max_length=500, blank=True)
+    metadata = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["-created_at"]
+        indexes = [models.Index(fields=["source", "content_hash"])]
+
+    def __str__(self) -> str:
+        return self.final_url or self.url or f"Document {self.pk}"
+
+
+class EmailMessageRecord(TimeStampedModel):
+    message_id = models.CharField(max_length=998, unique=True)
+    mailbox = models.CharField(max_length=255, default="INBOX")
+    sender = models.CharField(max_length=500, blank=True)
+    subject = models.CharField(max_length=998, blank=True)
+    received_at = models.DateTimeField(null=True, blank=True)
+    raw_document = models.OneToOneField(
+        RawDocument, on_delete=models.SET_NULL, null=True, blank=True, related_name="email_record"
+    )
+    links = models.JSONField(default=list, blank=True)
+    processed = models.BooleanField(default=False)
+    error_message = models.CharField(max_length=1000, blank=True)
+
+    class Meta:
+        ordering = ["-received_at", "-created_at"]
+
+    def __str__(self) -> str:
+        return self.subject or self.message_id