from __future__ import annotations from datetime import timedelta from urllib.parse import urlparse from django.conf import settings from django.core.exceptions import ValidationError from django.db import models from django.utils import timezone from apps.core.models import TimeStampedModel class Source(TimeStampedModel): class Type(models.TextChoices): EMPLOYER = "employer", "Werkgeverspagina" ATS = "ats", "Publieke ATS-pagina" RSS = "rss", "RSS/Atom" SITEMAP = "sitemap", "Sitemap" EMAIL = "email", "Vacaturemail" MANUAL = "manual", "Handmatige import" class Status(models.TextChoices): CANDIDATE = "candidate", "Kandidaat" TRIAL = "trial", "Proefrun" ACTIVE = "active", "Actief" QUARANTINED = "quarantined", "Quarantaine" PAUSED = "paused", "Gepauzeerd" DISABLED = "disabled", "Uitgeschakeld" class Policy(models.TextChoices): ALLOW = "allow", "Toestaan" REVIEW = "review", "Te beoordelen" DENY = "deny", "Blokkeren" name = models.CharField(max_length=200) source_type = models.CharField(max_length=24, choices=Type.choices) base_url = models.URLField(max_length=1000, blank=True) domain = models.CharField(max_length=255, db_index=True) status = models.CharField(max_length=24, choices=Status.choices, default=Status.CANDIDATE) policy = models.CharField(max_length=16, choices=Policy.choices, default=Policy.REVIEW) policy_reason = models.CharField(max_length=500, blank=True) parser_key = models.CharField(max_length=120, default="auto") strict_mode = models.BooleanField(default=True) allow_public_endpoint = models.BooleanField(default=False) honor_robots = models.BooleanField(default=True) crawl_interval_minutes = models.PositiveIntegerField(default=720) minimum_interval_seconds = models.PositiveIntegerField(default=30) max_concurrency = models.PositiveSmallIntegerField(default=1) next_run_at = models.DateTimeField(null=True, blank=True, db_index=True) last_success_at = models.DateTimeField(null=True, blank=True) last_failure_at = models.DateTimeField(null=True, blank=True) failure_count = models.PositiveIntegerField(default=0) etag = models.CharField(max_length=500, blank=True) last_modified = models.CharField(max_length=500, blank=True) robots_checked_at = models.DateTimeField(null=True, blank=True) terms_checked_at = models.DateTimeField(null=True, blank=True) metadata = models.JSONField(default=dict, blank=True) class Meta: ordering = ["name"] constraints = [ models.UniqueConstraint( fields=["source_type", "base_url"], condition=~models.Q(base_url=""), name="unique_source_type_base_url_nonempty", ) ] def __str__(self) -> str: return self.name def clean(self) -> None: if self.base_url: parsed = urlparse(self.base_url) if parsed.scheme not in {"http", "https"} or not parsed.hostname: raise ValidationError({"base_url": "Alleen geldige http(s)-URLs zijn toegestaan."}) if self.domain and parsed.hostname.lower().rstrip(".") != self.domain.lower().rstrip( "." ): raise ValidationError({"domain": "Domein moet overeenkomen met de base URL."}) if self.policy == self.Policy.DENY and self.status == self.Status.ACTIVE: raise ValidationError("Een geblokkeerde bron kan niet actief zijn.") @property def discovery_evidence(self) -> list[dict[str, str | float]]: raw = self.metadata if isinstance(self.metadata, dict) else {} entries = raw.get("discovery", []) if not isinstance(entries, list): return [] return [entry for entry in entries if isinstance(entry, dict)] @property def latest_policy_review(self) -> SourcePolicyReview | None: return self.policy_reviews.order_by("-created_at").first() @property def policy_review_state(self) -> str: review = self.latest_policy_review if not review: return "missing" if review.is_expired: return "expired" return "active" @property def policy_review_summary(self) -> str: review = self.latest_policy_review if not review: return "Geen review geregistreerd" if review.is_expired: return ( f"Review verlopen op {review.expires_at:%Y-%m-%d}" if review.expires_at else "Review verlopen" ) if review.expires_at: return f"{review.get_decision_display()} geldig tot {review.expires_at:%Y-%m-%d}" return f"{review.get_decision_display()} zonder vervaldatum" @property def requires_terms_review(self) -> bool: return self.policy == Source.Policy.ALLOW and self.policy_review_state in { "missing", "expired", } def schedule_after_success(self, *, now=None, jitter_seconds: int = 0) -> None: now = now or timezone.now() self.last_success_at = now self.failure_count = 0 self.next_run_at = ( now + timedelta(minutes=self.crawl_interval_minutes) + timedelta(seconds=max(0, jitter_seconds)) ) self.save(update_fields=["last_success_at", "failure_count", "next_run_at", "updated_at"]) def schedule_after_failure( self, *, now=None, backoff_minutes: int | None = None, backoff_seconds: int | None = None, ) -> None: now = now or timezone.now() self.last_failure_at = now self.failure_count += 1 if backoff_seconds is None: backoff = backoff_minutes or min(24 * 60, max(15, 2 ** min(self.failure_count, 10))) backoff_seconds = backoff * 60 self.next_run_at = now + timedelta(seconds=max(0, backoff_seconds)) self.save( update_fields=[ "last_failure_at", "failure_count", "next_run_at", "updated_at", ] ) class SourceRun(TimeStampedModel): class Status(models.TextChoices): RUNNING = "running", "Bezig" SUCCESS = "success", "Geslaagd" PARTIAL = "partial", "Gedeeltelijk" FAILED = "failed", "Mislukt" SKIPPED = "skipped", "Overgeslagen" source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="runs") status = models.CharField(max_length=16, choices=Status.choices, default=Status.RUNNING) started_at = models.DateTimeField(default=timezone.now) finished_at = models.DateTimeField(null=True, blank=True) http_status = models.PositiveSmallIntegerField(null=True, blank=True) discovered_count = models.PositiveIntegerField(default=0) extracted_count = models.PositiveIntegerField(default=0) created_count = models.PositiveIntegerField(default=0) updated_count = models.PositiveIntegerField(default=0) duplicate_count = models.PositiveIntegerField(default=0) error_category = models.CharField(max_length=80, blank=True) error_message = models.CharField(max_length=1000, blank=True) metrics = models.JSONField(default=dict, blank=True) class Meta: ordering = ["-started_at"] def finish(self, status: str, **metrics) -> None: self.status = status self.finished_at = timezone.now() for key, value in metrics.items(): if hasattr(self, key): setattr(self, key, value) self.save() class SourceLease(TimeStampedModel): source = models.OneToOneField(Source, on_delete=models.CASCADE, related_name="lease") token = models.CharField(max_length=64, default="") worker_id = models.CharField(max_length=128, blank=True) expires_at = models.DateTimeField(db_index=True) class Meta: ordering = ["source"] def __str__(self) -> str: return f"{self.source_id}:{self.token}" @property def is_expired(self) -> bool: return self.expires_at <= timezone.now() class SourceOriginState(TimeStampedModel): domain = models.CharField(max_length=255, unique=True) next_allowed_at = models.DateTimeField() class Meta: ordering = ["domain"] def __str__(self) -> str: return self.domain class SourcePolicyReview(TimeStampedModel): class Scope(models.TextChoices): SOURCE = "source", "Bronspecifiek" DOMAIN = "domain", "Domeinbrede controle" class Decision(models.TextChoices): ALLOW = "allow", "Toestaan" TRIAL = "trial", "Proefrun" PAUSE = "pause", "Pauzeren" DENY = "deny", "Blokkeren" source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="policy_reviews") actor = models.ForeignKey( settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, blank=True, related_name="policy_reviews", ) scope = models.CharField(max_length=20, choices=Scope.choices, default=Scope.SOURCE) decision = models.CharField(max_length=16, choices=Decision.choices) reason = models.CharField(max_length=500) evidence_link = models.URLField(max_length=500, blank=True) notes = models.TextField(blank=True) expires_at = models.DateTimeField(null=True, blank=True) metadata = models.JSONField(default=dict, blank=True) class Meta: ordering = ["-created_at"] @property def is_expired(self) -> bool: if not self.expires_at: return False return self.expires_at <= timezone.now() class SourceRobotsCache(TimeStampedModel): origin = models.CharField(max_length=255, unique=True) expires_at = models.DateTimeField() etag = models.CharField(max_length=255, blank=True) last_modified = models.CharField(max_length=255, blank=True) allow_rules = models.JSONField(default=dict, blank=True) disallow_rules = models.JSONField(default=dict, blank=True) error = models.CharField(max_length=500, blank=True) byte_length = models.PositiveIntegerField(default=0) class Meta: ordering = ["origin"] @property def is_fresh(self) -> bool: return self.expires_at > timezone.now() class RawDocument(TimeStampedModel): class Kind(models.TextChoices): HTML = "html", "HTML" XML = "xml", "XML" EMAIL = "email", "E-mail" JSON = "json", "JSON" TEXT = "text", "Tekst" source = models.ForeignKey( Source, on_delete=models.SET_NULL, null=True, related_name="documents" ) source_run = models.ForeignKey( SourceRun, on_delete=models.SET_NULL, null=True, blank=True, related_name="documents" ) url = models.URLField(max_length=2000, blank=True) final_url = models.URLField(max_length=2000, blank=True) kind = models.CharField(max_length=16, choices=Kind.choices) content_type = models.CharField(max_length=200, blank=True) http_status = models.PositiveSmallIntegerField(null=True, blank=True) response_headers = models.JSONField(default=dict, blank=True) content_hash = models.CharField(max_length=64, db_index=True) body_text = models.TextField(blank=True) byte_length = models.PositiveIntegerField(default=0) parser_key = models.CharField(max_length=120, blank=True) parser_version = models.CharField(max_length=80, blank=True) extraction_confidence = models.DecimalField(max_digits=4, decimal_places=3, default=0) retain_until = models.DateTimeField(null=True, blank=True) quarantined = models.BooleanField(default=False) quarantine_reason = models.CharField(max_length=500, blank=True) metadata = models.JSONField(default=dict, blank=True) class Meta: ordering = ["-created_at"] indexes = [models.Index(fields=["source", "content_hash"])] def __str__(self) -> str: return self.final_url or self.url or f"Document {self.pk}" class MailboxConnection(TimeStampedModel): class Platform(models.TextChoices): VDAB = "vdab", "VDAB" INDEED = "indeed", "Indeed" LINKEDIN = "linkedin", "LinkedIn" ICTJOB = "ictjob", "ictjob.be" JOBAT = "jobat", "Jobat" STEPSTONE = "stepstone", "StepStone" CAREERJET = "careerjet", "Careerjet" RANDSTAD = "randstad", "Randstad" ROBERTHALF = "roberthalf", "Robert Half" class Provider(models.TextChoices): GMAIL = "gmail", "Gmail" OUTLOOK = "outlook", "Outlook / Microsoft 365" CUSTOM = "custom", "Andere IMAP-provider" class PollInterval(models.IntegerChoices): FIVE_MINUTES = 5, "Elke 5 minuten" FIFTEEN_MINUTES = 15, "Elke 15 minuten" THIRTY_MINUTES = 30, "Elke 30 minuten" HOURLY = 60, "Elk uur" THREE_HOURS = 180, "Elke 3 uur" SIX_HOURS = 360, "Elke 6 uur" PROVIDER_HOSTS = { Provider.GMAIL: "imap.gmail.com", Provider.OUTLOOK: "outlook.office365.com", } user = models.ForeignKey( settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="mailbox_connections", ) platform = models.CharField(max_length=24, choices=Platform.choices) provider = models.CharField(max_length=24, choices=Provider.choices) custom_host = models.CharField(max_length=255, blank=True) port = models.PositiveIntegerField(default=993) username = models.CharField(max_length=320) encrypted_password = models.TextField(blank=True) mailbox = models.CharField(max_length=255, default="INBOX") enabled = models.BooleanField(default=True) poll_interval_minutes = models.PositiveIntegerField( choices=PollInterval.choices, default=PollInterval.THIRTY_MINUTES, ) next_poll_at = models.DateTimeField(default=timezone.now, db_index=True) last_polled_at = models.DateTimeField(null=True, blank=True) last_success_at = models.DateTimeField(null=True, blank=True) last_error_category = models.CharField(max_length=80, blank=True) last_error_message = models.CharField(max_length=500, blank=True) lease_token = models.CharField(max_length=64, blank=True) lease_expires_at = models.DateTimeField(null=True, blank=True, db_index=True) class Meta: ordering = ["platform"] constraints = [ models.UniqueConstraint( fields=["user", "platform"], name="unique_mailbox_platform_per_user" ) ] def __str__(self) -> str: return f"{self.get_platform_display()} ยท {self.username}" @property def imap_host(self) -> str: return self.PROVIDER_HOSTS.get(self.provider, self.custom_host).strip().lower() def clean(self) -> None: host = self.imap_host if ( not host or "://" in host or "/" in host or "@" in host or any(c.isspace() for c in host) ): raise ValidationError({"custom_host": "Vul een geldige IMAP-hostnaam in."}) if not 1 <= self.port <= 65535: raise ValidationError({"port": "De IMAP-poort moet tussen 1 en 65535 liggen."}) if "\n" in self.mailbox or "\r" in self.mailbox: raise ValidationError({"mailbox": "De mailboxnaam bevat ongeldige tekens."}) class EmailMessageRecord(TimeStampedModel): mailbox_connection = models.ForeignKey( MailboxConnection, on_delete=models.CASCADE, null=True, blank=True, related_name="messages", ) message_id = models.CharField(max_length=998) mailbox = models.CharField(max_length=255, default="INBOX") sender = models.CharField(max_length=500, blank=True) subject = models.CharField(max_length=998, blank=True) received_at = models.DateTimeField(null=True, blank=True) raw_document = models.OneToOneField( RawDocument, on_delete=models.SET_NULL, null=True, blank=True, related_name="email_record" ) links = models.JSONField(default=list, blank=True) processed = models.BooleanField(default=False) error_message = models.CharField(max_length=1000, blank=True) class Meta: ordering = ["-received_at", "-created_at"] constraints = [ models.UniqueConstraint( fields=["mailbox_connection", "message_id"], condition=models.Q(mailbox_connection__isnull=False), name="unique_message_per_mailbox_connection", ), models.UniqueConstraint( fields=["message_id"], condition=models.Q(mailbox_connection__isnull=True), name="unique_legacy_email_message_id", ), ] def __str__(self) -> str: return self.subject or self.message_id