Initial deploy setup

2026-07-21 14:00:00 +02:00
commit b8091e59bd
285 changed files with 27854 additions and 0 deletions
@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from difflib import SequenceMatcher
+
+from apps.sources.models import Source
+from django.db.models import Q
+
+from apps.jobs.models import JobPosting, JobSourceAlias
+
+from .employer_resolution import EmployerResolutionDecision, resolve_direct_employer_match
+from .normalization import CanonicalJobDraft, normalize_token
+
+
+@dataclass(frozen=True)
+class DedupeDecision:
+    job: JobPosting | None
+    reason: str
+    similarity: float
+    canonical_url: str | None = None
+    resolved_direct: bool = False
+    evidence: list[str] = field(default_factory=list)
+
+
+def text_similarity(left: str, right: str) -> float:
+    if not left or not right:
+        return 0.0
+    return SequenceMatcher(
+        None, normalize_token(left)[:12000], normalize_token(right)[:12000]
+    ).ratio()
+
+
+def candidate_similarity(job: JobPosting, draft: CanonicalJobDraft) -> float:
+    title = text_similarity(job.normalized_title, draft.normalized_title)
+    employer = (
+        text_similarity(job.employer_name, draft.employer_name) if draft.employer_name else 0.5
+    )
+    location = (
+        text_similarity(job.raw_location, draft.location_text) if draft.location_text else 0.5
+    )
+    description = (
+        text_similarity(job.description_text, draft.description_text)
+        if draft.description_text
+        else 0.5
+    )
+    return 0.38 * title + 0.24 * employer + 0.13 * location + 0.25 * description
+
+
+def find_existing_job(
+    draft: CanonicalJobDraft, *, threshold: float = 0.92, source: Source | None = None
+) -> DedupeDecision:
+    if draft.external_id:
+        alias = (
+            JobSourceAlias.objects.select_related("job")
+            .filter(external_id=draft.external_id)
+            .order_by("-last_seen")
+            .first()
+        )
+        if alias:
+            return DedupeDecision(alias.job, "exact_external_id", 1.0)
+    if draft.canonical_url:
+        alias = (
+            JobSourceAlias.objects.select_related("job")
+            .filter(canonical_url=draft.canonical_url)
+            .order_by("-last_seen")
+            .first()
+        )
+        if alias:
+            return DedupeDecision(alias.job, "exact_canonical_url", 1.0)
+    direct = JobPosting.objects.filter(canonical_key=draft.canonical_key).first()
+    if direct:
+        return DedupeDecision(direct, "exact_canonical_key", 1.0)
+
+    candidates = JobPosting.objects.filter(
+        status__in=[JobPosting.Status.ACTIVE, JobPosting.Status.NEW]
+    )
+    if draft.employer_name:
+        candidates = candidates.filter(
+            Q(employer__normalized_name=normalize_token(draft.employer_name))
+            | Q(normalized_title=draft.normalized_title)
+        )
+    else:
+        candidates = candidates.filter(normalized_title=draft.normalized_title)
+    if source is not None:
+        resolution: EmployerResolutionDecision = resolve_direct_employer_match(draft, source=source)
+        if resolution.job:
+            return DedupeDecision(
+                resolution.job,
+                resolution.reason,
+                resolution.confidence,
+                canonical_url=resolution.canonical_url,
+                resolved_direct=True,
+                evidence=resolution.evidence,
+            )
+        if resolution.conflict:
+            return DedupeDecision(
+                None,
+                resolution.reason,
+                resolution.confidence,
+                evidence=resolution.evidence,
+            )
+    best: JobPosting | None = None
+    best_score = 0.0
+    for candidate in candidates.select_related("employer")[:100]:
+        score = candidate_similarity(candidate, draft)
+        if score > best_score:
+            best, best_score = candidate, score
+    if best and best_score >= threshold:
+        return DedupeDecision(best, "fuzzy_strong", best_score)
+    return DedupeDecision(None, "new", best_score)