from __future__ import annotations from dataclasses import dataclass, field from difflib import SequenceMatcher from apps.sources.models import Source from django.db.models import Q from apps.jobs.models import JobPosting, JobSourceAlias from .employer_resolution import EmployerResolutionDecision, resolve_direct_employer_match from .normalization import CanonicalJobDraft, normalize_token @dataclass(frozen=True) class DedupeDecision: job: JobPosting | None reason: str similarity: float canonical_url: str | None = None resolved_direct: bool = False evidence: list[str] = field(default_factory=list) def text_similarity(left: str, right: str) -> float: if not left or not right: return 0.0 return SequenceMatcher( None, normalize_token(left)[:12000], normalize_token(right)[:12000] ).ratio() def candidate_similarity(job: JobPosting, draft: CanonicalJobDraft) -> float: title = text_similarity(job.normalized_title, draft.normalized_title) employer = ( text_similarity(job.employer_name, draft.employer_name) if draft.employer_name else 0.5 ) location = ( text_similarity(job.raw_location, draft.location_text) if draft.location_text else 0.5 ) description = ( text_similarity(job.description_text, draft.description_text) if draft.description_text else 0.5 ) return 0.38 * title + 0.24 * employer + 0.13 * location + 0.25 * description def find_existing_job( draft: CanonicalJobDraft, *, threshold: float = 0.92, source: Source | None = None ) -> DedupeDecision: if draft.external_id: alias = ( JobSourceAlias.objects.select_related("job") .filter(external_id=draft.external_id) .order_by("-last_seen") .first() ) if alias: return DedupeDecision(alias.job, "exact_external_id", 1.0) if draft.canonical_url: alias = ( JobSourceAlias.objects.select_related("job") .filter(canonical_url=draft.canonical_url) .order_by("-last_seen") .first() ) if alias: return DedupeDecision(alias.job, "exact_canonical_url", 1.0) direct = JobPosting.objects.filter(canonical_key=draft.canonical_key).first() if direct: return DedupeDecision(direct, "exact_canonical_key", 1.0) candidates = JobPosting.objects.filter( status__in=[JobPosting.Status.ACTIVE, JobPosting.Status.NEW] ) if draft.employer_name: candidates = candidates.filter( Q(employer__normalized_name=normalize_token(draft.employer_name)) | Q(normalized_title=draft.normalized_title) ) else: candidates = candidates.filter(normalized_title=draft.normalized_title) if source is not None: resolution: EmployerResolutionDecision = resolve_direct_employer_match(draft, source=source) if resolution.job: return DedupeDecision( resolution.job, resolution.reason, resolution.confidence, canonical_url=resolution.canonical_url, resolved_direct=True, evidence=resolution.evidence, ) if resolution.conflict: return DedupeDecision( None, resolution.reason, resolution.confidence, evidence=resolution.evidence, ) best: JobPosting | None = None best_score = 0.0 for candidate in candidates.select_related("employer")[:100]: score = candidate_similarity(candidate, draft) if score > best_score: best, best_score = candidate, score if best and best_score >= threshold: return DedupeDecision(best, "fuzzy_strong", best_score) return DedupeDecision(None, "new", best_score)