111 lines
3.8 KiB
Python
111 lines
3.8 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from difflib import SequenceMatcher
|
|
|
|
from django.db.models import Q
|
|
|
|
from apps.jobs.models import JobPosting, JobSourceAlias
|
|
from apps.sources.models import Source
|
|
|
|
from .employer_resolution import EmployerResolutionDecision, resolve_direct_employer_match
|
|
from .normalization import CanonicalJobDraft, normalize_token
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DedupeDecision:
|
|
job: JobPosting | None
|
|
reason: str
|
|
similarity: float
|
|
canonical_url: str | None = None
|
|
resolved_direct: bool = False
|
|
evidence: list[str] = field(default_factory=list)
|
|
|
|
|
|
def text_similarity(left: str, right: str) -> float:
|
|
if not left or not right:
|
|
return 0.0
|
|
return SequenceMatcher(
|
|
None, normalize_token(left)[:12000], normalize_token(right)[:12000]
|
|
).ratio()
|
|
|
|
|
|
def candidate_similarity(job: JobPosting, draft: CanonicalJobDraft) -> float:
|
|
title = text_similarity(job.normalized_title, draft.normalized_title)
|
|
employer = (
|
|
text_similarity(job.employer_name, draft.employer_name) if draft.employer_name else 0.5
|
|
)
|
|
location = (
|
|
text_similarity(job.raw_location, draft.location_text) if draft.location_text else 0.5
|
|
)
|
|
description = (
|
|
text_similarity(job.description_text, draft.description_text)
|
|
if draft.description_text
|
|
else 0.5
|
|
)
|
|
return 0.38 * title + 0.24 * employer + 0.13 * location + 0.25 * description
|
|
|
|
|
|
def find_existing_job(
|
|
draft: CanonicalJobDraft, *, threshold: float = 0.92, source: Source | None = None
|
|
) -> DedupeDecision:
|
|
if draft.external_id:
|
|
alias = (
|
|
JobSourceAlias.objects.select_related("job")
|
|
.filter(external_id=draft.external_id)
|
|
.order_by("-last_seen")
|
|
.first()
|
|
)
|
|
if alias:
|
|
return DedupeDecision(alias.job, "exact_external_id", 1.0)
|
|
if draft.canonical_url:
|
|
alias = (
|
|
JobSourceAlias.objects.select_related("job")
|
|
.filter(canonical_url=draft.canonical_url)
|
|
.order_by("-last_seen")
|
|
.first()
|
|
)
|
|
if alias:
|
|
return DedupeDecision(alias.job, "exact_canonical_url", 1.0)
|
|
direct = JobPosting.objects.filter(canonical_key=draft.canonical_key).first()
|
|
if direct:
|
|
return DedupeDecision(direct, "exact_canonical_key", 1.0)
|
|
|
|
candidates = JobPosting.objects.filter(
|
|
status__in=[JobPosting.Status.ACTIVE, JobPosting.Status.NEW]
|
|
)
|
|
if draft.employer_name:
|
|
candidates = candidates.filter(
|
|
Q(employer__normalized_name=normalize_token(draft.employer_name))
|
|
| Q(normalized_title=draft.normalized_title)
|
|
)
|
|
else:
|
|
candidates = candidates.filter(normalized_title=draft.normalized_title)
|
|
if source is not None:
|
|
resolution: EmployerResolutionDecision = resolve_direct_employer_match(draft, source=source)
|
|
if resolution.job:
|
|
return DedupeDecision(
|
|
resolution.job,
|
|
resolution.reason,
|
|
resolution.confidence,
|
|
canonical_url=resolution.canonical_url,
|
|
resolved_direct=True,
|
|
evidence=resolution.evidence,
|
|
)
|
|
if resolution.conflict:
|
|
return DedupeDecision(
|
|
None,
|
|
resolution.reason,
|
|
resolution.confidence,
|
|
evidence=resolution.evidence,
|
|
)
|
|
best: JobPosting | None = None
|
|
best_score = 0.0
|
|
for candidate in candidates.select_related("employer")[:100]:
|
|
score = candidate_similarity(candidate, draft)
|
|
if score > best_score:
|
|
best, best_score = candidate, score
|
|
if best and best_score >= threshold:
|
|
return DedupeDecision(best, "fuzzy_strong", best_score)
|
|
return DedupeDecision(None, "new", best_score)
|