342 lines
12 KiB
Python
342 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
from decimal import Decimal
|
|
from urllib.parse import urlsplit
|
|
|
|
from django.db import IntegrityError, transaction
|
|
from django.utils import timezone
|
|
|
|
from apps.jobs.models import Employer, FieldProvenance, JobPosting, JobSourceAlias, JobVersion
|
|
from apps.profiles.models import SearchProfile
|
|
from apps.sources.adapters.registry import registry
|
|
from apps.sources.models import RawDocument, Source
|
|
from apps.sources.services.policy import is_denied_domain
|
|
|
|
from .dedupe import DedupeDecision, find_existing_job
|
|
from .features import extract_deterministic_features
|
|
from .normalization import CanonicalJobDraft, normalize_extracted_job, normalize_token
|
|
from .scoring import score_and_save
|
|
|
|
RECRUITER_TERMS = {
|
|
"recruitment",
|
|
"recruiter",
|
|
"staffing",
|
|
"interim",
|
|
"consultancy",
|
|
"consulting",
|
|
"talent",
|
|
}
|
|
|
|
|
|
def _confidence(value: float) -> Decimal:
|
|
return Decimal(str(max(0.0, min(1.0, value))))
|
|
|
|
|
|
def resolve_employer(draft: CanonicalJobDraft) -> Employer | None:
|
|
name = draft.employer_name.strip()
|
|
domain = draft.employer_domain.strip()
|
|
if not name and not domain:
|
|
return None
|
|
display_name = name or domain
|
|
normalized = normalize_token(display_name)
|
|
recruiter = any(term in normalized for term in RECRUITER_TERMS)
|
|
employer, _ = Employer.objects.get_or_create(
|
|
normalized_name=normalized,
|
|
domain=domain,
|
|
defaults={
|
|
"name": display_name,
|
|
"is_direct_employer": not recruiter,
|
|
"is_recruiter": recruiter,
|
|
"confidence": _confidence(0.75 if name else 0.45),
|
|
},
|
|
)
|
|
changed: list[str] = []
|
|
if name and employer.name != name and len(name) > len(employer.name):
|
|
employer.name = name
|
|
changed.append("name")
|
|
if recruiter and not employer.is_recruiter:
|
|
employer.is_recruiter = True
|
|
employer.is_direct_employer = False
|
|
changed.extend(["is_recruiter", "is_direct_employer"])
|
|
if changed:
|
|
employer.save(update_fields=[*changed, "updated_at"])
|
|
return employer
|
|
|
|
|
|
def job_snapshot(job: JobPosting) -> dict[str, object]:
|
|
return {
|
|
"id": str(job.id),
|
|
"title": job.original_title,
|
|
"normalized_title": job.normalized_title,
|
|
"employer": job.employer_name,
|
|
"canonical_url": job.canonical_url,
|
|
"location": job.raw_location,
|
|
"region": job.region,
|
|
"municipality": job.municipality,
|
|
"workplace_type": job.workplace_type,
|
|
"employment_types": job.employment_types,
|
|
"description_text": job.description_text,
|
|
"skills_required": job.skills_required,
|
|
"skills_preferred": job.skills_preferred,
|
|
"date_posted": job.date_posted.isoformat() if job.date_posted else None,
|
|
"valid_through": job.valid_through.isoformat() if job.valid_through else None,
|
|
"status": job.status,
|
|
"content_hash": job.content_hash,
|
|
}
|
|
|
|
|
|
def _source_is_direct(source: Source | None, draft: CanonicalJobDraft) -> bool:
|
|
host = (urlsplit(draft.canonical_url).hostname or "").lower()
|
|
if is_denied_domain(host):
|
|
return False
|
|
return bool(source and source.source_type == Source.Type.EMPLOYER)
|
|
|
|
|
|
def _alias_payload(
|
|
raw_payload: object,
|
|
decision: DedupeDecision,
|
|
*,
|
|
fallback_canonical_url: str | None,
|
|
) -> dict[str, object]:
|
|
if not isinstance(raw_payload, dict):
|
|
raw_payload = {}
|
|
payload: dict[str, object] = dict(raw_payload)
|
|
if decision.reason == "review_direct_conflict" or decision.resolved_direct:
|
|
payload["employer_resolution"] = {
|
|
"reason": decision.reason,
|
|
"confidence": float(decision.similarity),
|
|
"canonical_url": decision.canonical_url or fallback_canonical_url,
|
|
"resolved_direct": decision.resolved_direct,
|
|
"evidence": decision.evidence,
|
|
}
|
|
return payload
|
|
|
|
|
|
def _apply_draft(
|
|
job: JobPosting,
|
|
draft: CanonicalJobDraft,
|
|
employer: Employer | None,
|
|
*,
|
|
direct: bool,
|
|
resolved_canonical_url: str | None = None,
|
|
) -> list[str]:
|
|
canonical_url = resolved_canonical_url if direct else draft.canonical_url
|
|
fields = {
|
|
"employer": employer,
|
|
"original_title": draft.title,
|
|
"normalized_title": draft.normalized_title,
|
|
"job_family": draft.job_family,
|
|
"language": draft.language,
|
|
"description_html_sanitized": draft.description_html,
|
|
"description_text": draft.description_text,
|
|
"raw_location": draft.location_text,
|
|
"country": draft.country,
|
|
"region": draft.region,
|
|
"municipality": draft.municipality,
|
|
"postal_code": draft.postal_code,
|
|
"workplace_type": draft.workplace_type,
|
|
"employment_types": draft.employment_types,
|
|
"compensation": draft.compensation,
|
|
"skills_required": draft.skills_required,
|
|
"skills_preferred": draft.skills_preferred,
|
|
"date_posted": draft.date_posted,
|
|
"valid_through": draft.valid_through,
|
|
"content_hash": draft.content_hash,
|
|
"analysis_features": extract_deterministic_features(draft.title, draft.description_text),
|
|
"status": JobPosting.Status.ACTIVE,
|
|
"last_seen": timezone.now(),
|
|
"direct_employer": direct or (employer.is_direct_employer if employer else False),
|
|
"recruiter": employer.is_recruiter if employer else not direct,
|
|
}
|
|
changed: list[str] = []
|
|
for field, value in fields.items():
|
|
if (value not in (None, "", [], {}) or field in {"status", "last_seen"}) and (
|
|
getattr(job, field) != value
|
|
):
|
|
setattr(job, field, value)
|
|
changed.append(field)
|
|
if direct and canonical_url and job.canonical_url != canonical_url:
|
|
job.canonical_url = canonical_url
|
|
changed.append("canonical_url")
|
|
if changed and "last_changed" not in changed:
|
|
job.last_changed = timezone.now()
|
|
changed.append("last_changed")
|
|
return changed
|
|
|
|
|
|
@transaction.atomic
|
|
def persist_draft(
|
|
draft: CanonicalJobDraft,
|
|
*,
|
|
document: RawDocument,
|
|
parser_key: str,
|
|
parser_version: str,
|
|
extraction_confidence: float,
|
|
) -> tuple[JobPosting, DedupeDecision, bool]:
|
|
source = document.source
|
|
employer = resolve_employer(draft)
|
|
decision = find_existing_job(draft, source=source)
|
|
direct = _source_is_direct(source, draft)
|
|
if decision.resolved_direct:
|
|
direct = True
|
|
created = False
|
|
|
|
if decision.job is None:
|
|
try:
|
|
job = JobPosting.objects.create(
|
|
employer=employer,
|
|
original_title=draft.title,
|
|
normalized_title=draft.normalized_title,
|
|
job_family=draft.job_family,
|
|
language=draft.language,
|
|
canonical_url=draft.canonical_url,
|
|
canonical_key=draft.canonical_key,
|
|
content_hash=draft.content_hash,
|
|
description_html_sanitized=draft.description_html,
|
|
description_text=draft.description_text,
|
|
raw_location=draft.location_text,
|
|
country=draft.country,
|
|
region=draft.region,
|
|
municipality=draft.municipality,
|
|
postal_code=draft.postal_code,
|
|
workplace_type=draft.workplace_type,
|
|
employment_types=draft.employment_types,
|
|
compensation=draft.compensation,
|
|
skills_required=draft.skills_required,
|
|
skills_preferred=draft.skills_preferred,
|
|
date_posted=draft.date_posted,
|
|
valid_through=draft.valid_through,
|
|
direct_employer=direct or (employer.is_direct_employer if employer else False),
|
|
recruiter=employer.is_recruiter if employer else not direct,
|
|
extraction_confidence=_confidence(extraction_confidence),
|
|
analysis_features=extract_deterministic_features(
|
|
draft.title, draft.description_text
|
|
),
|
|
status=JobPosting.Status.ACTIVE,
|
|
)
|
|
created = True
|
|
except IntegrityError:
|
|
job = JobPosting.objects.get(canonical_key=draft.canonical_key)
|
|
decision = DedupeDecision(job, "canonical_key_race", 1.0)
|
|
else:
|
|
job = decision.job
|
|
if job.content_hash != draft.content_hash:
|
|
JobVersion.objects.get_or_create(
|
|
job=job,
|
|
content_hash=job.content_hash,
|
|
defaults={"snapshot": job_snapshot(job), "changed_fields": []},
|
|
)
|
|
changed = _apply_draft(
|
|
job,
|
|
draft,
|
|
employer,
|
|
direct=direct,
|
|
resolved_canonical_url=decision.canonical_url,
|
|
)
|
|
if extraction_confidence > float(job.extraction_confidence):
|
|
job.extraction_confidence = _confidence(extraction_confidence)
|
|
changed.append("extraction_confidence")
|
|
if changed:
|
|
job.save(update_fields=list(dict.fromkeys([*changed, "updated_at"])))
|
|
|
|
alias = (
|
|
JobSourceAlias.objects.filter(
|
|
job=job,
|
|
source=source,
|
|
canonical_url=draft.canonical_url,
|
|
external_id=draft.external_id,
|
|
)
|
|
.order_by("-last_seen")
|
|
.first()
|
|
)
|
|
if alias is None:
|
|
alias = JobSourceAlias.objects.create(
|
|
job=job,
|
|
source=source,
|
|
raw_document=document,
|
|
url=draft.source_url,
|
|
canonical_url=draft.canonical_url,
|
|
external_id=draft.external_id,
|
|
source_title=draft.title,
|
|
source_employer=draft.employer_name,
|
|
extraction_method=parser_key,
|
|
extraction_confidence=_confidence(extraction_confidence),
|
|
is_canonical=direct,
|
|
payload=_alias_payload(
|
|
draft.raw,
|
|
decision,
|
|
fallback_canonical_url=draft.canonical_url,
|
|
),
|
|
)
|
|
else:
|
|
alias.last_seen = timezone.now()
|
|
alias.raw_document = document
|
|
alias.payload = _alias_payload(alias.payload, decision, fallback_canonical_url=draft.canonical_url)
|
|
if direct:
|
|
alias.is_canonical = True
|
|
alias.save(
|
|
update_fields=["last_seen", "raw_document", "payload", "is_canonical", "updated_at"]
|
|
)
|
|
|
|
for evidence in draft.evidence:
|
|
FieldProvenance.objects.update_or_create(
|
|
job=job,
|
|
source_alias=alias,
|
|
field_name=evidence.field_name,
|
|
extraction_method=evidence.method,
|
|
defaults={
|
|
"confidence": _confidence(evidence.confidence),
|
|
"evidence_excerpt": evidence.evidence[:1000],
|
|
"parser_version": parser_version,
|
|
},
|
|
)
|
|
|
|
JobVersion.objects.get_or_create(
|
|
job=job,
|
|
content_hash=job.content_hash,
|
|
defaults={"snapshot": job_snapshot(job), "changed_fields": []},
|
|
)
|
|
for profile in SearchProfile.objects.filter(is_active=True):
|
|
score_and_save(job, profile)
|
|
return job, decision, created
|
|
|
|
|
|
@transaction.atomic
|
|
def process_raw_document(document: RawDocument) -> dict[str, int | str | list[str]]:
|
|
result = registry.extract(document)
|
|
document.parser_key = result.parser_key
|
|
document.parser_version = result.parser_version
|
|
document.extraction_confidence = _confidence(result.confidence)
|
|
document.save(
|
|
update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"]
|
|
)
|
|
created = updated = duplicates = 0
|
|
for extracted in result.jobs:
|
|
draft = normalize_extracted_job(extracted)
|
|
_, decision, was_created = persist_draft(
|
|
draft,
|
|
document=document,
|
|
parser_key=result.parser_key,
|
|
parser_version=result.parser_version,
|
|
extraction_confidence=result.confidence,
|
|
)
|
|
if was_created:
|
|
created += 1
|
|
elif (
|
|
decision.reason.startswith("exact")
|
|
or decision.reason.startswith("fuzzy")
|
|
or decision.reason == "resolved_direct_match"
|
|
):
|
|
duplicates += 1
|
|
updated += 1
|
|
else:
|
|
updated += 1
|
|
return {
|
|
"extracted": len(result.jobs),
|
|
"created": created,
|
|
"updated": updated,
|
|
"duplicates": duplicates,
|
|
"parser": result.parser_key,
|
|
"warnings": result.warnings,
|
|
}
|