Jens
@@ -0,0 +1,341 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from django.db import IntegrityError, transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.jobs.models import Employer, FieldProvenance, JobPosting, JobSourceAlias, JobVersion
|
||||
from apps.profiles.models import SearchProfile
|
||||
from apps.sources.adapters.registry import registry
|
||||
from apps.sources.models import RawDocument, Source
|
||||
from apps.sources.services.policy import is_denied_domain
|
||||
|
||||
from .dedupe import DedupeDecision, find_existing_job
|
||||
from .features import extract_deterministic_features
|
||||
from .normalization import CanonicalJobDraft, normalize_extracted_job, normalize_token
|
||||
from .scoring import score_and_save
|
||||
|
||||
RECRUITER_TERMS = {
|
||||
"recruitment",
|
||||
"recruiter",
|
||||
"staffing",
|
||||
"interim",
|
||||
"consultancy",
|
||||
"consulting",
|
||||
"talent",
|
||||
}
|
||||
|
||||
|
||||
def _confidence(value: float) -> Decimal:
|
||||
return Decimal(str(max(0.0, min(1.0, value))))
|
||||
|
||||
|
||||
def resolve_employer(draft: CanonicalJobDraft) -> Employer | None:
|
||||
name = draft.employer_name.strip()
|
||||
domain = draft.employer_domain.strip()
|
||||
if not name and not domain:
|
||||
return None
|
||||
display_name = name or domain
|
||||
normalized = normalize_token(display_name)
|
||||
recruiter = any(term in normalized for term in RECRUITER_TERMS)
|
||||
employer, _ = Employer.objects.get_or_create(
|
||||
normalized_name=normalized,
|
||||
domain=domain,
|
||||
defaults={
|
||||
"name": display_name,
|
||||
"is_direct_employer": not recruiter,
|
||||
"is_recruiter": recruiter,
|
||||
"confidence": _confidence(0.75 if name else 0.45),
|
||||
},
|
||||
)
|
||||
changed: list[str] = []
|
||||
if name and employer.name != name and len(name) > len(employer.name):
|
||||
employer.name = name
|
||||
changed.append("name")
|
||||
if recruiter and not employer.is_recruiter:
|
||||
employer.is_recruiter = True
|
||||
employer.is_direct_employer = False
|
||||
changed.extend(["is_recruiter", "is_direct_employer"])
|
||||
if changed:
|
||||
employer.save(update_fields=[*changed, "updated_at"])
|
||||
return employer
|
||||
|
||||
|
||||
def job_snapshot(job: JobPosting) -> dict[str, object]:
|
||||
return {
|
||||
"id": str(job.id),
|
||||
"title": job.original_title,
|
||||
"normalized_title": job.normalized_title,
|
||||
"employer": job.employer_name,
|
||||
"canonical_url": job.canonical_url,
|
||||
"location": job.raw_location,
|
||||
"region": job.region,
|
||||
"municipality": job.municipality,
|
||||
"workplace_type": job.workplace_type,
|
||||
"employment_types": job.employment_types,
|
||||
"description_text": job.description_text,
|
||||
"skills_required": job.skills_required,
|
||||
"skills_preferred": job.skills_preferred,
|
||||
"date_posted": job.date_posted.isoformat() if job.date_posted else None,
|
||||
"valid_through": job.valid_through.isoformat() if job.valid_through else None,
|
||||
"status": job.status,
|
||||
"content_hash": job.content_hash,
|
||||
}
|
||||
|
||||
|
||||
def _source_is_direct(source: Source | None, draft: CanonicalJobDraft) -> bool:
|
||||
host = (urlsplit(draft.canonical_url).hostname or "").lower()
|
||||
if is_denied_domain(host):
|
||||
return False
|
||||
return bool(source and source.source_type == Source.Type.EMPLOYER)
|
||||
|
||||
|
||||
def _alias_payload(
|
||||
raw_payload: object,
|
||||
decision: DedupeDecision,
|
||||
*,
|
||||
fallback_canonical_url: str | None,
|
||||
) -> dict[str, object]:
|
||||
if not isinstance(raw_payload, dict):
|
||||
raw_payload = {}
|
||||
payload: dict[str, object] = dict(raw_payload)
|
||||
if decision.reason == "review_direct_conflict" or decision.resolved_direct:
|
||||
payload["employer_resolution"] = {
|
||||
"reason": decision.reason,
|
||||
"confidence": float(decision.similarity),
|
||||
"canonical_url": decision.canonical_url or fallback_canonical_url,
|
||||
"resolved_direct": decision.resolved_direct,
|
||||
"evidence": decision.evidence,
|
||||
}
|
||||
return payload
|
||||
|
||||
|
||||
def _apply_draft(
|
||||
job: JobPosting,
|
||||
draft: CanonicalJobDraft,
|
||||
employer: Employer | None,
|
||||
*,
|
||||
direct: bool,
|
||||
resolved_canonical_url: str | None = None,
|
||||
) -> list[str]:
|
||||
canonical_url = resolved_canonical_url if direct else draft.canonical_url
|
||||
fields = {
|
||||
"employer": employer,
|
||||
"original_title": draft.title,
|
||||
"normalized_title": draft.normalized_title,
|
||||
"job_family": draft.job_family,
|
||||
"language": draft.language,
|
||||
"description_html_sanitized": draft.description_html,
|
||||
"description_text": draft.description_text,
|
||||
"raw_location": draft.location_text,
|
||||
"country": draft.country,
|
||||
"region": draft.region,
|
||||
"municipality": draft.municipality,
|
||||
"postal_code": draft.postal_code,
|
||||
"workplace_type": draft.workplace_type,
|
||||
"employment_types": draft.employment_types,
|
||||
"compensation": draft.compensation,
|
||||
"skills_required": draft.skills_required,
|
||||
"skills_preferred": draft.skills_preferred,
|
||||
"date_posted": draft.date_posted,
|
||||
"valid_through": draft.valid_through,
|
||||
"content_hash": draft.content_hash,
|
||||
"analysis_features": extract_deterministic_features(draft.title, draft.description_text),
|
||||
"status": JobPosting.Status.ACTIVE,
|
||||
"last_seen": timezone.now(),
|
||||
"direct_employer": direct or (employer.is_direct_employer if employer else False),
|
||||
"recruiter": employer.is_recruiter if employer else not direct,
|
||||
}
|
||||
changed: list[str] = []
|
||||
for field, value in fields.items():
|
||||
if (value not in (None, "", [], {}) or field in {"status", "last_seen"}) and (
|
||||
getattr(job, field) != value
|
||||
):
|
||||
setattr(job, field, value)
|
||||
changed.append(field)
|
||||
if direct and canonical_url and job.canonical_url != canonical_url:
|
||||
job.canonical_url = canonical_url
|
||||
changed.append("canonical_url")
|
||||
if changed and "last_changed" not in changed:
|
||||
job.last_changed = timezone.now()
|
||||
changed.append("last_changed")
|
||||
return changed
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def persist_draft(
|
||||
draft: CanonicalJobDraft,
|
||||
*,
|
||||
document: RawDocument,
|
||||
parser_key: str,
|
||||
parser_version: str,
|
||||
extraction_confidence: float,
|
||||
) -> tuple[JobPosting, DedupeDecision, bool]:
|
||||
source = document.source
|
||||
employer = resolve_employer(draft)
|
||||
decision = find_existing_job(draft, source=source)
|
||||
direct = _source_is_direct(source, draft)
|
||||
if decision.resolved_direct:
|
||||
direct = True
|
||||
created = False
|
||||
|
||||
if decision.job is None:
|
||||
try:
|
||||
job = JobPosting.objects.create(
|
||||
employer=employer,
|
||||
original_title=draft.title,
|
||||
normalized_title=draft.normalized_title,
|
||||
job_family=draft.job_family,
|
||||
language=draft.language,
|
||||
canonical_url=draft.canonical_url,
|
||||
canonical_key=draft.canonical_key,
|
||||
content_hash=draft.content_hash,
|
||||
description_html_sanitized=draft.description_html,
|
||||
description_text=draft.description_text,
|
||||
raw_location=draft.location_text,
|
||||
country=draft.country,
|
||||
region=draft.region,
|
||||
municipality=draft.municipality,
|
||||
postal_code=draft.postal_code,
|
||||
workplace_type=draft.workplace_type,
|
||||
employment_types=draft.employment_types,
|
||||
compensation=draft.compensation,
|
||||
skills_required=draft.skills_required,
|
||||
skills_preferred=draft.skills_preferred,
|
||||
date_posted=draft.date_posted,
|
||||
valid_through=draft.valid_through,
|
||||
direct_employer=direct or (employer.is_direct_employer if employer else False),
|
||||
recruiter=employer.is_recruiter if employer else not direct,
|
||||
extraction_confidence=_confidence(extraction_confidence),
|
||||
analysis_features=extract_deterministic_features(
|
||||
draft.title, draft.description_text
|
||||
),
|
||||
status=JobPosting.Status.ACTIVE,
|
||||
)
|
||||
created = True
|
||||
except IntegrityError:
|
||||
job = JobPosting.objects.get(canonical_key=draft.canonical_key)
|
||||
decision = DedupeDecision(job, "canonical_key_race", 1.0)
|
||||
else:
|
||||
job = decision.job
|
||||
if job.content_hash != draft.content_hash:
|
||||
JobVersion.objects.get_or_create(
|
||||
job=job,
|
||||
content_hash=job.content_hash,
|
||||
defaults={"snapshot": job_snapshot(job), "changed_fields": []},
|
||||
)
|
||||
changed = _apply_draft(
|
||||
job,
|
||||
draft,
|
||||
employer,
|
||||
direct=direct,
|
||||
resolved_canonical_url=decision.canonical_url,
|
||||
)
|
||||
if extraction_confidence > float(job.extraction_confidence):
|
||||
job.extraction_confidence = _confidence(extraction_confidence)
|
||||
changed.append("extraction_confidence")
|
||||
if changed:
|
||||
job.save(update_fields=list(dict.fromkeys([*changed, "updated_at"])))
|
||||
|
||||
alias = (
|
||||
JobSourceAlias.objects.filter(
|
||||
job=job,
|
||||
source=source,
|
||||
canonical_url=draft.canonical_url,
|
||||
external_id=draft.external_id,
|
||||
)
|
||||
.order_by("-last_seen")
|
||||
.first()
|
||||
)
|
||||
if alias is None:
|
||||
alias = JobSourceAlias.objects.create(
|
||||
job=job,
|
||||
source=source,
|
||||
raw_document=document,
|
||||
url=draft.source_url,
|
||||
canonical_url=draft.canonical_url,
|
||||
external_id=draft.external_id,
|
||||
source_title=draft.title,
|
||||
source_employer=draft.employer_name,
|
||||
extraction_method=parser_key,
|
||||
extraction_confidence=_confidence(extraction_confidence),
|
||||
is_canonical=direct,
|
||||
payload=_alias_payload(
|
||||
draft.raw,
|
||||
decision,
|
||||
fallback_canonical_url=draft.canonical_url,
|
||||
),
|
||||
)
|
||||
else:
|
||||
alias.last_seen = timezone.now()
|
||||
alias.raw_document = document
|
||||
alias.payload = _alias_payload(alias.payload, decision, fallback_canonical_url=draft.canonical_url)
|
||||
if direct:
|
||||
alias.is_canonical = True
|
||||
alias.save(
|
||||
update_fields=["last_seen", "raw_document", "payload", "is_canonical", "updated_at"]
|
||||
)
|
||||
|
||||
for evidence in draft.evidence:
|
||||
FieldProvenance.objects.update_or_create(
|
||||
job=job,
|
||||
source_alias=alias,
|
||||
field_name=evidence.field_name,
|
||||
extraction_method=evidence.method,
|
||||
defaults={
|
||||
"confidence": _confidence(evidence.confidence),
|
||||
"evidence_excerpt": evidence.evidence[:1000],
|
||||
"parser_version": parser_version,
|
||||
},
|
||||
)
|
||||
|
||||
JobVersion.objects.get_or_create(
|
||||
job=job,
|
||||
content_hash=job.content_hash,
|
||||
defaults={"snapshot": job_snapshot(job), "changed_fields": []},
|
||||
)
|
||||
for profile in SearchProfile.objects.filter(is_active=True):
|
||||
score_and_save(job, profile)
|
||||
return job, decision, created
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def process_raw_document(document: RawDocument) -> dict[str, int | str | list[str]]:
|
||||
result = registry.extract(document)
|
||||
document.parser_key = result.parser_key
|
||||
document.parser_version = result.parser_version
|
||||
document.extraction_confidence = _confidence(result.confidence)
|
||||
document.save(
|
||||
update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"]
|
||||
)
|
||||
created = updated = duplicates = 0
|
||||
for extracted in result.jobs:
|
||||
draft = normalize_extracted_job(extracted)
|
||||
_, decision, was_created = persist_draft(
|
||||
draft,
|
||||
document=document,
|
||||
parser_key=result.parser_key,
|
||||
parser_version=result.parser_version,
|
||||
extraction_confidence=result.confidence,
|
||||
)
|
||||
if was_created:
|
||||
created += 1
|
||||
elif (
|
||||
decision.reason.startswith("exact")
|
||||
or decision.reason.startswith("fuzzy")
|
||||
or decision.reason == "resolved_direct_match"
|
||||
):
|
||||
duplicates += 1
|
||||
updated += 1
|
||||
else:
|
||||
updated += 1
|
||||
return {
|
||||
"extracted": len(result.jobs),
|
||||
"created": created,
|
||||
"updated": updated,
|
||||
"duplicates": duplicates,
|
||||
"parser": result.parser_key,
|
||||
"warnings": result.warnings,
|
||||
}
|
||||
Reference in New Issue
Block a user