from __future__ import annotations from decimal import Decimal from urllib.parse import urlsplit from django.db import IntegrityError, transaction from django.utils import timezone from apps.jobs.models import Employer, FieldProvenance, JobPosting, JobSourceAlias, JobVersion from apps.profiles.models import SearchProfile from apps.sources.adapters.base import FieldEvidence from apps.sources.adapters.registry import registry from apps.sources.models import RawDocument, Source from apps.sources.services.policy import is_denied_domain from .dedupe import DedupeDecision, find_existing_job from .features import extract_deterministic_features from .normalization import CanonicalJobDraft, normalize_extracted_job, normalize_token from .scoring import score_and_save RECRUITER_TERMS = { "recruitment", "recruiter", "staffing", "interim", "consultancy", "consulting", "talent", } def _confidence(value: float) -> Decimal: return Decimal(str(max(0.0, min(1.0, value)))) def resolve_employer(draft: CanonicalJobDraft) -> Employer | None: name = draft.employer_name.strip() domain = draft.employer_domain.strip() if not name and not domain: return None display_name = name or domain normalized = normalize_token(display_name) recruiter = any(term in normalized for term in RECRUITER_TERMS) employer, _ = Employer.objects.get_or_create( normalized_name=normalized, domain=domain, defaults={ "name": display_name, "is_direct_employer": not recruiter, "is_recruiter": recruiter, "confidence": _confidence(0.75 if name else 0.45), }, ) changed: list[str] = [] if name and employer.name != name and len(name) > len(employer.name): employer.name = name changed.append("name") if recruiter and not employer.is_recruiter: employer.is_recruiter = True employer.is_direct_employer = False changed.extend(["is_recruiter", "is_direct_employer"]) if changed: employer.save(update_fields=[*changed, "updated_at"]) return employer def job_snapshot(job: JobPosting) -> dict[str, object]: return { "id": str(job.id), "title": job.original_title, "normalized_title": job.normalized_title, "employer": job.employer_name, "canonical_url": job.canonical_url, "location": job.raw_location, "region": job.region, "municipality": job.municipality, "workplace_type": job.workplace_type, "employment_types": job.employment_types, "description_text": job.description_text, "skills_required": job.skills_required, "skills_preferred": job.skills_preferred, "date_posted": job.date_posted.isoformat() if job.date_posted else None, "valid_through": job.valid_through.isoformat() if job.valid_through else None, "status": job.status, "content_hash": job.content_hash, } def _source_is_direct(source: Source | None, draft: CanonicalJobDraft) -> bool: host = (urlsplit(draft.canonical_url).hostname or "").lower() if is_denied_domain(host): return False if not source: return False metadata = source.metadata if isinstance(source.metadata, dict) else {} return source.source_type == Source.Type.EMPLOYER or metadata.get("direct_employer") is True def _alias_payload( raw_payload: object, decision: DedupeDecision, *, fallback_canonical_url: str | None, ) -> dict[str, object]: if not isinstance(raw_payload, dict): raw_payload = {} payload: dict[str, object] = dict(raw_payload) if decision.reason == "review_direct_conflict" or decision.resolved_direct: payload["employer_resolution"] = { "reason": decision.reason, "confidence": float(decision.similarity), "canonical_url": decision.canonical_url or fallback_canonical_url, "resolved_direct": decision.resolved_direct, "evidence": decision.evidence, } return payload def _apply_draft( job: JobPosting, draft: CanonicalJobDraft, employer: Employer | None, *, direct: bool, resolved_canonical_url: str | None = None, ) -> list[str]: canonical_url = resolved_canonical_url if direct else draft.canonical_url fields = { "employer": employer, "original_title": draft.title, "normalized_title": draft.normalized_title, "job_family": draft.job_family, "language": draft.language, "description_html_sanitized": draft.description_html, "description_text": draft.description_text, "raw_location": draft.location_text, "country": draft.country, "region": draft.region, "municipality": draft.municipality, "postal_code": draft.postal_code, "workplace_type": draft.workplace_type, "employment_types": draft.employment_types, "compensation": draft.compensation, "skills_required": draft.skills_required, "skills_preferred": draft.skills_preferred, "date_posted": draft.date_posted, "valid_through": draft.valid_through, "content_hash": draft.content_hash, "analysis_features": extract_deterministic_features(draft.title, draft.description_text), "status": JobPosting.Status.ACTIVE, "last_seen": timezone.now(), "direct_employer": direct or (employer.is_direct_employer if employer else False), "recruiter": employer.is_recruiter if employer else not direct, } changed: list[str] = [] for field, value in fields.items(): if (value not in (None, "", [], {}) or field in {"status", "last_seen"}) and ( getattr(job, field) != value ): setattr(job, field, value) changed.append(field) if direct and canonical_url and job.canonical_url != canonical_url: job.canonical_url = canonical_url changed.append("canonical_url") if changed and "last_changed" not in changed: job.last_changed = timezone.now() changed.append("last_changed") return changed @transaction.atomic def persist_draft( draft: CanonicalJobDraft, *, document: RawDocument, parser_key: str, parser_version: str, extraction_confidence: float, ) -> tuple[JobPosting, DedupeDecision, bool]: source = document.source employer = resolve_employer(draft) decision = find_existing_job(draft, source=source) direct = _source_is_direct(source, draft) if decision.resolved_direct: direct = True created = False if decision.job is None: try: job = JobPosting.objects.create( employer=employer, original_title=draft.title, normalized_title=draft.normalized_title, job_family=draft.job_family, language=draft.language, canonical_url=draft.canonical_url, canonical_key=draft.canonical_key, content_hash=draft.content_hash, description_html_sanitized=draft.description_html, description_text=draft.description_text, raw_location=draft.location_text, country=draft.country, region=draft.region, municipality=draft.municipality, postal_code=draft.postal_code, workplace_type=draft.workplace_type, employment_types=draft.employment_types, compensation=draft.compensation, skills_required=draft.skills_required, skills_preferred=draft.skills_preferred, date_posted=draft.date_posted, valid_through=draft.valid_through, direct_employer=direct or (employer.is_direct_employer if employer else False), recruiter=employer.is_recruiter if employer else not direct, extraction_confidence=_confidence(extraction_confidence), analysis_features=extract_deterministic_features( draft.title, draft.description_text ), status=JobPosting.Status.ACTIVE, ) created = True except IntegrityError: job = JobPosting.objects.get(canonical_key=draft.canonical_key) decision = DedupeDecision(job, "canonical_key_race", 1.0) else: job = decision.job if job.content_hash != draft.content_hash: JobVersion.objects.get_or_create( job=job, content_hash=job.content_hash, defaults={"snapshot": job_snapshot(job), "changed_fields": []}, ) changed = _apply_draft( job, draft, employer, direct=direct, resolved_canonical_url=decision.canonical_url, ) if extraction_confidence > float(job.extraction_confidence): job.extraction_confidence = _confidence(extraction_confidence) changed.append("extraction_confidence") if changed: job.save(update_fields=list(dict.fromkeys([*changed, "updated_at"]))) alias = ( JobSourceAlias.objects.filter( job=job, source=source, canonical_url=draft.canonical_url, external_id=draft.external_id, ) .order_by("-last_seen") .first() ) if alias is None: alias = JobSourceAlias.objects.create( job=job, source=source, raw_document=document, url=draft.source_url, canonical_url=draft.canonical_url, external_id=draft.external_id, source_title=draft.title, source_employer=draft.employer_name, extraction_method=parser_key, extraction_confidence=_confidence(extraction_confidence), is_canonical=direct, payload=_alias_payload( draft.raw, decision, fallback_canonical_url=draft.canonical_url, ), ) else: alias.last_seen = timezone.now() alias.raw_document = document alias.payload = _alias_payload( alias.payload, decision, fallback_canonical_url=draft.canonical_url ) if direct: alias.is_canonical = True alias.save( update_fields=["last_seen", "raw_document", "payload", "is_canonical", "updated_at"] ) for evidence in draft.evidence: FieldProvenance.objects.update_or_create( job=job, source_alias=alias, field_name=evidence.field_name, extraction_method=evidence.method, defaults={ "confidence": _confidence(evidence.confidence), "evidence_excerpt": evidence.evidence[:1000], "parser_version": parser_version, }, ) JobVersion.objects.get_or_create( job=job, content_hash=job.content_hash, defaults={"snapshot": job_snapshot(job), "changed_fields": []}, ) for profile in SearchProfile.objects.filter(is_active=True): score_and_save(job, profile) return job, decision, created @transaction.atomic def process_raw_document(document: RawDocument) -> dict[str, int | str | list[str]]: result = registry.extract(document) document.parser_key = result.parser_key document.parser_version = result.parser_version document.extraction_confidence = _confidence(result.confidence) document.save( update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"] ) created = updated = duplicates = 0 for extracted in result.jobs: source_metadata = ( document.source.metadata if document.source and isinstance(document.source.metadata, dict) else {} ) configured_employer = str(source_metadata.get("employer_name") or "").strip() if configured_employer: extracted.employer_name = configured_employer extracted.evidence.append( FieldEvidence( "employer_name", "reviewed-source-config", 0.99, configured_employer, ) ) draft = normalize_extracted_job(extracted) _, decision, was_created = persist_draft( draft, document=document, parser_key=result.parser_key, parser_version=result.parser_version, extraction_confidence=result.confidence, ) if was_created: created += 1 elif ( decision.reason.startswith("exact") or decision.reason.startswith("fuzzy") or decision.reason == "resolved_direct_match" ): duplicates += 1 updated += 1 else: updated += 1 return { "extracted": len(result.jobs), "created": created, "updated": updated, "duplicates": duplicates, "parser": result.parser_key, "warnings": result.warnings, }