from __future__ import annotations import hashlib import re import unicodedata from dataclasses import dataclass, field from datetime import UTC, datetime from typing import Any from urllib.parse import urlsplit from dateutil import parser as date_parser from django.utils import timezone from apps.sources.adapters.base import ExtractedJob, FieldEvidence from apps.sources.services.canonicalize import canonicalize_url from .sanitize import sanitize_job_html TITLE_STOPWORDS = { "m/v", "m/v/x", "f/m/x", "h/f/x", "voltijds", "fulltime", "full-time", "parttime", "part-time", } EMPLOYMENT_MAP = { "full_time": "full_time", "full-time": "full_time", "fulltime": "full_time", "voltijds": "full_time", "temps plein": "full_time", "part_time": "part_time", "part-time": "part_time", "parttime": "part_time", "deeltijds": "part_time", "temps partiel": "part_time", "contractor": "freelance", "freelance": "freelance", "temporary": "temporary", "tijdelijk": "temporary", "interim": "temporary", "internship": "internship", "stage": "internship", "permanent": "permanent", "vast": "permanent", } TITLE_FAMILIES = { "it field engineer": "field-support", "field service engineer": "field-support", "it engineer": "infrastructure", "system engineer": "infrastructure", "system network engineer": "infrastructure", "system administrator": "infrastructure", "systeembeheerder": "infrastructure", "infrastructure engineer": "infrastructure", "network engineer": "network", "network administrator": "network", "netwerkbeheerder": "network", "workplace engineer": "workplace", "support engineer": "support", "it technician": "support", "pc technician": "support", "helpdesk": "support", "developer": "software-development", "data engineer": "data", "security engineer": "security", } @dataclass(slots=True) class CanonicalJobDraft: source_url: str canonical_url: str external_id: str title: str normalized_title: str job_family: str employer_name: str employer_domain: str location_text: str region: str municipality: str postal_code: str country: str workplace_type: str employment_types: list[str] language: str description_html: str description_text: str date_posted: datetime | None valid_through: datetime | None compensation: dict[str, Any] skills_required: list[str] skills_preferred: list[str] content_hash: str canonical_key: str evidence: list[FieldEvidence] = field(default_factory=list) raw: dict[str, Any] = field(default_factory=dict) def normalize_space(value: str) -> str: return " ".join((value or "").replace("\xa0", " ").split()) def normalize_token(value: str) -> str: normalized = unicodedata.normalize("NFKD", value or "") asciiish = "".join(ch for ch in normalized if not unicodedata.combining(ch)) asciiish = asciiish.casefold() asciiish = re.sub(r"[^\w+.#/-]+", " ", asciiish, flags=re.UNICODE) return normalize_space(asciiish) def normalize_title(value: str) -> str: title = normalize_token(value) for stopword in sorted(TITLE_STOPWORDS, key=len, reverse=True): title = re.sub(rf"\b{re.escape(stopword)}\b", " ", title) return normalize_space(title.strip(" -|/")) def infer_job_family(normalized_title: str) -> str: for term, family in TITLE_FAMILIES.items(): if term in normalized_title: return family return normalized_title.split(" ", 1)[0] if normalized_title else "unknown" def normalize_employment_types(values: list[str]) -> list[str]: result: list[str] = [] for raw in values: token = normalize_token(str(raw)).replace(" ", "_") mapped = EMPLOYMENT_MAP.get(token) or EMPLOYMENT_MAP.get(normalize_token(str(raw))) mapped = mapped or token if mapped and mapped not in result: result.append(mapped) return result def infer_language(text: str) -> str: sample = f" {normalize_token(text[:5000])} " scores = { "nl": sum(sample.count(f" {word} ") for word in ["de", "het", "een", "voor", "met"]), "fr": sum(sample.count(f" {word} ") for word in ["le", "la", "les", "pour", "avec"]), "en": sum(sample.count(f" {word} ") for word in ["the", "and", "for", "with", "you"]), } language, score = max(scores.items(), key=lambda item: item[1]) return language if score > 0 else "" def parse_datetime(value: str) -> datetime | None: if not value: return None try: parsed = date_parser.parse(value) except (ValueError, TypeError, OverflowError): return None if timezone.is_naive(parsed): parsed = timezone.make_aware(parsed, timezone.get_current_timezone()) return parsed.astimezone(UTC) def infer_workplace(value: str, text: str) -> str: token = normalize_token(f"{value} {text[:4000]}") if "telecommute" in token or re.search(r"\b(remote|thuiswerk|telewerk|homeworking)\b", token): if re.search(r"\b(hybrid|hybride|hybrid work|partly remote)\b", token): return "hybrid" return "remote" if re.search(r"\b(hybrid|hybride)\b", token): return "hybrid" if re.search(r"\b(on site|onsite|op locatie|sur site)\b", token): return "on_site" return "unknown" def canonical_key_for(draft_parts: list[str]) -> str: value = "|".join(normalize_token(part) for part in draft_parts if part) return hashlib.sha256(value.encode("utf-8")).hexdigest() def normalize_extracted_job(item: ExtractedJob) -> CanonicalJobDraft: canonical_url = canonicalize_url(item.url) normalized_title = normalize_title(item.title) employer_name = normalize_space(item.employer_name) employer_domain = (urlsplit(canonical_url).hostname or "").lower() description_text = normalize_space(item.description_text) description_html = sanitize_job_html(item.description_html) if not description_text and description_html: from bs4 import BeautifulSoup description_text = normalize_space(BeautifulSoup(description_html, "lxml").get_text(" ")) language = (item.language or "").split("-", 1)[0].lower() or infer_language( f"{item.title} {description_text}" ) municipality = normalize_space(item.location_text.split(",", 1)[0]) workplace_type = infer_workplace(item.workplace_type, description_text) employment_types = normalize_employment_types(item.employment_types) content_material = "|".join( [ normalized_title, normalize_token(employer_name), normalize_token(item.location_text), description_text, str(item.valid_through), ] ) content_hash = hashlib.sha256(content_material.encode("utf-8")).hexdigest() key_parts = [item.external_id, canonical_url] if not any(key_parts): key_parts = [employer_name, normalized_title, item.location_text] canonical_key = canonical_key_for(key_parts) return CanonicalJobDraft( source_url=item.url, canonical_url=canonical_url, external_id=normalize_space(item.external_id), title=normalize_space(item.title), normalized_title=normalized_title, job_family=infer_job_family(normalized_title), employer_name=employer_name, employer_domain=employer_domain, location_text=normalize_space(item.location_text), region=normalize_space(item.region), municipality=municipality, postal_code=normalize_space(item.postal_code), country=normalize_space(item.country), workplace_type=workplace_type, employment_types=employment_types, language=language, description_html=description_html, description_text=description_text, date_posted=parse_datetime(item.date_posted), valid_through=parse_datetime(item.valid_through), compensation=item.compensation, skills_required=[normalize_space(v) for v in item.skills_required if normalize_space(v)], skills_preferred=[normalize_space(v) for v in item.skills_preferred if normalize_space(v)], content_hash=content_hash, canonical_key=canonical_key, evidence=item.evidence, raw=item.raw, )