235 lines
7.9 KiB
Python
235 lines
7.9 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import re
|
|
import unicodedata
|
|
from dataclasses import dataclass, field
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
from urllib.parse import urlsplit
|
|
|
|
from dateutil import parser as date_parser
|
|
from django.utils import timezone
|
|
|
|
from apps.sources.adapters.base import ExtractedJob, FieldEvidence
|
|
from apps.sources.services.canonicalize import canonicalize_url
|
|
|
|
from .sanitize import sanitize_job_html
|
|
|
|
TITLE_STOPWORDS = {
|
|
"m/v",
|
|
"m/v/x",
|
|
"f/m/x",
|
|
"h/f/x",
|
|
"voltijds",
|
|
"fulltime",
|
|
"full-time",
|
|
"parttime",
|
|
"part-time",
|
|
}
|
|
EMPLOYMENT_MAP = {
|
|
"full_time": "full_time",
|
|
"full-time": "full_time",
|
|
"fulltime": "full_time",
|
|
"voltijds": "full_time",
|
|
"temps plein": "full_time",
|
|
"part_time": "part_time",
|
|
"part-time": "part_time",
|
|
"parttime": "part_time",
|
|
"deeltijds": "part_time",
|
|
"temps partiel": "part_time",
|
|
"contractor": "freelance",
|
|
"freelance": "freelance",
|
|
"temporary": "temporary",
|
|
"tijdelijk": "temporary",
|
|
"interim": "temporary",
|
|
"internship": "internship",
|
|
"stage": "internship",
|
|
"permanent": "permanent",
|
|
"vast": "permanent",
|
|
}
|
|
TITLE_FAMILIES = {
|
|
"system engineer": "infrastructure",
|
|
"systeembeheerder": "infrastructure",
|
|
"infrastructure engineer": "infrastructure",
|
|
"network engineer": "network",
|
|
"netwerkbeheerder": "network",
|
|
"workplace engineer": "workplace",
|
|
"support engineer": "support",
|
|
"helpdesk": "support",
|
|
"developer": "software-development",
|
|
"data engineer": "data",
|
|
"security engineer": "security",
|
|
}
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class CanonicalJobDraft:
|
|
source_url: str
|
|
canonical_url: str
|
|
external_id: str
|
|
title: str
|
|
normalized_title: str
|
|
job_family: str
|
|
employer_name: str
|
|
employer_domain: str
|
|
location_text: str
|
|
region: str
|
|
municipality: str
|
|
postal_code: str
|
|
country: str
|
|
workplace_type: str
|
|
employment_types: list[str]
|
|
language: str
|
|
description_html: str
|
|
description_text: str
|
|
date_posted: datetime | None
|
|
valid_through: datetime | None
|
|
compensation: dict[str, Any]
|
|
skills_required: list[str]
|
|
skills_preferred: list[str]
|
|
content_hash: str
|
|
canonical_key: str
|
|
evidence: list[FieldEvidence] = field(default_factory=list)
|
|
raw: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
def normalize_space(value: str) -> str:
|
|
return " ".join((value or "").replace("\xa0", " ").split())
|
|
|
|
|
|
def normalize_token(value: str) -> str:
|
|
normalized = unicodedata.normalize("NFKD", value or "")
|
|
asciiish = "".join(ch for ch in normalized if not unicodedata.combining(ch))
|
|
asciiish = asciiish.casefold()
|
|
asciiish = re.sub(r"[^\w+.#/-]+", " ", asciiish, flags=re.UNICODE)
|
|
return normalize_space(asciiish)
|
|
|
|
|
|
def normalize_title(value: str) -> str:
|
|
title = normalize_token(value)
|
|
for stopword in sorted(TITLE_STOPWORDS, key=len, reverse=True):
|
|
title = re.sub(rf"\b{re.escape(stopword)}\b", " ", title)
|
|
return normalize_space(title.strip(" -|/"))
|
|
|
|
|
|
def infer_job_family(normalized_title: str) -> str:
|
|
for term, family in TITLE_FAMILIES.items():
|
|
if term in normalized_title:
|
|
return family
|
|
return normalized_title.split(" ", 1)[0] if normalized_title else "unknown"
|
|
|
|
|
|
def normalize_employment_types(values: list[str]) -> list[str]:
|
|
result: list[str] = []
|
|
for raw in values:
|
|
token = normalize_token(str(raw)).replace(" ", "_")
|
|
mapped = EMPLOYMENT_MAP.get(token) or EMPLOYMENT_MAP.get(normalize_token(str(raw)))
|
|
mapped = mapped or token
|
|
if mapped and mapped not in result:
|
|
result.append(mapped)
|
|
return result
|
|
|
|
|
|
def infer_language(text: str) -> str:
|
|
sample = f" {normalize_token(text[:5000])} "
|
|
scores = {
|
|
"nl": sum(sample.count(f" {word} ") for word in ["de", "het", "een", "voor", "met"]),
|
|
"fr": sum(sample.count(f" {word} ") for word in ["le", "la", "les", "pour", "avec"]),
|
|
"en": sum(sample.count(f" {word} ") for word in ["the", "and", "for", "with", "you"]),
|
|
}
|
|
language, score = max(scores.items(), key=lambda item: item[1])
|
|
return language if score > 0 else ""
|
|
|
|
|
|
def parse_datetime(value: str) -> datetime | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
parsed = date_parser.parse(value)
|
|
except (ValueError, TypeError, OverflowError):
|
|
return None
|
|
if timezone.is_naive(parsed):
|
|
parsed = timezone.make_aware(parsed, timezone.get_current_timezone())
|
|
return parsed.astimezone(UTC)
|
|
|
|
|
|
def infer_workplace(value: str, text: str) -> str:
|
|
token = normalize_token(f"{value} {text[:4000]}")
|
|
if "telecommute" in token or re.search(r"\b(remote|thuiswerk|telewerk|homeworking)\b", token):
|
|
if re.search(r"\b(hybrid|hybride|hybrid work|partly remote)\b", token):
|
|
return "hybrid"
|
|
return "remote"
|
|
if re.search(r"\b(hybrid|hybride)\b", token):
|
|
return "hybrid"
|
|
if re.search(r"\b(on site|onsite|op locatie|sur site)\b", token):
|
|
return "on_site"
|
|
return "unknown"
|
|
|
|
|
|
def canonical_key_for(draft_parts: list[str]) -> str:
|
|
value = "|".join(normalize_token(part) for part in draft_parts if part)
|
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def normalize_extracted_job(item: ExtractedJob) -> CanonicalJobDraft:
|
|
canonical_url = canonicalize_url(item.url)
|
|
normalized_title = normalize_title(item.title)
|
|
employer_name = normalize_space(item.employer_name)
|
|
employer_domain = (urlsplit(canonical_url).hostname or "").lower()
|
|
description_text = normalize_space(item.description_text)
|
|
description_html = sanitize_job_html(item.description_html)
|
|
if not description_text and description_html:
|
|
from bs4 import BeautifulSoup
|
|
|
|
description_text = normalize_space(BeautifulSoup(description_html, "lxml").get_text(" "))
|
|
language = (item.language or "").split("-", 1)[0].lower() or infer_language(
|
|
f"{item.title} {description_text}"
|
|
)
|
|
municipality = normalize_space(item.location_text.split(",", 1)[0])
|
|
workplace_type = infer_workplace(item.workplace_type, description_text)
|
|
employment_types = normalize_employment_types(item.employment_types)
|
|
content_material = "|".join(
|
|
[
|
|
normalized_title,
|
|
normalize_token(employer_name),
|
|
normalize_token(item.location_text),
|
|
description_text,
|
|
str(item.valid_through),
|
|
]
|
|
)
|
|
content_hash = hashlib.sha256(content_material.encode("utf-8")).hexdigest()
|
|
key_parts = [item.external_id, canonical_url]
|
|
if not any(key_parts):
|
|
key_parts = [employer_name, normalized_title, item.location_text]
|
|
canonical_key = canonical_key_for(key_parts)
|
|
return CanonicalJobDraft(
|
|
source_url=item.url,
|
|
canonical_url=canonical_url,
|
|
external_id=normalize_space(item.external_id),
|
|
title=normalize_space(item.title),
|
|
normalized_title=normalized_title,
|
|
job_family=infer_job_family(normalized_title),
|
|
employer_name=employer_name,
|
|
employer_domain=employer_domain,
|
|
location_text=normalize_space(item.location_text),
|
|
region=normalize_space(item.region),
|
|
municipality=municipality,
|
|
postal_code=normalize_space(item.postal_code),
|
|
country=normalize_space(item.country),
|
|
workplace_type=workplace_type,
|
|
employment_types=employment_types,
|
|
language=language,
|
|
description_html=description_html,
|
|
description_text=description_text,
|
|
date_posted=parse_datetime(item.date_posted),
|
|
valid_through=parse_datetime(item.valid_through),
|
|
compensation=item.compensation,
|
|
skills_required=[normalize_space(v) for v in item.skills_required if normalize_space(v)],
|
|
skills_preferred=[normalize_space(v) for v in item.skills_preferred if normalize_space(v)],
|
|
content_hash=content_hash,
|
|
canonical_key=canonical_key,
|
|
evidence=item.evidence,
|
|
raw=item.raw,
|
|
)
|