Jens
@@ -0,0 +1,234 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from dateutil import parser as date_parser
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.sources.adapters.base import ExtractedJob, FieldEvidence
|
||||
from apps.sources.services.canonicalize import canonicalize_url
|
||||
|
||||
from .sanitize import sanitize_job_html
|
||||
|
||||
TITLE_STOPWORDS = {
|
||||
"m/v",
|
||||
"m/v/x",
|
||||
"f/m/x",
|
||||
"h/f/x",
|
||||
"voltijds",
|
||||
"fulltime",
|
||||
"full-time",
|
||||
"parttime",
|
||||
"part-time",
|
||||
}
|
||||
EMPLOYMENT_MAP = {
|
||||
"full_time": "full_time",
|
||||
"full-time": "full_time",
|
||||
"fulltime": "full_time",
|
||||
"voltijds": "full_time",
|
||||
"temps plein": "full_time",
|
||||
"part_time": "part_time",
|
||||
"part-time": "part_time",
|
||||
"parttime": "part_time",
|
||||
"deeltijds": "part_time",
|
||||
"temps partiel": "part_time",
|
||||
"contractor": "freelance",
|
||||
"freelance": "freelance",
|
||||
"temporary": "temporary",
|
||||
"tijdelijk": "temporary",
|
||||
"interim": "temporary",
|
||||
"internship": "internship",
|
||||
"stage": "internship",
|
||||
"permanent": "permanent",
|
||||
"vast": "permanent",
|
||||
}
|
||||
TITLE_FAMILIES = {
|
||||
"system engineer": "infrastructure",
|
||||
"systeembeheerder": "infrastructure",
|
||||
"infrastructure engineer": "infrastructure",
|
||||
"network engineer": "network",
|
||||
"netwerkbeheerder": "network",
|
||||
"workplace engineer": "workplace",
|
||||
"support engineer": "support",
|
||||
"helpdesk": "support",
|
||||
"developer": "software-development",
|
||||
"data engineer": "data",
|
||||
"security engineer": "security",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CanonicalJobDraft:
|
||||
source_url: str
|
||||
canonical_url: str
|
||||
external_id: str
|
||||
title: str
|
||||
normalized_title: str
|
||||
job_family: str
|
||||
employer_name: str
|
||||
employer_domain: str
|
||||
location_text: str
|
||||
region: str
|
||||
municipality: str
|
||||
postal_code: str
|
||||
country: str
|
||||
workplace_type: str
|
||||
employment_types: list[str]
|
||||
language: str
|
||||
description_html: str
|
||||
description_text: str
|
||||
date_posted: datetime | None
|
||||
valid_through: datetime | None
|
||||
compensation: dict[str, Any]
|
||||
skills_required: list[str]
|
||||
skills_preferred: list[str]
|
||||
content_hash: str
|
||||
canonical_key: str
|
||||
evidence: list[FieldEvidence] = field(default_factory=list)
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def normalize_space(value: str) -> str:
|
||||
return " ".join((value or "").replace("\xa0", " ").split())
|
||||
|
||||
|
||||
def normalize_token(value: str) -> str:
|
||||
normalized = unicodedata.normalize("NFKD", value or "")
|
||||
asciiish = "".join(ch for ch in normalized if not unicodedata.combining(ch))
|
||||
asciiish = asciiish.casefold()
|
||||
asciiish = re.sub(r"[^\w+.#/-]+", " ", asciiish, flags=re.UNICODE)
|
||||
return normalize_space(asciiish)
|
||||
|
||||
|
||||
def normalize_title(value: str) -> str:
|
||||
title = normalize_token(value)
|
||||
for stopword in sorted(TITLE_STOPWORDS, key=len, reverse=True):
|
||||
title = re.sub(rf"\b{re.escape(stopword)}\b", " ", title)
|
||||
return normalize_space(title.strip(" -|/"))
|
||||
|
||||
|
||||
def infer_job_family(normalized_title: str) -> str:
|
||||
for term, family in TITLE_FAMILIES.items():
|
||||
if term in normalized_title:
|
||||
return family
|
||||
return normalized_title.split(" ", 1)[0] if normalized_title else "unknown"
|
||||
|
||||
|
||||
def normalize_employment_types(values: list[str]) -> list[str]:
|
||||
result: list[str] = []
|
||||
for raw in values:
|
||||
token = normalize_token(str(raw)).replace(" ", "_")
|
||||
mapped = EMPLOYMENT_MAP.get(token) or EMPLOYMENT_MAP.get(normalize_token(str(raw)))
|
||||
mapped = mapped or token
|
||||
if mapped and mapped not in result:
|
||||
result.append(mapped)
|
||||
return result
|
||||
|
||||
|
||||
def infer_language(text: str) -> str:
|
||||
sample = f" {normalize_token(text[:5000])} "
|
||||
scores = {
|
||||
"nl": sum(sample.count(f" {word} ") for word in ["de", "het", "een", "voor", "met"]),
|
||||
"fr": sum(sample.count(f" {word} ") for word in ["le", "la", "les", "pour", "avec"]),
|
||||
"en": sum(sample.count(f" {word} ") for word in ["the", "and", "for", "with", "you"]),
|
||||
}
|
||||
language, score = max(scores.items(), key=lambda item: item[1])
|
||||
return language if score > 0 else ""
|
||||
|
||||
|
||||
def parse_datetime(value: str) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
parsed = date_parser.parse(value)
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
return None
|
||||
if timezone.is_naive(parsed):
|
||||
parsed = timezone.make_aware(parsed, timezone.get_current_timezone())
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def infer_workplace(value: str, text: str) -> str:
|
||||
token = normalize_token(f"{value} {text[:4000]}")
|
||||
if "telecommute" in token or re.search(r"\b(remote|thuiswerk|telewerk|homeworking)\b", token):
|
||||
if re.search(r"\b(hybrid|hybride|hybrid work|partly remote)\b", token):
|
||||
return "hybrid"
|
||||
return "remote"
|
||||
if re.search(r"\b(hybrid|hybride)\b", token):
|
||||
return "hybrid"
|
||||
if re.search(r"\b(on site|onsite|op locatie|sur site)\b", token):
|
||||
return "on_site"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def canonical_key_for(draft_parts: list[str]) -> str:
|
||||
value = "|".join(normalize_token(part) for part in draft_parts if part)
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def normalize_extracted_job(item: ExtractedJob) -> CanonicalJobDraft:
|
||||
canonical_url = canonicalize_url(item.url)
|
||||
normalized_title = normalize_title(item.title)
|
||||
employer_name = normalize_space(item.employer_name)
|
||||
employer_domain = (urlsplit(canonical_url).hostname or "").lower()
|
||||
description_text = normalize_space(item.description_text)
|
||||
description_html = sanitize_job_html(item.description_html)
|
||||
if not description_text and description_html:
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
description_text = normalize_space(BeautifulSoup(description_html, "lxml").get_text(" "))
|
||||
language = (item.language or "").split("-", 1)[0].lower() or infer_language(
|
||||
f"{item.title} {description_text}"
|
||||
)
|
||||
municipality = normalize_space(item.location_text.split(",", 1)[0])
|
||||
workplace_type = infer_workplace(item.workplace_type, description_text)
|
||||
employment_types = normalize_employment_types(item.employment_types)
|
||||
content_material = "|".join(
|
||||
[
|
||||
normalized_title,
|
||||
normalize_token(employer_name),
|
||||
normalize_token(item.location_text),
|
||||
description_text,
|
||||
str(item.valid_through),
|
||||
]
|
||||
)
|
||||
content_hash = hashlib.sha256(content_material.encode("utf-8")).hexdigest()
|
||||
key_parts = [item.external_id, canonical_url]
|
||||
if not any(key_parts):
|
||||
key_parts = [employer_name, normalized_title, item.location_text]
|
||||
canonical_key = canonical_key_for(key_parts)
|
||||
return CanonicalJobDraft(
|
||||
source_url=item.url,
|
||||
canonical_url=canonical_url,
|
||||
external_id=normalize_space(item.external_id),
|
||||
title=normalize_space(item.title),
|
||||
normalized_title=normalized_title,
|
||||
job_family=infer_job_family(normalized_title),
|
||||
employer_name=employer_name,
|
||||
employer_domain=employer_domain,
|
||||
location_text=normalize_space(item.location_text),
|
||||
region=normalize_space(item.region),
|
||||
municipality=municipality,
|
||||
postal_code=normalize_space(item.postal_code),
|
||||
country=normalize_space(item.country),
|
||||
workplace_type=workplace_type,
|
||||
employment_types=employment_types,
|
||||
language=language,
|
||||
description_html=description_html,
|
||||
description_text=description_text,
|
||||
date_posted=parse_datetime(item.date_posted),
|
||||
valid_through=parse_datetime(item.valid_through),
|
||||
compensation=item.compensation,
|
||||
skills_required=[normalize_space(v) for v in item.skills_required if normalize_space(v)],
|
||||
skills_preferred=[normalize_space(v) for v in item.skills_preferred if normalize_space(v)],
|
||||
content_hash=content_hash,
|
||||
canonical_key=canonical_key,
|
||||
evidence=item.evidence,
|
||||
raw=item.raw,
|
||||
)
|
||||
Reference in New Issue
Block a user