Jens
@@ -0,0 +1,24 @@
|
||||
from .base import ExtractedJob, ExtractionResult
|
||||
from .generic_html import GenericHtmlAdapter
|
||||
from .jsonld import JsonLdJobPostingAdapter
|
||||
from .rss import RssAdapter
|
||||
from .ats import (
|
||||
GreenhouseAdapter,
|
||||
LeverAdapter,
|
||||
RecruiteeAdapter,
|
||||
SmartRecruitersAdapter,
|
||||
WorkableAdapter,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ExtractedJob",
|
||||
"ExtractionResult",
|
||||
"GenericHtmlAdapter",
|
||||
"JsonLdJobPostingAdapter",
|
||||
"RssAdapter",
|
||||
"GreenhouseAdapter",
|
||||
"LeverAdapter",
|
||||
"RecruiteeAdapter",
|
||||
"SmartRecruitersAdapter",
|
||||
"WorkableAdapter",
|
||||
]
|
||||
@@ -0,0 +1,546 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
||||
|
||||
|
||||
CLOSED_STATUSES = {
|
||||
"closed",
|
||||
"inactive",
|
||||
"withdrawn",
|
||||
"removed",
|
||||
"filled",
|
||||
"expired",
|
||||
"archived",
|
||||
}
|
||||
|
||||
|
||||
def _to_text(value):
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
if isinstance(value, bool):
|
||||
return str(value).lower()
|
||||
if isinstance(value, int | float):
|
||||
return str(value)
|
||||
return ""
|
||||
|
||||
|
||||
def _find_nested(data, path: str):
|
||||
if data is None:
|
||||
return None
|
||||
node = data
|
||||
for part in path.split("."):
|
||||
if not isinstance(node, dict) or part not in node:
|
||||
return None
|
||||
node = node[part]
|
||||
return node
|
||||
|
||||
|
||||
def _first_text(data, *candidates):
|
||||
for candidate in candidates:
|
||||
if "." in candidate:
|
||||
value = _find_nested(data, candidate)
|
||||
else:
|
||||
value = data.get(candidate) if isinstance(data, dict) else None
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, (list, tuple)):
|
||||
for item in value:
|
||||
text = _to_text(item)
|
||||
if text:
|
||||
return text
|
||||
continue
|
||||
text = _to_text(value)
|
||||
if text:
|
||||
return text
|
||||
if isinstance(value, dict):
|
||||
nested = value.values()
|
||||
for nested_value in nested:
|
||||
nested_text = _to_text(nested_value)
|
||||
if nested_text:
|
||||
return nested_text
|
||||
return ""
|
||||
|
||||
|
||||
def _collect_texts(value) -> list[str]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, str):
|
||||
return [value.strip()] if value.strip() else []
|
||||
if isinstance(value, list):
|
||||
out: list[str] = []
|
||||
for item in value:
|
||||
for inner in _collect_texts(item):
|
||||
if inner:
|
||||
out.append(inner)
|
||||
return out
|
||||
if isinstance(value, dict):
|
||||
out: list[str] = []
|
||||
for key in ("name", "city", "location", "address", "raw", "region", "country"):
|
||||
if key in value:
|
||||
for inner in _collect_texts(value[key]):
|
||||
if inner:
|
||||
out.append(inner)
|
||||
return out
|
||||
if isinstance(value, bool | int | float):
|
||||
return [_to_text(value)]
|
||||
return []
|
||||
|
||||
|
||||
def _to_list(value) -> list[str]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return [item.strip() for item in (_to_text(v) for v in value) if item.strip()]
|
||||
text = _to_text(value)
|
||||
if not text:
|
||||
return []
|
||||
return [part.strip() for part in text.replace(";", ",").split(",") if part.strip()]
|
||||
|
||||
|
||||
def _as_text(value) -> str:
|
||||
text = _to_text(value)
|
||||
if not text:
|
||||
return ""
|
||||
return BeautifulSoup(text, "lxml").get_text(" ", strip=True)
|
||||
|
||||
|
||||
def _extract_payload(content: str):
|
||||
try:
|
||||
return json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
soup = BeautifulSoup(content, "lxml")
|
||||
for script in soup.find_all("script", attrs={"type": "application/json"}):
|
||||
script_text = script.string or script.get_text("", strip=True)
|
||||
if not script_text:
|
||||
continue
|
||||
try:
|
||||
return json.loads(script_text)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
class _AtsAdapter(ABC):
|
||||
parser_key = "ats-provider"
|
||||
parser_version = "1.0.0"
|
||||
source_hosts: tuple[str, ...] = ()
|
||||
support_markers: tuple[str, ...] = ()
|
||||
listing_paths: tuple[tuple[str, ...], ...] = ()
|
||||
detail_paths: tuple[tuple[str, ...], ...] = ()
|
||||
closed_statuses = CLOSED_STATUSES
|
||||
|
||||
@abstractmethod
|
||||
def _extract_records(self, payload) -> list[dict[str, object]]:
|
||||
...
|
||||
|
||||
def _supports_url(self, url: str) -> bool:
|
||||
host = (urlsplit(url).hostname or "").lower()
|
||||
return any(host.endswith(suffix) for suffix in self.source_hosts)
|
||||
|
||||
def _supports_payload(self, payload, content: str) -> bool:
|
||||
if not payload:
|
||||
return False
|
||||
payload_text = str(payload).lower()
|
||||
return any(
|
||||
marker in payload_text or marker in content.lower() for marker in self.support_markers
|
||||
)
|
||||
|
||||
def _job_is_closed(self, record: dict[str, object]) -> bool:
|
||||
status = _first_text(record, "status", "state", "job_status", "data.status").lower()
|
||||
if status in self.closed_statuses:
|
||||
return True
|
||||
active = record.get("active")
|
||||
if isinstance(active, bool):
|
||||
return not active
|
||||
return False
|
||||
|
||||
def _coerce_url(self, raw_url: str, base_url: str) -> str:
|
||||
if raw_url:
|
||||
return urljoin(base_url, raw_url)
|
||||
return base_url
|
||||
|
||||
def _extract_location(self, record: dict[str, object]) -> tuple[str, str, str, str]:
|
||||
location_raw = _first_text(
|
||||
record,
|
||||
"location",
|
||||
"city",
|
||||
"cityName",
|
||||
"place",
|
||||
"office",
|
||||
"location.address",
|
||||
"address",
|
||||
"data.location",
|
||||
"officeLocation",
|
||||
"locationName",
|
||||
)
|
||||
if not location_raw:
|
||||
location_raw = _first_text(record, "data.city", "data.location")
|
||||
if not location_raw:
|
||||
location_values = []
|
||||
for key in ("city", "address", "location", "region", "office"):
|
||||
location_values.extend(_collect_texts(record.get(key, "")))
|
||||
location_raw = ", ".join(location_values)
|
||||
|
||||
location = location_raw
|
||||
location_parts = [part.strip() for part in _to_text(location).split(",") if part.strip()]
|
||||
region = _first_text(
|
||||
record,
|
||||
"region",
|
||||
"data.region",
|
||||
"location.region",
|
||||
"address.region",
|
||||
)
|
||||
postal_code = _first_text(
|
||||
record,
|
||||
"postal_code",
|
||||
"postalCode",
|
||||
"location.postalCode",
|
||||
"zipCode",
|
||||
)
|
||||
country = _first_text(
|
||||
record,
|
||||
"country",
|
||||
"countryName",
|
||||
"location.country",
|
||||
"address.country",
|
||||
"data.country",
|
||||
)
|
||||
return location, region, postal_code, country
|
||||
|
||||
def _extract_workplace(self, record: dict[str, object]) -> str:
|
||||
workplace_raw = _first_text(
|
||||
record,
|
||||
"workplaceType",
|
||||
"workplace",
|
||||
"remoteType",
|
||||
"remote_type",
|
||||
"job_type",
|
||||
).lower()
|
||||
if "remote" in workplace_raw:
|
||||
return "remote" if "hybrid" not in workplace_raw else "hybrid"
|
||||
if "hybrid" in workplace_raw:
|
||||
return "hybrid"
|
||||
if "onsite" in workplace_raw or "on site" in workplace_raw or "on-site" in workplace_raw:
|
||||
return "on_site"
|
||||
return ""
|
||||
|
||||
def _to_job(self, record: dict[str, object], base_url: str) -> ExtractedJob:
|
||||
external_id = _first_text(
|
||||
record,
|
||||
"id",
|
||||
"jobId",
|
||||
"requisitionId",
|
||||
"postingId",
|
||||
"referenceId",
|
||||
"positionId",
|
||||
)
|
||||
title = _first_text(
|
||||
record,
|
||||
"title",
|
||||
"name",
|
||||
"position",
|
||||
"positionName",
|
||||
"jobTitle",
|
||||
"text",
|
||||
"title.value",
|
||||
)
|
||||
raw_url = _first_text(
|
||||
record,
|
||||
"url",
|
||||
"jobUrl",
|
||||
"job_url",
|
||||
"applyUrl",
|
||||
"link",
|
||||
"absoluteUrl",
|
||||
"data.url",
|
||||
)
|
||||
employer_name = _first_text(
|
||||
record,
|
||||
"company",
|
||||
"employer",
|
||||
"organization",
|
||||
"organizationName",
|
||||
"company.name",
|
||||
"department",
|
||||
"hiringOrganization",
|
||||
)
|
||||
location_text, region, postal_code, country = self._extract_location(record)
|
||||
description_raw = _first_text(
|
||||
record,
|
||||
"description",
|
||||
"jobDescription",
|
||||
"content",
|
||||
"descriptionHtml",
|
||||
"descriptionText",
|
||||
)
|
||||
description_text = _as_text(description_raw)
|
||||
date_posted = _first_text(
|
||||
record,
|
||||
"createdAt",
|
||||
"created",
|
||||
"datePosted",
|
||||
"publishedAt",
|
||||
"published_at",
|
||||
"created_at",
|
||||
"jobCreated",
|
||||
)
|
||||
valid_through = _first_text(
|
||||
record,
|
||||
"closeDate",
|
||||
"expiresAt",
|
||||
"validThrough",
|
||||
"valid_until",
|
||||
"expirationDate",
|
||||
"expiryDate",
|
||||
)
|
||||
employment_types = _to_list(
|
||||
_first_text(
|
||||
record,
|
||||
"employmentType",
|
||||
"employment_types",
|
||||
"jobType",
|
||||
"type",
|
||||
"data.employmentType",
|
||||
)
|
||||
)
|
||||
workplace_type = self._extract_workplace(record)
|
||||
|
||||
evidence = [
|
||||
FieldEvidence("external_id", "ats-id", 0.95, external_id[:240]),
|
||||
FieldEvidence("url", "ats-url", 0.9, raw_url[:240]),
|
||||
FieldEvidence("location_text", "ats-location", 0.88, location_text[:240]),
|
||||
FieldEvidence("date_posted", "ats-date", 0.8, date_posted[:240]),
|
||||
FieldEvidence("valid_through", "ats-date", 0.8, valid_through[:240]),
|
||||
FieldEvidence(
|
||||
"employment_types",
|
||||
"ats-employment",
|
||||
0.9,
|
||||
", ".join(employment_types)[:240],
|
||||
),
|
||||
]
|
||||
|
||||
return ExtractedJob(
|
||||
url=self._coerce_url(raw_url, base_url),
|
||||
title=title,
|
||||
employer_name=employer_name,
|
||||
external_id=external_id,
|
||||
location_text=location_text,
|
||||
region=region,
|
||||
postal_code=postal_code,
|
||||
country=country,
|
||||
description_html=description_raw,
|
||||
description_text=description_text,
|
||||
date_posted=date_posted,
|
||||
valid_through=valid_through,
|
||||
employment_types=employment_types,
|
||||
workplace_type=workplace_type,
|
||||
raw=record,
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
def extract(self, content: str, *, url: str) -> ExtractionResult:
|
||||
if not self._supports_url(url):
|
||||
return ExtractionResult(
|
||||
[],
|
||||
self.parser_key,
|
||||
self.parser_version,
|
||||
0.0,
|
||||
["Onherkenbare ATS-host"],
|
||||
)
|
||||
|
||||
payload = _extract_payload(content)
|
||||
if not payload:
|
||||
return ExtractionResult(
|
||||
[],
|
||||
self.parser_key,
|
||||
self.parser_version,
|
||||
0.0,
|
||||
["Geen parseerbare ATS-response"],
|
||||
)
|
||||
|
||||
if not self._supports_payload(payload, content):
|
||||
return ExtractionResult(
|
||||
[],
|
||||
self.parser_key,
|
||||
self.parser_version,
|
||||
0.0,
|
||||
["Geen herkenbare ATS-markup voor deze adapter"],
|
||||
)
|
||||
|
||||
jobs: list[ExtractedJob] = []
|
||||
for record in self._extract_records(payload):
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
if self._job_is_closed(record):
|
||||
continue
|
||||
job = self._to_job(record, base_url=url)
|
||||
if job.title:
|
||||
jobs.append(job)
|
||||
|
||||
warnings: list[str] = []
|
||||
if not jobs:
|
||||
warnings.append("Geen actieve ATS-vacatures gevonden")
|
||||
return ExtractionResult(jobs, self.parser_key, self.parser_version, 0.9 if jobs else 0.0, warnings)
|
||||
|
||||
|
||||
class GreenhouseAdapter(_AtsAdapter):
|
||||
parser_key = "ats-greenhouse"
|
||||
source_hosts = ("greenhouse.io", "boards.greenhouse.io")
|
||||
support_markers = ("greenhouse", "job board", "jobboard")
|
||||
listing_paths = (("jobs",), ("data", "jobs"), ("data", "results"))
|
||||
detail_paths = (("job",), ("data", "job"), ("result", "job"), ("result", "position"))
|
||||
closed_statuses = CLOSED_STATUSES | {"published", "draft", "deleted"}
|
||||
|
||||
def _extract_records(self, payload):
|
||||
for path in self.listing_paths:
|
||||
value = payload
|
||||
for key in path:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
value = None
|
||||
break
|
||||
value = value[key]
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
if isinstance(value, dict) and "results" in value:
|
||||
possible = value.get("results")
|
||||
if isinstance(possible, list):
|
||||
return [item for item in possible if isinstance(item, dict)]
|
||||
for path in self.detail_paths:
|
||||
value = payload
|
||||
for key in path:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
value = None
|
||||
break
|
||||
value = value[key]
|
||||
if isinstance(value, dict):
|
||||
return [value]
|
||||
return []
|
||||
|
||||
|
||||
class LeverAdapter(_AtsAdapter):
|
||||
parser_key = "ats-lever"
|
||||
source_hosts = ("jobs.lever.co",)
|
||||
support_markers = ("lever", "requisition", "posting")
|
||||
listing_paths = (("data",), ("jobs",), ("results",))
|
||||
detail_paths = (("data",), ("job",), ("position",), ("result",))
|
||||
closed_statuses = CLOSED_STATUSES | {"archived", "deleted"}
|
||||
|
||||
def _extract_records(self, payload):
|
||||
for path in self.listing_paths:
|
||||
value = payload
|
||||
for key in path:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
value = None
|
||||
break
|
||||
value = value[key]
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
for path in self.detail_paths:
|
||||
value = payload
|
||||
for key in path:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
value = None
|
||||
break
|
||||
value = value[key]
|
||||
if isinstance(value, dict):
|
||||
return [value]
|
||||
return []
|
||||
|
||||
|
||||
class RecruiteeAdapter(_AtsAdapter):
|
||||
parser_key = "ats-recruitee"
|
||||
source_hosts = ("recruitee.com",)
|
||||
support_markers = ("recruitee", "career", "vacancy")
|
||||
listing_paths = (("jobs",), ("data", "jobs"), ("vacancies",))
|
||||
detail_paths = (("job",), ("data", "job"), ("vacancy",), ("result",))
|
||||
closed_statuses = CLOSED_STATUSES | {"hidden", "paused"}
|
||||
|
||||
def _extract_records(self, payload):
|
||||
for path in self.listing_paths:
|
||||
value = payload
|
||||
for key in path:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
value = None
|
||||
break
|
||||
value = value[key]
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
for path in self.detail_paths:
|
||||
value = payload
|
||||
for key in path:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
value = None
|
||||
break
|
||||
value = value[key]
|
||||
if isinstance(value, dict):
|
||||
return [value]
|
||||
return []
|
||||
|
||||
|
||||
class SmartRecruitersAdapter(_AtsAdapter):
|
||||
parser_key = "ats-smartrecruiters"
|
||||
source_hosts = ("smartrecruiters.com",)
|
||||
support_markers = ("smartrecruiters", "smart recruiter")
|
||||
listing_paths = (("jobs",), ("data", "jobs"), ("results",))
|
||||
detail_paths = (("job",), ("data", "job"), ("posting",), ("result",))
|
||||
closed_statuses = CLOSED_STATUSES | {"unpublished", "expired"}
|
||||
|
||||
def _extract_records(self, payload):
|
||||
for path in self.listing_paths:
|
||||
value = payload
|
||||
for key in path:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
value = None
|
||||
break
|
||||
value = value[key]
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
for path in self.detail_paths:
|
||||
value = payload
|
||||
for key in path:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
value = None
|
||||
break
|
||||
value = value[key]
|
||||
if isinstance(value, dict):
|
||||
return [value]
|
||||
return []
|
||||
|
||||
|
||||
class WorkableAdapter(_AtsAdapter):
|
||||
parser_key = "ats-workable"
|
||||
source_hosts = ("apply.workable.com",)
|
||||
support_markers = ("workable", "workable job")
|
||||
listing_paths = (("jobs",), ("data", "jobs"), ("results",))
|
||||
detail_paths = (("job",), ("data", "job"), ("position",), ("result",))
|
||||
closed_statuses = CLOSED_STATUSES | {"draft", "inactive"}
|
||||
|
||||
def _extract_records(self, payload):
|
||||
for path in self.listing_paths:
|
||||
value = payload
|
||||
for key in path:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
value = None
|
||||
break
|
||||
value = value[key]
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
for path in self.detail_paths:
|
||||
value = payload
|
||||
for key in path:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
value = None
|
||||
break
|
||||
value = value[key]
|
||||
if isinstance(value, dict):
|
||||
return [value]
|
||||
return []
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FieldEvidence:
|
||||
field_name: str
|
||||
method: str
|
||||
confidence: float
|
||||
evidence: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ExtractedJob:
|
||||
url: str
|
||||
title: str
|
||||
employer_name: str = ""
|
||||
external_id: str = ""
|
||||
location_text: str = ""
|
||||
region: str = ""
|
||||
postal_code: str = ""
|
||||
country: str = ""
|
||||
description_html: str = ""
|
||||
description_text: str = ""
|
||||
language: str = ""
|
||||
date_posted: str = ""
|
||||
valid_through: str = ""
|
||||
employment_types: list[str] = field(default_factory=list)
|
||||
workplace_type: str = ""
|
||||
compensation: dict[str, Any] = field(default_factory=dict)
|
||||
skills_required: list[str] = field(default_factory=list)
|
||||
skills_preferred: list[str] = field(default_factory=list)
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
evidence: list[FieldEvidence] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ExtractionResult:
|
||||
jobs: list[ExtractedJob]
|
||||
parser_key: str
|
||||
parser_version: str
|
||||
confidence: float
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class SourceAdapter(Protocol):
|
||||
parser_key: str
|
||||
parser_version: str
|
||||
|
||||
def extract(self, content: str, *, url: str) -> ExtractionResult: ...
|
||||
@@ -0,0 +1,100 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
from email import policy
|
||||
from email.message import Message
|
||||
from email.parser import BytesParser
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from apps.sources.services.canonicalize import canonicalize_url
|
||||
|
||||
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
||||
|
||||
URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I)
|
||||
SKIP_TEXT = re.compile(r"unsubscribe|afmelden|uitschrijven|privacy|view in browser", re.I)
|
||||
|
||||
|
||||
class EmailAlertAdapter:
|
||||
parser_key = "email-alert"
|
||||
parser_version = "1.0.0"
|
||||
|
||||
@staticmethod
|
||||
def _decode_parts(message: Message) -> tuple[str, str]:
|
||||
plain_parts: list[str] = []
|
||||
html_parts: list[str] = []
|
||||
for part in message.walk() if message.is_multipart() else [message]:
|
||||
disposition = str(part.get("Content-Disposition") or "")
|
||||
if "attachment" in disposition.lower():
|
||||
continue
|
||||
content_type = part.get_content_type()
|
||||
try:
|
||||
payload = part.get_content()
|
||||
except Exception:
|
||||
raw = part.get_payload(decode=True) or b""
|
||||
payload = raw.decode(part.get_content_charset() or "utf-8", errors="replace")
|
||||
if content_type == "text/plain":
|
||||
plain_parts.append(str(payload))
|
||||
elif content_type == "text/html":
|
||||
html_parts.append(str(payload))
|
||||
return "\n".join(plain_parts), "\n".join(html_parts)
|
||||
|
||||
def extract_message(self, raw_message: bytes) -> ExtractionResult:
|
||||
message = BytesParser(policy=policy.default).parsebytes(raw_message)
|
||||
plain, html_body = self._decode_parts(message)
|
||||
candidates: list[tuple[str, str]] = []
|
||||
if html_body:
|
||||
soup = BeautifulSoup(html_body, "lxml")
|
||||
for anchor in soup.find_all("a", href=True):
|
||||
label = " ".join(anchor.get_text(" ", strip=True).split())
|
||||
href = html.unescape(str(anchor["href"]).strip())
|
||||
if not href.lower().startswith(("http://", "https://")):
|
||||
continue
|
||||
if SKIP_TEXT.search(label) or SKIP_TEXT.search(href):
|
||||
continue
|
||||
candidates.append((label, href))
|
||||
for href in URL_RE.findall(plain):
|
||||
clean_href = href.rstrip(".,);]")
|
||||
if SKIP_TEXT.search(clean_href):
|
||||
continue
|
||||
candidates.append(("", clean_href))
|
||||
|
||||
jobs: list[ExtractedJob] = []
|
||||
seen: set[str] = set()
|
||||
subject = str(message.get("subject") or "Vacature uit e-mail").strip()
|
||||
for label, href in candidates:
|
||||
canonical = canonicalize_url(urljoin("https://invalid.local/", href))
|
||||
if not canonical or canonical in seen:
|
||||
continue
|
||||
seen.add(canonical)
|
||||
hostname = urlsplit(canonical).hostname or ""
|
||||
title = label if len(label) >= 4 else subject
|
||||
if len(title) > 300:
|
||||
title = title[:300]
|
||||
jobs.append(
|
||||
ExtractedJob(
|
||||
url=canonical,
|
||||
title=title,
|
||||
employer_name="",
|
||||
description_text=plain[:5000]
|
||||
or BeautifulSoup(html_body, "lxml").get_text("\n", strip=True)[:5000],
|
||||
raw={
|
||||
"email_subject": subject,
|
||||
"email_sender": str(message.get("from") or ""),
|
||||
"target_domain": hostname,
|
||||
},
|
||||
evidence=[FieldEvidence("url", "email-anchor", 0.75, label[:240])],
|
||||
)
|
||||
)
|
||||
return ExtractionResult(
|
||||
jobs,
|
||||
self.parser_key,
|
||||
self.parser_version,
|
||||
0.65 if jobs else 0.0,
|
||||
[] if jobs else ["Geen vacaturelinks in e-mail gevonden"],
|
||||
)
|
||||
|
||||
def extract(self, content: str, *, url: str = "") -> ExtractionResult:
|
||||
return self.extract_message(content.encode("utf-8", errors="replace"))
|
||||
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
||||
|
||||
LABEL_PATTERNS = {
|
||||
"location": re.compile(r"^(locatie|location|lieu|plaats|standplaats)\s*:?$", re.I),
|
||||
"employer": re.compile(r"^(werkgever|employer|company|organisatie|société)\s*:?$", re.I),
|
||||
}
|
||||
|
||||
|
||||
class GenericHtmlAdapter:
|
||||
parser_key = "generic-html"
|
||||
parser_version = "1.0.0"
|
||||
|
||||
@staticmethod
|
||||
def _meta(soup: BeautifulSoup, *names: str) -> str:
|
||||
for name in names:
|
||||
tag = soup.find("meta", attrs={"property": name}) or soup.find(
|
||||
"meta", attrs={"name": name}
|
||||
)
|
||||
if tag and tag.get("content"):
|
||||
return str(tag["content"]).strip()
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _label_value(soup: BeautifulSoup, pattern: re.Pattern[str]) -> str:
|
||||
label = soup.find(string=lambda value: bool(value and pattern.match(value.strip())))
|
||||
if not label:
|
||||
return ""
|
||||
parent = label.parent
|
||||
if not parent:
|
||||
return ""
|
||||
sibling = parent.find_next_sibling()
|
||||
if sibling:
|
||||
return sibling.get_text(" ", strip=True)
|
||||
text = parent.get_text(" ", strip=True)
|
||||
return pattern.sub("", text).strip(" :-")
|
||||
|
||||
def extract(self, content: str, *, url: str) -> ExtractionResult:
|
||||
soup = BeautifulSoup(content, "lxml")
|
||||
for element in soup(["script", "style", "noscript", "template"]):
|
||||
element.decompose()
|
||||
h1 = soup.find("h1")
|
||||
title = (h1.get_text(" ", strip=True) if h1 else "") or self._meta(
|
||||
soup, "og:title", "twitter:title"
|
||||
)
|
||||
if not title and soup.title:
|
||||
title = soup.title.get_text(" ", strip=True)
|
||||
title = re.sub(r"\s+[|\\u2013\\u2014-]\s+.*$", "", title).strip()
|
||||
if not title:
|
||||
return ExtractionResult([], self.parser_key, self.parser_version, 0.0, ["Geen titel"])
|
||||
|
||||
employer = self._meta(soup, "og:site_name", "application-name") or self._label_value(
|
||||
soup, LABEL_PATTERNS["employer"]
|
||||
)
|
||||
location = self._label_value(soup, LABEL_PATTERNS["location"])
|
||||
main = soup.find("main") or soup.find("article") or soup.body
|
||||
description_html = str(main) if main else ""
|
||||
description_text = (
|
||||
main.get_text("\n", strip=True) if main else soup.get_text("\n", strip=True)
|
||||
)
|
||||
canonical = soup.find("link", rel=lambda value: value and "canonical" in value)
|
||||
job_url = (
|
||||
urljoin(url, canonical.get("href")) if canonical and canonical.get("href") else url
|
||||
)
|
||||
evidence = [
|
||||
FieldEvidence("title", "html-heading", 0.78, title[:240]),
|
||||
FieldEvidence("employer_name", "html-meta-or-label", 0.65, employer[:240]),
|
||||
FieldEvidence("location_text", "html-label", 0.62, location[:240]),
|
||||
FieldEvidence("description", "html-main", 0.70, description_text[:300]),
|
||||
]
|
||||
job = ExtractedJob(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer_name=employer,
|
||||
location_text=location,
|
||||
description_html=description_html,
|
||||
description_text=description_text,
|
||||
raw={"generic_html": True},
|
||||
evidence=evidence,
|
||||
)
|
||||
return ExtractionResult([job], self.parser_key, self.parser_version, 0.70)
|
||||
@@ -0,0 +1,166 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
||||
|
||||
|
||||
class JsonLdJobPostingAdapter:
|
||||
parser_key = "jsonld-jobposting"
|
||||
parser_version = "1.0.0"
|
||||
|
||||
@staticmethod
|
||||
def _is_jobposting(value: Any) -> bool:
|
||||
types = value if isinstance(value, list) else [value]
|
||||
return any(str(item).lower() == "jobposting" for item in types)
|
||||
|
||||
def _walk(self, value: Any) -> Iterable[dict[str, Any]]:
|
||||
if isinstance(value, dict):
|
||||
if self._is_jobposting(value.get("@type")):
|
||||
yield value
|
||||
graph = value.get("@graph")
|
||||
if graph is not None:
|
||||
yield from self._walk(graph)
|
||||
for child in value.values():
|
||||
if isinstance(child, dict | list):
|
||||
yield from self._walk(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
yield from self._walk(child)
|
||||
|
||||
@staticmethod
|
||||
def _name(value: Any) -> str:
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
if isinstance(value, dict):
|
||||
return str(value.get("name") or value.get("legalName") or "").strip()
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _location(value: Any) -> tuple[str, str, str, str]:
|
||||
locations = value if isinstance(value, list) else [value]
|
||||
parts: list[str] = []
|
||||
region = postal = country = ""
|
||||
for location in locations:
|
||||
if not isinstance(location, dict):
|
||||
continue
|
||||
address = location.get("address", location)
|
||||
if isinstance(address, str):
|
||||
parts.append(address)
|
||||
continue
|
||||
if not isinstance(address, dict):
|
||||
continue
|
||||
locality = str(address.get("addressLocality") or "").strip()
|
||||
region = region or str(address.get("addressRegion") or "").strip()
|
||||
postal = postal or str(address.get("postalCode") or "").strip()
|
||||
country_value = address.get("addressCountry")
|
||||
country = (
|
||||
country
|
||||
or JsonLdJobPostingAdapter._name(country_value)
|
||||
or str(country_value or "").strip()
|
||||
)
|
||||
label = ", ".join(part for part in [locality, region, postal, country] if part)
|
||||
if label:
|
||||
parts.append(label)
|
||||
return " | ".join(dict.fromkeys(parts)), region, postal, country
|
||||
|
||||
@staticmethod
|
||||
def _employment_types(value: Any) -> list[str]:
|
||||
if isinstance(value, list):
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
if value:
|
||||
return [str(value).strip()]
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _identifier(value: Any) -> str:
|
||||
if isinstance(value, dict):
|
||||
return str(value.get("value") or value.get("name") or "").strip()
|
||||
return str(value or "").strip()
|
||||
|
||||
@staticmethod
|
||||
def _salary(value: Any) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
result: dict[str, Any] = {}
|
||||
currency = value.get("currency")
|
||||
if currency:
|
||||
result["currency"] = currency
|
||||
raw_value = value.get("value")
|
||||
if isinstance(raw_value, dict):
|
||||
for key in ("minValue", "maxValue", "value", "unitText"):
|
||||
if key in raw_value:
|
||||
result[key] = raw_value[key]
|
||||
elif raw_value is not None:
|
||||
result["value"] = raw_value
|
||||
return result
|
||||
|
||||
def extract(self, content: str, *, url: str) -> ExtractionResult:
|
||||
soup = BeautifulSoup(content, "lxml")
|
||||
records: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
for script in soup.find_all(
|
||||
"script", attrs={"type": lambda value: value and "ld+json" in value}
|
||||
):
|
||||
raw = script.string or script.get_text("", strip=True)
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
warnings.append("Ongeldige JSON-LD overgeslagen")
|
||||
continue
|
||||
records.extend(self._walk(payload))
|
||||
|
||||
jobs: list[ExtractedJob] = []
|
||||
for record in records:
|
||||
title = str(record.get("title") or record.get("name") or "").strip()
|
||||
if not title:
|
||||
warnings.append("JobPosting zonder titel overgeslagen")
|
||||
continue
|
||||
description_html = str(record.get("description") or "").strip()
|
||||
description_text = BeautifulSoup(description_html, "lxml").get_text("\n", strip=True)
|
||||
employer = self._name(record.get("hiringOrganization"))
|
||||
location, region, postal, country = self._location(record.get("jobLocation"))
|
||||
workplace_type = str(record.get("jobLocationType") or "").strip()
|
||||
if not location and record.get("applicantLocationRequirements"):
|
||||
location, region, postal, country = self._location(
|
||||
record.get("applicantLocationRequirements")
|
||||
)
|
||||
job_url = str(record.get("url") or url).strip()
|
||||
job_url = urljoin(url, job_url)
|
||||
evidence = [
|
||||
FieldEvidence("title", "jsonld", 0.98, title[:240]),
|
||||
FieldEvidence("employer_name", "jsonld", 0.95, employer[:240]),
|
||||
FieldEvidence("location_text", "jsonld", 0.92, location[:240]),
|
||||
FieldEvidence("description", "jsonld", 0.95, description_text[:300]),
|
||||
]
|
||||
jobs.append(
|
||||
ExtractedJob(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer_name=employer,
|
||||
external_id=self._identifier(record.get("identifier")),
|
||||
location_text=location,
|
||||
region=region,
|
||||
postal_code=postal,
|
||||
country=country,
|
||||
description_html=description_html,
|
||||
description_text=description_text,
|
||||
language=str(record.get("inLanguage") or "").strip(),
|
||||
date_posted=str(record.get("datePosted") or "").strip(),
|
||||
valid_through=str(record.get("validThrough") or "").strip(),
|
||||
employment_types=self._employment_types(record.get("employmentType")),
|
||||
workplace_type=workplace_type,
|
||||
compensation=self._salary(record.get("baseSalary")),
|
||||
raw=record,
|
||||
evidence=evidence,
|
||||
)
|
||||
)
|
||||
confidence = 0.95 if jobs else 0.0
|
||||
return ExtractionResult(jobs, self.parser_key, self.parser_version, confidence, warnings)
|
||||
@@ -0,0 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from apps.sources.models import RawDocument
|
||||
|
||||
from .base import ExtractionResult
|
||||
from .ats import (
|
||||
GreenhouseAdapter,
|
||||
LeverAdapter,
|
||||
RecruiteeAdapter,
|
||||
SmartRecruitersAdapter,
|
||||
WorkableAdapter,
|
||||
)
|
||||
from .generic_html import GenericHtmlAdapter
|
||||
from .jsonld import JsonLdJobPostingAdapter
|
||||
from .rss import RssAdapter
|
||||
|
||||
|
||||
class AdapterRegistry:
|
||||
def __init__(self) -> None:
|
||||
self.greenhouse = GreenhouseAdapter()
|
||||
self.lever = LeverAdapter()
|
||||
self.recruitee = RecruiteeAdapter()
|
||||
self.smartrecruiters = SmartRecruitersAdapter()
|
||||
self.workable = WorkableAdapter()
|
||||
self.jsonld = JsonLdJobPostingAdapter()
|
||||
self.generic = GenericHtmlAdapter()
|
||||
self.rss = RssAdapter()
|
||||
self.providers = [
|
||||
self.greenhouse,
|
||||
self.lever,
|
||||
self.recruitee,
|
||||
self.smartrecruiters,
|
||||
self.workable,
|
||||
]
|
||||
|
||||
def extract(self, document: RawDocument) -> ExtractionResult:
|
||||
content = document.body_text
|
||||
url = document.final_url or document.url
|
||||
content_type = (document.content_type or "").lower()
|
||||
if document.kind == RawDocument.Kind.XML or "rss" in content_type or "atom" in content_type:
|
||||
return self.rss.extract(content, url=url)
|
||||
|
||||
for provider in self.providers:
|
||||
if provider._supports_url(url):
|
||||
result = provider.extract(content, url=url)
|
||||
if any(
|
||||
msg in result.warnings
|
||||
for msg in ("Geen parseerbare ATS-response", "Geen herkenbare ATS-markup voor deze adapter")
|
||||
):
|
||||
continue
|
||||
return result
|
||||
|
||||
jsonld_result = self.jsonld.extract(content, url=url)
|
||||
if jsonld_result.jobs:
|
||||
return jsonld_result
|
||||
generic_result = self.generic.extract(content, url=url)
|
||||
generic_result.warnings = jsonld_result.warnings + generic_result.warnings
|
||||
return generic_result
|
||||
|
||||
|
||||
registry = AdapterRegistry()
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from time import mktime
|
||||
|
||||
import feedparser
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
||||
|
||||
|
||||
class RssAdapter:
|
||||
parser_key = "rss-atom"
|
||||
parser_version = "1.0.0"
|
||||
|
||||
def extract(self, content: str, *, url: str) -> ExtractionResult:
|
||||
feed = feedparser.parse(content)
|
||||
jobs: list[ExtractedJob] = []
|
||||
for entry in feed.entries:
|
||||
title = str(entry.get("title") or "").strip()
|
||||
link = str(entry.get("link") or "").strip()
|
||||
if not title or not link:
|
||||
continue
|
||||
summary = str(entry.get("summary") or entry.get("description") or "")
|
||||
text = BeautifulSoup(summary, "lxml").get_text("\n", strip=True)
|
||||
published = ""
|
||||
if entry.get("published_parsed"):
|
||||
published = datetime.fromtimestamp(mktime(entry.published_parsed)).isoformat()
|
||||
jobs.append(
|
||||
ExtractedJob(
|
||||
url=link,
|
||||
title=title,
|
||||
external_id=str(entry.get("id") or ""),
|
||||
description_html=summary,
|
||||
description_text=text,
|
||||
date_posted=published,
|
||||
raw=dict(entry),
|
||||
evidence=[
|
||||
FieldEvidence("title", "rss", 0.88, title[:240]),
|
||||
FieldEvidence("description", "rss", 0.80, text[:300]),
|
||||
],
|
||||
)
|
||||
)
|
||||
warnings: list[str] = []
|
||||
if getattr(feed, "bozo", False):
|
||||
warnings.append(str(getattr(feed, "bozo_exception", "Ongeldige feed")))
|
||||
return ExtractionResult(
|
||||
jobs,
|
||||
self.parser_key,
|
||||
self.parser_version,
|
||||
0.82 if jobs else 0.0,
|
||||
warnings,
|
||||
)
|
||||
Reference in New Issue
Block a user