254 lines
9.9 KiB
Python
254 lines
9.9 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
from urllib.parse import parse_qs, urljoin, urlsplit
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
|
|
|
|
|
class MolRegionEmployerAdapter:
|
|
"""Extract reviewed employer listings around postcode 2400 without detail fetches."""
|
|
|
|
parser_key = "regional-mol-employers"
|
|
parser_version = "1.0.0"
|
|
thomas_more_company_guid = "eab9ca13-ee10-4504-8a87-a785d0b037ef"
|
|
supported_routes = {
|
|
"www.sckcen.be": {"/nl/carriere/vacatures"},
|
|
"cipalschaubroeck.teamtailor.com": {"/jobs"},
|
|
"www.vanroey.be": {"/en/job-overview"},
|
|
"netropolix.recruitee.com": {"/"},
|
|
"jobpage.cvwarehouse.com": {"/"},
|
|
}
|
|
|
|
def _route(self, url: str) -> tuple[str, str] | None:
|
|
parts = urlsplit(url)
|
|
host = (parts.hostname or "").lower()
|
|
path = parts.path.rstrip("/") or "/"
|
|
if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
|
|
return None
|
|
if host == "jobpage.cvwarehouse.com":
|
|
query = parse_qs(parts.query)
|
|
if query.get("companyGuid") != [self.thomas_more_company_guid] or "job" in query:
|
|
return None
|
|
return host, path
|
|
|
|
def _supports_url(self, url: str) -> bool:
|
|
return self._route(url) is not None
|
|
|
|
@staticmethod
|
|
def _same_host_url(base_url: str, href: str) -> str:
|
|
job_url = urljoin(base_url, href)
|
|
if urlsplit(job_url).scheme != "https":
|
|
return ""
|
|
if (urlsplit(job_url).hostname or "").lower() != (
|
|
urlsplit(base_url).hostname or ""
|
|
).lower():
|
|
return ""
|
|
return job_url
|
|
|
|
@staticmethod
|
|
def _job(
|
|
*,
|
|
url: str,
|
|
title: str,
|
|
employer: str,
|
|
location: str,
|
|
postal_code: str,
|
|
description: str,
|
|
external_id: str,
|
|
evidence_source: str,
|
|
) -> ExtractedJob:
|
|
return ExtractedJob(
|
|
url=url,
|
|
title=title,
|
|
employer_name=employer,
|
|
external_id=external_id,
|
|
location_text=location,
|
|
postal_code=postal_code,
|
|
region="Antwerpen",
|
|
country="BE",
|
|
description_text=description,
|
|
raw={"regional_listing": evidence_source, "region_center": "2400 Mol"},
|
|
evidence=[
|
|
FieldEvidence("title", evidence_source, 0.94, title[:240]),
|
|
FieldEvidence("employer_name", "reviewed-source", 0.96, employer),
|
|
FieldEvidence("location_text", "mol-region-review", 0.82, location),
|
|
],
|
|
)
|
|
|
|
def _sck_cen(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
|
jobs = []
|
|
for link in soup.select("article a[href]"):
|
|
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
|
path = urlsplit(job_url).path if job_url else ""
|
|
if not path.startswith("/nl/carriere/vacatures/"):
|
|
continue
|
|
title = link.get_text(" ", strip=True)
|
|
if not title:
|
|
continue
|
|
card = link.find_parent("article")
|
|
description = card.get_text(" ", strip=True) if card else title
|
|
jobs.append(
|
|
self._job(
|
|
url=job_url,
|
|
title=title,
|
|
employer="SCK CEN",
|
|
location="Mol, Antwerpen",
|
|
postal_code="2400",
|
|
description=description,
|
|
external_id=path.rstrip("/").rsplit("/", 1)[-1],
|
|
evidence_source="sck-cen-vacancy-card",
|
|
)
|
|
)
|
|
return jobs
|
|
|
|
def _cipal(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
|
jobs = []
|
|
for card in soup.select("li"):
|
|
card_text = card.get_text(" ", strip=True)
|
|
if "Westerlo" not in card_text and "Geel" not in card_text:
|
|
continue
|
|
link = card.find("a", href=True)
|
|
if not link:
|
|
continue
|
|
title = link.get_text(" ", strip=True)
|
|
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
|
path = urlsplit(job_url).path if job_url else ""
|
|
match = re.fullmatch(r"/jobs/(\d+)-[^/]+", path.rstrip("/"))
|
|
if not title or not match:
|
|
continue
|
|
location = "Geel, Antwerpen" if "Geel" in card_text else "Westerlo, Antwerpen"
|
|
jobs.append(
|
|
self._job(
|
|
url=job_url,
|
|
title=title,
|
|
employer="Cipal Schaubroeck",
|
|
location=location,
|
|
postal_code="2440" if "Geel" in card_text else "2260",
|
|
description=card_text,
|
|
external_id=match.group(1),
|
|
evidence_source="cipal-teamtailor-card",
|
|
)
|
|
)
|
|
return jobs
|
|
|
|
def _vanroey(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
|
jobs = []
|
|
for heading in soup.select("h3.elementor-heading-title"):
|
|
link = heading.find("a", href=True)
|
|
if not link:
|
|
continue
|
|
title = link.get_text(" ", strip=True)
|
|
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
|
path = urlsplit(job_url).path if job_url else ""
|
|
if not title or not re.fullmatch(r"/en/job/[^/]+/", path):
|
|
continue
|
|
slug = path.rstrip("/").rsplit("/", 1)[-1]
|
|
if "oost-vlaanderen" in slug:
|
|
continue
|
|
card = heading.find_parent("section") or heading.parent
|
|
description = card.get_text(" ", strip=True) if card else title
|
|
jobs.append(
|
|
self._job(
|
|
url=job_url,
|
|
title=title,
|
|
employer="VanRoey",
|
|
location="Turnhout/Geel (hybride; controleer vacaturedetail)",
|
|
postal_code="",
|
|
description=description,
|
|
external_id=slug,
|
|
evidence_source="vanroey-job-card",
|
|
)
|
|
)
|
|
return jobs
|
|
|
|
def _netropolix(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
|
jobs = []
|
|
for link in soup.find_all("a", href=True):
|
|
title = link.get_text(" ", strip=True)
|
|
if "Geel" not in title:
|
|
continue
|
|
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
|
path = urlsplit(job_url).path if job_url else ""
|
|
if not re.fullmatch(r"/o/[^/]+", path.rstrip("/")):
|
|
continue
|
|
card = link.find_parent("div")
|
|
description = card.get_text(" ", strip=True) if card else title
|
|
jobs.append(
|
|
self._job(
|
|
url=job_url,
|
|
title=title,
|
|
employer="NTX (Netropolix)",
|
|
location="Geel, Antwerpen",
|
|
postal_code="2440",
|
|
description=description,
|
|
external_id=path.rstrip("/").rsplit("/", 1)[-1],
|
|
evidence_source="netropolix-recruitee-card",
|
|
)
|
|
)
|
|
return jobs
|
|
|
|
def _thomas_more(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
|
jobs = []
|
|
for link in soup.select("a.jobLink[data-item='readmore'][data-jobid][href]"):
|
|
title = link.get_text(" ", strip=True)
|
|
title_casefold = title.casefold()
|
|
if not any(place in title_casefold for place in ("geel", "turnhout", "vorselaar")):
|
|
continue
|
|
external_id = str(link.get("data-jobid") or "")
|
|
if not external_id.isdigit():
|
|
continue
|
|
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
|
query = parse_qs(urlsplit(job_url).query) if job_url else {}
|
|
if query.get("companyGuid") != [self.thomas_more_company_guid] or query.get("job") != [
|
|
external_id
|
|
]:
|
|
continue
|
|
if "turnhout" in title_casefold:
|
|
location, postal_code = "Turnhout, Antwerpen", "2300"
|
|
elif "vorselaar" in title_casefold:
|
|
location, postal_code = "Vorselaar, Antwerpen", "2290"
|
|
else:
|
|
location, postal_code = "Geel, Antwerpen", "2440"
|
|
jobs.append(
|
|
self._job(
|
|
url=job_url,
|
|
title=title,
|
|
employer="Thomas More",
|
|
location=location,
|
|
postal_code=postal_code,
|
|
description=f"Publieke regionale vacature bij Thomas More: {title}.",
|
|
external_id=external_id,
|
|
evidence_source="thomas-more-cvwarehouse-card",
|
|
)
|
|
)
|
|
return jobs
|
|
|
|
def extract(self, content: str, *, url: str) -> ExtractionResult:
|
|
route = self._route(url)
|
|
if route is None:
|
|
return ExtractionResult(
|
|
[], self.parser_key, self.parser_version, 0.0, ["Onherkende Mol-regiobron"]
|
|
)
|
|
host, _ = route
|
|
soup = BeautifulSoup(content, "lxml")
|
|
extractors = {
|
|
"www.sckcen.be": self._sck_cen,
|
|
"cipalschaubroeck.teamtailor.com": self._cipal,
|
|
"www.vanroey.be": self._vanroey,
|
|
"netropolix.recruitee.com": self._netropolix,
|
|
"jobpage.cvwarehouse.com": self._thomas_more,
|
|
}
|
|
jobs = extractors[host](soup, url)
|
|
unique_jobs = list({job.url: job for job in jobs}.values())
|
|
warnings = [] if unique_jobs else ["Geen actuele vacatures rond Mol gevonden"]
|
|
return ExtractionResult(
|
|
unique_jobs,
|
|
self.parser_key,
|
|
self.parser_version,
|
|
0.9 if unique_jobs else 0.0,
|
|
warnings,
|
|
)
|