This commit is contained in:
@@ -8,6 +8,9 @@ from .ats import (
|
||||
from .base import ExtractedJob, ExtractionResult
|
||||
from .generic_html import GenericHtmlAdapter
|
||||
from .jsonld import JsonLdJobPostingAdapter
|
||||
from .kempen import KempenEmployerAdapter
|
||||
from .mol_region import MolRegionEmployerAdapter
|
||||
from .regional import LocalEmployerListingAdapter
|
||||
from .rss import RssAdapter
|
||||
|
||||
__all__ = [
|
||||
@@ -16,7 +19,10 @@ __all__ = [
|
||||
"GenericHtmlAdapter",
|
||||
"GreenhouseAdapter",
|
||||
"JsonLdJobPostingAdapter",
|
||||
"KempenEmployerAdapter",
|
||||
"LeverAdapter",
|
||||
"LocalEmployerListingAdapter",
|
||||
"MolRegionEmployerAdapter",
|
||||
"RecruiteeAdapter",
|
||||
"RssAdapter",
|
||||
"SmartRecruitersAdapter",
|
||||
|
||||
@@ -169,16 +169,20 @@ class _AtsAdapter(ABC):
|
||||
def _extract_location(self, record: dict[str, object]) -> tuple[str, str, str, str]:
|
||||
location_raw = _first_text(
|
||||
record,
|
||||
"location",
|
||||
"location.city",
|
||||
"location.name",
|
||||
"location.address",
|
||||
"location.raw",
|
||||
"categories.location",
|
||||
"city",
|
||||
"cityName",
|
||||
"place",
|
||||
"office",
|
||||
"location.address",
|
||||
"address",
|
||||
"data.location",
|
||||
"officeLocation",
|
||||
"locationName",
|
||||
"location",
|
||||
)
|
||||
if not location_raw:
|
||||
location_raw = _first_text(record, "data.city", "data.location")
|
||||
@@ -223,6 +227,13 @@ class _AtsAdapter(ABC):
|
||||
"remote_type",
|
||||
"job_type",
|
||||
).lower()
|
||||
if not workplace_raw:
|
||||
if record.get("hybrid") is True:
|
||||
return "hybrid"
|
||||
if record.get("remote") is True:
|
||||
return "remote"
|
||||
if record.get("on_site") is True:
|
||||
return "on_site"
|
||||
if "remote" in workplace_raw:
|
||||
return "remote" if "hybrid" not in workplace_raw else "hybrid"
|
||||
if "hybrid" in workplace_raw:
|
||||
@@ -256,18 +267,21 @@ class _AtsAdapter(ABC):
|
||||
"url",
|
||||
"jobUrl",
|
||||
"job_url",
|
||||
"hostedUrl",
|
||||
"applyUrl",
|
||||
"link",
|
||||
"absoluteUrl",
|
||||
"careers_url",
|
||||
"data.url",
|
||||
)
|
||||
employer_name = _first_text(
|
||||
record,
|
||||
"company_name",
|
||||
"company.name",
|
||||
"company",
|
||||
"employer",
|
||||
"organization",
|
||||
"organizationName",
|
||||
"company.name",
|
||||
"department",
|
||||
"hiringOrganization",
|
||||
)
|
||||
@@ -290,6 +304,7 @@ class _AtsAdapter(ABC):
|
||||
"published_at",
|
||||
"created_at",
|
||||
"jobCreated",
|
||||
"releasedDate",
|
||||
)
|
||||
valid_through = _first_text(
|
||||
record,
|
||||
@@ -308,6 +323,9 @@ class _AtsAdapter(ABC):
|
||||
"jobType",
|
||||
"type",
|
||||
"data.employmentType",
|
||||
"employment_type_code",
|
||||
"categories.commitment",
|
||||
"typeOfEmployment.label",
|
||||
)
|
||||
)
|
||||
workplace_type = self._extract_workplace(record)
|
||||
@@ -428,13 +446,20 @@ class GreenhouseAdapter(_AtsAdapter):
|
||||
|
||||
class LeverAdapter(_AtsAdapter):
|
||||
parser_key = "ats-lever"
|
||||
source_hosts = ("jobs.lever.co",)
|
||||
source_hosts = (
|
||||
"jobs.lever.co",
|
||||
"jobs.eu.lever.co",
|
||||
"api.lever.co",
|
||||
"api.eu.lever.co",
|
||||
)
|
||||
support_markers = ("lever", "requisition", "posting")
|
||||
listing_paths = (("data",), ("jobs",), ("results",))
|
||||
detail_paths = (("data",), ("job",), ("position",), ("result",))
|
||||
closed_statuses = CLOSED_STATUSES | {"archived", "deleted"}
|
||||
|
||||
def _extract_records(self, payload):
|
||||
if isinstance(payload, list):
|
||||
return [item for item in payload if isinstance(item, dict)]
|
||||
for path in self.listing_paths:
|
||||
value = payload
|
||||
for key in path:
|
||||
@@ -466,7 +491,7 @@ class RecruiteeAdapter(_AtsAdapter):
|
||||
parser_key = "ats-recruitee"
|
||||
source_hosts = ("recruitee.com",)
|
||||
support_markers = ("recruitee", "career", "vacancy")
|
||||
listing_paths = (("jobs",), ("data", "jobs"), ("vacancies",))
|
||||
listing_paths = (("jobs",), ("offers",), ("data", "jobs"), ("vacancies",))
|
||||
detail_paths = (("job",), ("data", "job"), ("vacancy",), ("result",))
|
||||
closed_statuses = CLOSED_STATUSES | {"hidden", "paused"}
|
||||
|
||||
@@ -494,12 +519,26 @@ class RecruiteeAdapter(_AtsAdapter):
|
||||
|
||||
class SmartRecruitersAdapter(_AtsAdapter):
|
||||
parser_key = "ats-smartrecruiters"
|
||||
parser_version = "1.1.0"
|
||||
source_hosts = ("smartrecruiters.com",)
|
||||
support_markers = ("smartrecruiters", "smart recruiter")
|
||||
listing_paths = (("jobs",), ("data", "jobs"), ("results",))
|
||||
listing_paths = (("jobs",), ("data", "jobs"), ("results",), ("content",))
|
||||
detail_paths = (("job",), ("data", "job"), ("posting",), ("result",))
|
||||
closed_statuses = CLOSED_STATUSES | {"unpublished", "expired"}
|
||||
|
||||
def _to_job(self, record: dict[str, object], base_url: str) -> ExtractedJob:
|
||||
prepared = dict(record)
|
||||
if not _first_text(prepared, "url", "jobUrl", "hostedUrl", "applyUrl", "link"):
|
||||
company_identifier = _first_text(prepared, "company.identifier")
|
||||
posting_id = _first_text(prepared, "id", "uuid")
|
||||
if company_identifier and posting_id:
|
||||
prepared["url"] = (
|
||||
f"https://jobs.smartrecruiters.com/{company_identifier}/{posting_id}"
|
||||
)
|
||||
if _find_nested(prepared, "location.remote") is True:
|
||||
prepared["remote"] = True
|
||||
return super()._to_job(prepared, base_url)
|
||||
|
||||
def _extract_records(self, payload):
|
||||
for path in self.listing_paths:
|
||||
value = payload
|
||||
|
||||
@@ -9,17 +9,58 @@ from urllib.parse import urljoin, urlsplit
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from apps.sources.platforms import PLATFORM_ALERTS
|
||||
from apps.sources.services.canonicalize import canonicalize_url
|
||||
|
||||
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
||||
|
||||
URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I)
|
||||
SKIP_TEXT = re.compile(r"unsubscribe|afmelden|uitschrijven|privacy|view in browser", re.I)
|
||||
SKIP_TEXT = re.compile(
|
||||
r"unsubscribe|afmelden|uitschrijven|privacy|view in browser|bekijk online|"
|
||||
r"account|aanmelden|inloggen|login|voorkeuren|preferences|voorwaarden|terms|"
|
||||
r"contact|help|hulp|over ons|about us",
|
||||
re.I,
|
||||
)
|
||||
SKIP_PATH = re.compile(
|
||||
r"/(?:unsubscribe|uitschrijven|afmelden|privacy|account|login|signin|preferences|"
|
||||
r"settings|terms|legal|help|contact)(?:/|$)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def _matches_domain(hostname: str, domains: tuple[str, ...]) -> bool:
|
||||
return any(hostname == domain or hostname.endswith(f".{domain}") for domain in domains)
|
||||
|
||||
|
||||
class EmailAlertAdapter:
|
||||
parser_key = "email-alert"
|
||||
parser_version = "1.0.0"
|
||||
parser_version = "1.1.0"
|
||||
|
||||
@staticmethod
|
||||
def _provider_for_candidates(candidates: list[tuple[str, str]]) -> str:
|
||||
hostnames = {
|
||||
(urlsplit(href).hostname or "").lower()
|
||||
for _, href in candidates
|
||||
if href.lower().startswith(("http://", "https://"))
|
||||
}
|
||||
for provider, alert in PLATFORM_ALERTS.items():
|
||||
if any(_matches_domain(hostname, alert.domains) for hostname in hostnames):
|
||||
return provider
|
||||
return "other"
|
||||
|
||||
@staticmethod
|
||||
def _is_expected_platform_link(*, label: str, href: str, provider: str) -> bool:
|
||||
alert = PLATFORM_ALERTS.get(provider)
|
||||
if alert is None or len(label.strip()) < 4:
|
||||
return False
|
||||
parsed = urlsplit(href)
|
||||
hostname = (parsed.hostname or "").lower()
|
||||
return (
|
||||
parsed.scheme.lower() == "https"
|
||||
and _matches_domain(hostname, alert.domains)
|
||||
and not SKIP_TEXT.search(label)
|
||||
and not SKIP_PATH.search(parsed.path)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _decode_parts(message: Message) -> tuple[str, str]:
|
||||
@@ -41,7 +82,9 @@ class EmailAlertAdapter:
|
||||
html_parts.append(str(payload))
|
||||
return "\n".join(plain_parts), "\n".join(html_parts)
|
||||
|
||||
def extract_message(self, raw_message: bytes) -> ExtractionResult:
|
||||
def extract_message(
|
||||
self, raw_message: bytes, *, expected_provider: str | None = None
|
||||
) -> ExtractionResult:
|
||||
message = BytesParser(policy=policy.default).parsebytes(raw_message)
|
||||
plain, html_body = self._decode_parts(message)
|
||||
candidates: list[tuple[str, str]] = []
|
||||
@@ -61,6 +104,18 @@ class EmailAlertAdapter:
|
||||
continue
|
||||
candidates.append(("", clean_href))
|
||||
|
||||
alert_provider = expected_provider or self._provider_for_candidates(candidates)
|
||||
if expected_provider:
|
||||
candidates = [
|
||||
(label, href)
|
||||
for label, href in candidates
|
||||
if self._is_expected_platform_link(
|
||||
label=label,
|
||||
href=href,
|
||||
provider=expected_provider,
|
||||
)
|
||||
]
|
||||
|
||||
jobs: list[ExtractedJob] = []
|
||||
seen: set[str] = set()
|
||||
subject = str(message.get("subject") or "Vacature uit e-mail").strip()
|
||||
@@ -84,6 +139,7 @@ class EmailAlertAdapter:
|
||||
"email_subject": subject,
|
||||
"email_sender": str(message.get("from") or ""),
|
||||
"target_domain": hostname,
|
||||
"alert_provider": alert_provider,
|
||||
},
|
||||
evidence=[FieldEvidence("url", "email-anchor", 0.75, label[:240])],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from urllib.parse import parse_qs, urljoin, urlsplit
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
||||
|
||||
|
||||
class KempenEmployerAdapter:
|
||||
"""Extract public employer lists around Mol from exact reviewed routes."""
|
||||
|
||||
parser_key = "regional-kempen-employers"
|
||||
parser_version = "1.0.0"
|
||||
supported_routes = {
|
||||
"ziekenhuisgeel.careersite.be": {"/nl/vacatures"},
|
||||
"geel.hro.be": {"/"},
|
||||
"jobs.turnhout.be": {"/"},
|
||||
"jobs.renotec.be": {"/nl/alle-jobs"},
|
||||
"ravago.softgarden.io": {"/en/vacancies"},
|
||||
"jobs.sanofi.com": {"/en/belgium"},
|
||||
"www.daf.com": {"/nl-nl/werken-bij-daf/vacatures"},
|
||||
}
|
||||
|
||||
def _route(self, url: str) -> tuple[str, str] | None:
|
||||
parts = urlsplit(url)
|
||||
host = (parts.hostname or "").lower()
|
||||
path = parts.path.rstrip("/") or "/"
|
||||
if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
|
||||
return None
|
||||
return host, path
|
||||
|
||||
def _supports_url(self, url: str) -> bool:
|
||||
return self._route(url) is not None
|
||||
|
||||
@staticmethod
|
||||
def _same_host_url(base_url: str, href: str) -> str:
|
||||
job_url = urljoin(base_url, href)
|
||||
parts = urlsplit(job_url)
|
||||
if parts.scheme != "https":
|
||||
return ""
|
||||
if (parts.hostname or "").lower() != (urlsplit(base_url).hostname or "").lower():
|
||||
return ""
|
||||
return job_url
|
||||
|
||||
@staticmethod
|
||||
def _job(
|
||||
*,
|
||||
url: str,
|
||||
title: str,
|
||||
employer: str,
|
||||
location: str,
|
||||
postal_code: str,
|
||||
description: str,
|
||||
external_id: str,
|
||||
evidence_source: str,
|
||||
) -> ExtractedJob:
|
||||
return ExtractedJob(
|
||||
url=url,
|
||||
title=title,
|
||||
employer_name=employer,
|
||||
external_id=external_id,
|
||||
location_text=location,
|
||||
postal_code=postal_code,
|
||||
region="Antwerpen",
|
||||
country="BE",
|
||||
description_text=description,
|
||||
raw={"regional_listing": evidence_source, "region_center": "2400 Mol"},
|
||||
evidence=[
|
||||
FieldEvidence("title", evidence_source, 0.94, title[:240]),
|
||||
FieldEvidence("employer_name", "reviewed-source", 0.97, employer),
|
||||
FieldEvidence("location_text", "kempen-location-marker", 0.9, location),
|
||||
],
|
||||
)
|
||||
|
||||
def _ziekenhuis_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for card in soup.select(".vacature-tegel"):
|
||||
link = card.select_one("a.vacature-tegel__link[href]")
|
||||
title_node = card.select_one(".vacature-tegel__titel")
|
||||
if not isinstance(link, Tag) or not isinstance(title_node, Tag):
|
||||
continue
|
||||
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
||||
match = re.fullmatch(r"/nl/vacature/(\d+)/[^/]+", urlsplit(job_url).path)
|
||||
title = title_node.get_text(" ", strip=True)
|
||||
if not title or not match:
|
||||
continue
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="Ziekenhuis Geel",
|
||||
location="Geel, Antwerpen",
|
||||
postal_code="2440",
|
||||
description=card.get_text(" ", strip=True),
|
||||
external_id=match.group(1),
|
||||
evidence_source="ziekenhuis-geel-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _stad_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for card in soup.select(".vacatureKader[data-href]"):
|
||||
href = str(card.get("data-href") or "")
|
||||
match = re.fullmatch(r"vacature\.php\?id=(\d+)", href)
|
||||
title_node = card.find("h4")
|
||||
title = title_node.get_text(" ", strip=True) if title_node else ""
|
||||
if not title or not match:
|
||||
continue
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=self._same_host_url(url, href),
|
||||
title=title,
|
||||
employer="Lokaal bestuur Geel",
|
||||
location="Geel, Antwerpen",
|
||||
postal_code="2440",
|
||||
description=card.get_text(" ", strip=True),
|
||||
external_id=match.group(1),
|
||||
evidence_source="stad-geel-hro-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _stad_turnhout(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for link in soup.find_all("a", href=True):
|
||||
href = str(link.get("href") or "")
|
||||
match = re.fullmatch(r"vacature\.php\?id=(\d+)", href)
|
||||
title_node = link.select_one(".block-update__body__title__inner")
|
||||
title = title_node.get_text(" ", strip=True) if title_node else ""
|
||||
if not title or not match:
|
||||
continue
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=self._same_host_url(url, href),
|
||||
title=title,
|
||||
employer="Stad Turnhout",
|
||||
location="Turnhout, Antwerpen",
|
||||
postal_code="2300",
|
||||
description=link.get_text(" ", strip=True),
|
||||
external_id=match.group(1),
|
||||
evidence_source="stad-turnhout-hro-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _renotec(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for card in soup.select(".s-tile.s-card"):
|
||||
card_text = card.get_text(" ", strip=True)
|
||||
if "Geel" not in card_text:
|
||||
continue
|
||||
title_node = card.find("h3")
|
||||
link = card.find("a", href=True)
|
||||
title = title_node.get_text(" ", strip=True) if title_node else ""
|
||||
job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
|
||||
parts = urlsplit(job_url)
|
||||
query = parse_qs(parts.query)
|
||||
external_id = (query.get("id") or [""])[0]
|
||||
if parts.path != "/nl/detail/" or not external_id.isdigit() or not title:
|
||||
continue
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="Group Renotec",
|
||||
location="Geel, Antwerpen (mogelijk meerdere werfregio's)",
|
||||
postal_code="2440",
|
||||
description=card_text,
|
||||
external_id=external_id,
|
||||
evidence_source="renotec-geel-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _ravago(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for card in soup.select(".matchElement"):
|
||||
locations = {
|
||||
node.get_text(" ", strip=True) for node in card.select(".location-view-item")
|
||||
}
|
||||
local_places = locations.intersection({"Arendonk", "Olen"})
|
||||
if not local_places:
|
||||
continue
|
||||
link = card.find("a", href=True)
|
||||
title = link.get_text(" ", strip=True) if link else ""
|
||||
job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
|
||||
match = re.fullmatch(r"/job/(\d+)/[^/]+/?", urlsplit(job_url).path)
|
||||
if not title or not match:
|
||||
continue
|
||||
place = "Arendonk" if "Arendonk" in local_places else "Olen"
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="Ravago",
|
||||
location=f"{place}, Antwerpen",
|
||||
postal_code="2370" if place == "Arendonk" else "2250",
|
||||
description=card.get_text(" ", strip=True),
|
||||
external_id=match.group(1),
|
||||
evidence_source="ravago-softgarden-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _sanofi(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for link in soup.select(".job-list a[data-job-id][href]"):
|
||||
location_node = link.select_one(".job-location")
|
||||
title_node = link.select_one(".job-title")
|
||||
location = location_node.get_text(" ", strip=True) if location_node else ""
|
||||
title = title_node.get_text(" ", strip=True) if title_node else ""
|
||||
external_id = str(link.get("data-job-id") or "")
|
||||
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
||||
if location != "Geel, Belgium" or not title or not external_id.isdigit():
|
||||
continue
|
||||
if not re.fullmatch(r"/en/job/geel/[^/]+/\d+/\d+", urlsplit(job_url).path):
|
||||
continue
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="Sanofi",
|
||||
location="Geel, Antwerpen",
|
||||
postal_code="2440",
|
||||
description=link.get_text(" ", strip=True),
|
||||
external_id=external_id,
|
||||
evidence_source="sanofi-belgium-geel-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _daf(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for card in soup.select("li.itemlist__item"):
|
||||
location_node = card.select_one(".js-vac-metalocation")
|
||||
title_link = card.select_one("a.js-vac-title[href]")
|
||||
location = location_node.get_text(" ", strip=True) if location_node else ""
|
||||
title = title_link.get_text(" ", strip=True) if title_link else ""
|
||||
job_url = (
|
||||
self._same_host_url(url, str(title_link.get("href") or "")) if title_link else ""
|
||||
)
|
||||
path = urlsplit(job_url).path
|
||||
prefix = "/nl-nl/werken-bij-daf/vacatures/"
|
||||
if location != "Westerlo" or not title or not path.startswith(prefix):
|
||||
continue
|
||||
slug = path.removeprefix(prefix).strip("/")
|
||||
if not slug or "/" in slug:
|
||||
continue
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="DAF Trucks",
|
||||
location="Westerlo, Antwerpen",
|
||||
postal_code="2260",
|
||||
description=card.get_text(" ", strip=True),
|
||||
external_id=slug,
|
||||
evidence_source="daf-westerlo-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def extract(self, content: str, *, url: str) -> ExtractionResult:
|
||||
route = self._route(url)
|
||||
if route is None:
|
||||
return ExtractionResult(
|
||||
[], self.parser_key, self.parser_version, 0.0, ["Onherkende Kempen-bron"]
|
||||
)
|
||||
host, _ = route
|
||||
extractors = {
|
||||
"ziekenhuisgeel.careersite.be": self._ziekenhuis_geel,
|
||||
"geel.hro.be": self._stad_geel,
|
||||
"jobs.turnhout.be": self._stad_turnhout,
|
||||
"jobs.renotec.be": self._renotec,
|
||||
"ravago.softgarden.io": self._ravago,
|
||||
"jobs.sanofi.com": self._sanofi,
|
||||
"www.daf.com": self._daf,
|
||||
}
|
||||
jobs = extractors[host](BeautifulSoup(content, "lxml"), url)
|
||||
unique_jobs = list({job.url: job for job in jobs if job.url}.values())
|
||||
return ExtractionResult(
|
||||
unique_jobs,
|
||||
self.parser_key,
|
||||
self.parser_version,
|
||||
0.92 if unique_jobs else 0.0,
|
||||
[] if unique_jobs else ["Geen actuele regionale vacatures gevonden"],
|
||||
)
|
||||
@@ -0,0 +1,253 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from urllib.parse import parse_qs, urljoin, urlsplit
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
||||
|
||||
|
||||
class MolRegionEmployerAdapter:
|
||||
"""Extract reviewed employer listings around postcode 2400 without detail fetches."""
|
||||
|
||||
parser_key = "regional-mol-employers"
|
||||
parser_version = "1.0.0"
|
||||
thomas_more_company_guid = "eab9ca13-ee10-4504-8a87-a785d0b037ef"
|
||||
supported_routes = {
|
||||
"www.sckcen.be": {"/nl/carriere/vacatures"},
|
||||
"cipalschaubroeck.teamtailor.com": {"/jobs"},
|
||||
"www.vanroey.be": {"/en/job-overview"},
|
||||
"netropolix.recruitee.com": {"/"},
|
||||
"jobpage.cvwarehouse.com": {"/"},
|
||||
}
|
||||
|
||||
def _route(self, url: str) -> tuple[str, str] | None:
|
||||
parts = urlsplit(url)
|
||||
host = (parts.hostname or "").lower()
|
||||
path = parts.path.rstrip("/") or "/"
|
||||
if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
|
||||
return None
|
||||
if host == "jobpage.cvwarehouse.com":
|
||||
query = parse_qs(parts.query)
|
||||
if query.get("companyGuid") != [self.thomas_more_company_guid] or "job" in query:
|
||||
return None
|
||||
return host, path
|
||||
|
||||
def _supports_url(self, url: str) -> bool:
|
||||
return self._route(url) is not None
|
||||
|
||||
@staticmethod
|
||||
def _same_host_url(base_url: str, href: str) -> str:
|
||||
job_url = urljoin(base_url, href)
|
||||
if urlsplit(job_url).scheme != "https":
|
||||
return ""
|
||||
if (urlsplit(job_url).hostname or "").lower() != (
|
||||
urlsplit(base_url).hostname or ""
|
||||
).lower():
|
||||
return ""
|
||||
return job_url
|
||||
|
||||
@staticmethod
|
||||
def _job(
|
||||
*,
|
||||
url: str,
|
||||
title: str,
|
||||
employer: str,
|
||||
location: str,
|
||||
postal_code: str,
|
||||
description: str,
|
||||
external_id: str,
|
||||
evidence_source: str,
|
||||
) -> ExtractedJob:
|
||||
return ExtractedJob(
|
||||
url=url,
|
||||
title=title,
|
||||
employer_name=employer,
|
||||
external_id=external_id,
|
||||
location_text=location,
|
||||
postal_code=postal_code,
|
||||
region="Antwerpen",
|
||||
country="BE",
|
||||
description_text=description,
|
||||
raw={"regional_listing": evidence_source, "region_center": "2400 Mol"},
|
||||
evidence=[
|
||||
FieldEvidence("title", evidence_source, 0.94, title[:240]),
|
||||
FieldEvidence("employer_name", "reviewed-source", 0.96, employer),
|
||||
FieldEvidence("location_text", "mol-region-review", 0.82, location),
|
||||
],
|
||||
)
|
||||
|
||||
def _sck_cen(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for link in soup.select("article a[href]"):
|
||||
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
||||
path = urlsplit(job_url).path if job_url else ""
|
||||
if not path.startswith("/nl/carriere/vacatures/"):
|
||||
continue
|
||||
title = link.get_text(" ", strip=True)
|
||||
if not title:
|
||||
continue
|
||||
card = link.find_parent("article")
|
||||
description = card.get_text(" ", strip=True) if card else title
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="SCK CEN",
|
||||
location="Mol, Antwerpen",
|
||||
postal_code="2400",
|
||||
description=description,
|
||||
external_id=path.rstrip("/").rsplit("/", 1)[-1],
|
||||
evidence_source="sck-cen-vacancy-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _cipal(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for card in soup.select("li"):
|
||||
card_text = card.get_text(" ", strip=True)
|
||||
if "Westerlo" not in card_text and "Geel" not in card_text:
|
||||
continue
|
||||
link = card.find("a", href=True)
|
||||
if not link:
|
||||
continue
|
||||
title = link.get_text(" ", strip=True)
|
||||
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
||||
path = urlsplit(job_url).path if job_url else ""
|
||||
match = re.fullmatch(r"/jobs/(\d+)-[^/]+", path.rstrip("/"))
|
||||
if not title or not match:
|
||||
continue
|
||||
location = "Geel, Antwerpen" if "Geel" in card_text else "Westerlo, Antwerpen"
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="Cipal Schaubroeck",
|
||||
location=location,
|
||||
postal_code="2440" if "Geel" in card_text else "2260",
|
||||
description=card_text,
|
||||
external_id=match.group(1),
|
||||
evidence_source="cipal-teamtailor-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _vanroey(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for heading in soup.select("h3.elementor-heading-title"):
|
||||
link = heading.find("a", href=True)
|
||||
if not link:
|
||||
continue
|
||||
title = link.get_text(" ", strip=True)
|
||||
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
||||
path = urlsplit(job_url).path if job_url else ""
|
||||
if not title or not re.fullmatch(r"/en/job/[^/]+/", path):
|
||||
continue
|
||||
slug = path.rstrip("/").rsplit("/", 1)[-1]
|
||||
if "oost-vlaanderen" in slug:
|
||||
continue
|
||||
card = heading.find_parent("section") or heading.parent
|
||||
description = card.get_text(" ", strip=True) if card else title
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="VanRoey",
|
||||
location="Turnhout/Geel (hybride; controleer vacaturedetail)",
|
||||
postal_code="",
|
||||
description=description,
|
||||
external_id=slug,
|
||||
evidence_source="vanroey-job-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _netropolix(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for link in soup.find_all("a", href=True):
|
||||
title = link.get_text(" ", strip=True)
|
||||
if "Geel" not in title:
|
||||
continue
|
||||
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
||||
path = urlsplit(job_url).path if job_url else ""
|
||||
if not re.fullmatch(r"/o/[^/]+", path.rstrip("/")):
|
||||
continue
|
||||
card = link.find_parent("div")
|
||||
description = card.get_text(" ", strip=True) if card else title
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="NTX (Netropolix)",
|
||||
location="Geel, Antwerpen",
|
||||
postal_code="2440",
|
||||
description=description,
|
||||
external_id=path.rstrip("/").rsplit("/", 1)[-1],
|
||||
evidence_source="netropolix-recruitee-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _thomas_more(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for link in soup.select("a.jobLink[data-item='readmore'][data-jobid][href]"):
|
||||
title = link.get_text(" ", strip=True)
|
||||
title_casefold = title.casefold()
|
||||
if not any(place in title_casefold for place in ("geel", "turnhout", "vorselaar")):
|
||||
continue
|
||||
external_id = str(link.get("data-jobid") or "")
|
||||
if not external_id.isdigit():
|
||||
continue
|
||||
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
||||
query = parse_qs(urlsplit(job_url).query) if job_url else {}
|
||||
if query.get("companyGuid") != [self.thomas_more_company_guid] or query.get("job") != [
|
||||
external_id
|
||||
]:
|
||||
continue
|
||||
if "turnhout" in title_casefold:
|
||||
location, postal_code = "Turnhout, Antwerpen", "2300"
|
||||
elif "vorselaar" in title_casefold:
|
||||
location, postal_code = "Vorselaar, Antwerpen", "2290"
|
||||
else:
|
||||
location, postal_code = "Geel, Antwerpen", "2440"
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="Thomas More",
|
||||
location=location,
|
||||
postal_code=postal_code,
|
||||
description=f"Publieke regionale vacature bij Thomas More: {title}.",
|
||||
external_id=external_id,
|
||||
evidence_source="thomas-more-cvwarehouse-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def extract(self, content: str, *, url: str) -> ExtractionResult:
|
||||
route = self._route(url)
|
||||
if route is None:
|
||||
return ExtractionResult(
|
||||
[], self.parser_key, self.parser_version, 0.0, ["Onherkende Mol-regiobron"]
|
||||
)
|
||||
host, _ = route
|
||||
soup = BeautifulSoup(content, "lxml")
|
||||
extractors = {
|
||||
"www.sckcen.be": self._sck_cen,
|
||||
"cipalschaubroeck.teamtailor.com": self._cipal,
|
||||
"www.vanroey.be": self._vanroey,
|
||||
"netropolix.recruitee.com": self._netropolix,
|
||||
"jobpage.cvwarehouse.com": self._thomas_more,
|
||||
}
|
||||
jobs = extractors[host](soup, url)
|
||||
unique_jobs = list({job.url: job for job in jobs}.values())
|
||||
warnings = [] if unique_jobs else ["Geen actuele vacatures rond Mol gevonden"]
|
||||
return ExtractionResult(
|
||||
unique_jobs,
|
||||
self.parser_key,
|
||||
self.parser_version,
|
||||
0.9 if unique_jobs else 0.0,
|
||||
warnings,
|
||||
)
|
||||
@@ -0,0 +1,312 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
||||
|
||||
|
||||
class CordaCampusAdapter:
|
||||
"""Extract the public job cards intentionally published by Corda Campus."""
|
||||
|
||||
parser_key = "regional-corda-campus"
|
||||
parser_version = "1.0.0"
|
||||
source_hosts = ("cordacampus.com",)
|
||||
|
||||
def _supports_url(self, url: str) -> bool:
|
||||
parts = urlsplit(url)
|
||||
host = (parts.hostname or "").lower()
|
||||
return host.endswith(self.source_hosts) and parts.path.rstrip("/") == "/jobs"
|
||||
|
||||
def extract(self, content: str, *, url: str) -> ExtractionResult:
|
||||
if not self._supports_url(url):
|
||||
return ExtractionResult(
|
||||
[], self.parser_key, self.parser_version, 0.0, ["Onherkenbare regiobron"]
|
||||
)
|
||||
|
||||
source_host = (urlsplit(url).hostname or "").lower()
|
||||
soup = BeautifulSoup(content, "lxml")
|
||||
jobs: list[ExtractedJob] = []
|
||||
seen_urls: set[str] = set()
|
||||
|
||||
for card in soup.select('a.event-item[href*="/job/"]'):
|
||||
job_url = urljoin(url, str(card.get("href") or ""))
|
||||
job_host = (urlsplit(job_url).hostname or "").lower()
|
||||
if job_host != source_host or job_url in seen_urls:
|
||||
continue
|
||||
title_node = card.select_one(".job-title")
|
||||
employer_node = card.select_one(".company-title")
|
||||
title = title_node.get_text(" ", strip=True) if title_node else ""
|
||||
employer = employer_node.get_text(" ", strip=True) if employer_node else ""
|
||||
if not title:
|
||||
continue
|
||||
|
||||
date_node = card.select_one(".bottom-info")
|
||||
date_posted = date_node.get_text(" ", strip=True) if date_node else ""
|
||||
external_id = urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1]
|
||||
description = f"Vacature van {employer or 'een Corda-werkgever'} via Corda Campus."
|
||||
jobs.append(
|
||||
ExtractedJob(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer_name=employer,
|
||||
external_id=external_id,
|
||||
location_text="Hasselt, Limburg",
|
||||
region="Limburg",
|
||||
country="BE",
|
||||
description_text=description,
|
||||
date_posted=date_posted,
|
||||
raw={"regional_listing": "corda-campus"},
|
||||
evidence=[
|
||||
FieldEvidence("title", "corda-job-card", 0.95, title[:240]),
|
||||
FieldEvidence("employer_name", "corda-job-card", 0.92, employer[:240]),
|
||||
FieldEvidence(
|
||||
"location_text",
|
||||
"regional-source-scope",
|
||||
0.72,
|
||||
"Corda Campus, Hasselt",
|
||||
),
|
||||
],
|
||||
)
|
||||
)
|
||||
seen_urls.add(job_url)
|
||||
|
||||
warnings = [] if jobs else ["Geen actuele Corda-vacatures gevonden"]
|
||||
return ExtractionResult(
|
||||
jobs,
|
||||
self.parser_key,
|
||||
self.parser_version,
|
||||
0.9 if jobs else 0.0,
|
||||
warnings,
|
||||
)
|
||||
|
||||
|
||||
class LocalEmployerListingAdapter:
|
||||
"""Extract reviewed public listing pages of employers in the Hasselt-Genk area."""
|
||||
|
||||
parser_key = "regional-local-employers"
|
||||
parser_version = "1.0.0"
|
||||
supported_routes = {
|
||||
"acagroup.be": {"/en/jobs"},
|
||||
"www.xploregroup.be": {"/en/jobs"},
|
||||
"www.uhasselt.be": {"/vacatures"},
|
||||
"ses.pxl.be": {"/"},
|
||||
"ziekenhuis-oost-limburg.cvw.io": {"/"},
|
||||
}
|
||||
|
||||
def _supports_url(self, url: str) -> bool:
|
||||
return self._route(url) is not None
|
||||
|
||||
def _route(self, url: str) -> tuple[str, str] | None:
|
||||
parts = urlsplit(url)
|
||||
host = (parts.hostname or "").lower()
|
||||
path = parts.path.rstrip("/") or "/"
|
||||
if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
|
||||
return None
|
||||
return host, path
|
||||
|
||||
@staticmethod
|
||||
def _job(
|
||||
*,
|
||||
url: str,
|
||||
title: str,
|
||||
employer: str,
|
||||
location: str,
|
||||
description: str,
|
||||
external_id: str,
|
||||
evidence_source: str,
|
||||
) -> ExtractedJob:
|
||||
return ExtractedJob(
|
||||
url=url,
|
||||
title=title,
|
||||
employer_name=employer,
|
||||
external_id=external_id,
|
||||
location_text=location,
|
||||
region="Limburg",
|
||||
country="BE",
|
||||
description_text=description,
|
||||
raw={"regional_listing": evidence_source},
|
||||
evidence=[
|
||||
FieldEvidence("title", evidence_source, 0.94, title[:240]),
|
||||
FieldEvidence("employer_name", "reviewed-source", 0.96, employer),
|
||||
FieldEvidence("location_text", "regional-source-scope", 0.78, location),
|
||||
],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _same_host_url(base_url: str, href: str) -> str:
|
||||
job_url = urljoin(base_url, href)
|
||||
if (urlsplit(job_url).hostname or "").lower() != (
|
||||
urlsplit(base_url).hostname or ""
|
||||
).lower():
|
||||
return ""
|
||||
return job_url
|
||||
|
||||
def _aca(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for link in soup.find_all("a", href=True):
|
||||
href = str(link.get("href") or "")
|
||||
if not href.startswith("/en/jobs/"):
|
||||
continue
|
||||
title_node = link.find("h3")
|
||||
title = title_node.get_text(" ", strip=True) if title_node else ""
|
||||
job_url = self._same_host_url(url, href)
|
||||
if not title or not job_url:
|
||||
continue
|
||||
description_node = link.find("p")
|
||||
description = (
|
||||
description_node.get_text(" ", strip=True)
|
||||
if description_node
|
||||
else f"Vacature bij ACA Group: {title}."
|
||||
)
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="ACA Group",
|
||||
location="Hasselt (hybride; kantoorselectie per vacature)",
|
||||
description=description,
|
||||
external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1],
|
||||
evidence_source="aca-job-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _xplore(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for heading in soup.find_all("h3"):
|
||||
link = heading.find("a", href=True)
|
||||
if not link or not str(link.get("href") or "").startswith("/en/jobs/"):
|
||||
continue
|
||||
card = heading.parent
|
||||
location_node = next(
|
||||
(
|
||||
node
|
||||
for node in card.find_all("p")
|
||||
if "Hasselt" in node.get_text(" ", strip=True)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not location_node:
|
||||
continue
|
||||
title = link.get_text(" ", strip=True)
|
||||
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
||||
if not title or not job_url:
|
||||
continue
|
||||
location = location_node.get_text(" ", strip=True)
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="Xplore Group",
|
||||
location=location,
|
||||
description=(
|
||||
f"Vacature bij Xplore Group met Hasselt als mogelijke werklocatie: {title}."
|
||||
),
|
||||
external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1],
|
||||
evidence_source="xplore-job-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _uhasselt(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for card in soup.select("section.vacancy-item"):
|
||||
title_node = card.find("h3")
|
||||
link = card.find("a", href=True)
|
||||
title = title_node.get_text(" ", strip=True) if title_node else ""
|
||||
job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
|
||||
if not title or not job_url or "/vacatures/detail/" not in urlsplit(job_url).path:
|
||||
continue
|
||||
external_id = urlsplit(job_url).path.split("/detail/", 1)[-1].split("-", 1)[0]
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="Universiteit Hasselt",
|
||||
location="Hasselt/Diepenbeek, Limburg",
|
||||
description=card.get_text(" ", strip=True),
|
||||
external_id=external_id,
|
||||
evidence_source="uhasselt-vacancy-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _pxl(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for card in soup.select(".vacature-card[id^='Vacature_']"):
|
||||
external_id = str(card.get("id") or "").removeprefix("Vacature_")
|
||||
title_node = card.select_one(".vacature-card-titel")
|
||||
title = title_node.get_text(" ", strip=True) if title_node else ""
|
||||
if not title or not external_id.isdigit():
|
||||
continue
|
||||
cells = [
|
||||
node.get_text(" ", strip=True) for node in card.select("td.vacature-card-td-text")
|
||||
]
|
||||
campus = next((value for value in cells if value.startswith("Campus ")), "Hasselt")
|
||||
job_url = f"{url.rstrip('/')}?vacature_id={external_id}"
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="Hogeschool PXL",
|
||||
location=f"{campus}, Limburg",
|
||||
description=" · ".join(cells),
|
||||
external_id=external_id,
|
||||
evidence_source="pxl-vacancy-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def _zol(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
|
||||
jobs = []
|
||||
for link in soup.select("a[data-item='readmore'][data-jobid][href]"):
|
||||
external_id = str(link.get("data-jobid") or "")
|
||||
title_node = link.select_one(".job-title")
|
||||
title = title_node.get_text(" ", strip=True) if title_node else ""
|
||||
job_url = self._same_host_url(url, str(link.get("href") or ""))
|
||||
if not title or not external_id.isdigit() or not job_url:
|
||||
continue
|
||||
jobs.append(
|
||||
self._job(
|
||||
url=job_url,
|
||||
title=title,
|
||||
employer="Ziekenhuis Oost-Limburg",
|
||||
location="Genk/Lanaken/Maaseik, Limburg",
|
||||
description=f"Publieke vacature van Ziekenhuis Oost-Limburg: {title}.",
|
||||
external_id=external_id,
|
||||
evidence_source="zol-cvwarehouse-card",
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
def extract(self, content: str, *, url: str) -> ExtractionResult:
|
||||
route = self._route(url)
|
||||
if route is None:
|
||||
return ExtractionResult(
|
||||
[],
|
||||
self.parser_key,
|
||||
self.parser_version,
|
||||
0.0,
|
||||
["Onherkenbare lokale werkgeversbron"],
|
||||
)
|
||||
host, _ = route
|
||||
soup = BeautifulSoup(content, "lxml")
|
||||
extractors = {
|
||||
"acagroup.be": self._aca,
|
||||
"www.xploregroup.be": self._xplore,
|
||||
"www.uhasselt.be": self._uhasselt,
|
||||
"ses.pxl.be": self._pxl,
|
||||
"ziekenhuis-oost-limburg.cvw.io": self._zol,
|
||||
}
|
||||
jobs = extractors[host](soup, url)
|
||||
unique_jobs = list({job.url: job for job in jobs}.values())
|
||||
warnings = [] if unique_jobs else ["Geen actuele lokale vacatures gevonden"]
|
||||
return ExtractionResult(
|
||||
unique_jobs,
|
||||
self.parser_key,
|
||||
self.parser_version,
|
||||
0.9 if unique_jobs else 0.0,
|
||||
warnings,
|
||||
)
|
||||
@@ -12,6 +12,9 @@ from .ats import (
|
||||
from .base import ExtractionResult
|
||||
from .generic_html import GenericHtmlAdapter
|
||||
from .jsonld import JsonLdJobPostingAdapter
|
||||
from .kempen import KempenEmployerAdapter
|
||||
from .mol_region import MolRegionEmployerAdapter
|
||||
from .regional import CordaCampusAdapter, LocalEmployerListingAdapter
|
||||
from .rss import RssAdapter
|
||||
|
||||
|
||||
@@ -22,10 +25,18 @@ class AdapterRegistry:
|
||||
self.recruitee = RecruiteeAdapter()
|
||||
self.smartrecruiters = SmartRecruitersAdapter()
|
||||
self.workable = WorkableAdapter()
|
||||
self.corda_campus = CordaCampusAdapter()
|
||||
self.local_employers = LocalEmployerListingAdapter()
|
||||
self.mol_region_employers = MolRegionEmployerAdapter()
|
||||
self.kempen_employers = KempenEmployerAdapter()
|
||||
self.jsonld = JsonLdJobPostingAdapter()
|
||||
self.generic = GenericHtmlAdapter()
|
||||
self.rss = RssAdapter()
|
||||
self.providers = [
|
||||
self.corda_campus,
|
||||
self.local_employers,
|
||||
self.mol_region_employers,
|
||||
self.kempen_employers,
|
||||
self.greenhouse,
|
||||
self.lever,
|
||||
self.recruitee,
|
||||
|
||||
@@ -2,6 +2,7 @@ from django.contrib import admin
|
||||
|
||||
from .models import (
|
||||
EmailMessageRecord,
|
||||
MailboxConnection,
|
||||
RawDocument,
|
||||
Source,
|
||||
SourceLease,
|
||||
@@ -12,6 +13,22 @@ from .models import (
|
||||
)
|
||||
|
||||
|
||||
@admin.register(MailboxConnection)
|
||||
class MailboxConnectionAdmin(admin.ModelAdmin):
|
||||
list_display = (
|
||||
"platform",
|
||||
"provider",
|
||||
"username",
|
||||
"enabled",
|
||||
"poll_interval_minutes",
|
||||
"last_success_at",
|
||||
"next_poll_at",
|
||||
)
|
||||
list_filter = ("platform", "provider", "enabled")
|
||||
search_fields = ("username",)
|
||||
exclude = ("encrypted_password", "lease_token")
|
||||
|
||||
|
||||
@admin.register(Source)
|
||||
class SourceAdmin(admin.ModelAdmin):
|
||||
list_display = (
|
||||
|
||||
@@ -2,6 +2,58 @@ from __future__ import annotations
|
||||
|
||||
from django import forms
|
||||
|
||||
from .models import MailboxConnection
|
||||
|
||||
|
||||
class MailboxConnectionForm(forms.ModelForm):
|
||||
password = forms.CharField(
|
||||
required=False,
|
||||
label="App-wachtwoord",
|
||||
help_text="Laat leeg bij wijzigen om het bestaande app-wachtwoord te behouden.",
|
||||
widget=forms.PasswordInput(
|
||||
attrs={"autocomplete": "new-password", "placeholder": "App-wachtwoord"},
|
||||
render_value=False,
|
||||
),
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = MailboxConnection
|
||||
fields = (
|
||||
"platform",
|
||||
"provider",
|
||||
"custom_host",
|
||||
"port",
|
||||
"username",
|
||||
"mailbox",
|
||||
"poll_interval_minutes",
|
||||
"enabled",
|
||||
)
|
||||
labels = {
|
||||
"platform": "Vacatureplatform",
|
||||
"provider": "Mailboxprovider",
|
||||
"custom_host": "Aangepaste IMAP-host",
|
||||
"port": "IMAP-poort",
|
||||
"username": "Mailboxaccount",
|
||||
"mailbox": "Map",
|
||||
"poll_interval_minutes": "Controlefrequentie",
|
||||
"enabled": "Automatisch synchroniseren",
|
||||
}
|
||||
widgets = {
|
||||
"username": forms.EmailInput(attrs={"autocomplete": "username"}),
|
||||
"custom_host": forms.TextInput(
|
||||
attrs={"autocomplete": "off", "placeholder": "imap.provider.be"}
|
||||
),
|
||||
"mailbox": forms.TextInput(attrs={"autocomplete": "off"}),
|
||||
}
|
||||
|
||||
def clean(self):
|
||||
data = super().clean()
|
||||
if data.get("provider") != MailboxConnection.Provider.CUSTOM:
|
||||
data["custom_host"] = ""
|
||||
if not self.instance.pk and not data.get("password"):
|
||||
self.add_error("password", "Een app-wachtwoord is verplicht voor een nieuwe koppeling.")
|
||||
return data
|
||||
|
||||
|
||||
class ManualImportForm(forms.Form):
|
||||
source_url = forms.URLField(
|
||||
|
||||
@@ -31,11 +31,11 @@ class Command(BaseCommand):
|
||||
if not hostname:
|
||||
raise CommandError("--url bevat geen geldig domein")
|
||||
source, _ = Source.objects.get_or_create(
|
||||
domain=hostname,
|
||||
base_url=options["url"],
|
||||
source_type=options["source_type"],
|
||||
defaults={
|
||||
"domain": hostname,
|
||||
"name": options["source_name"],
|
||||
"base_url": options["url"],
|
||||
"status": Source.Status.ACTIVE,
|
||||
"policy": Source.Policy.ALLOW,
|
||||
"parser_key": "auto",
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from apps.sources.services.mailbox_connections import (
|
||||
MailboxCredentialError,
|
||||
rotate_mailbox_credentials,
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Versleutel alle mailbox-app-wachtwoorden opnieuw met de eerste ingestelde sleutel."
|
||||
|
||||
def handle(self, *args, **options):
|
||||
try:
|
||||
count = rotate_mailbox_credentials()
|
||||
except MailboxCredentialError as exc:
|
||||
raise CommandError(str(exc)) from exc
|
||||
self.stdout.write(self.style.SUCCESS(f"{count} mailboxcredential(s) geroteerd."))
|
||||
@@ -5,8 +5,9 @@ from urllib.parse import urlsplit
|
||||
|
||||
import yaml
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.utils.dateparse import parse_datetime
|
||||
|
||||
from apps.sources.models import Source
|
||||
from apps.sources.models import Source, SourcePolicyReview
|
||||
from apps.sources.services.policy import is_denied_domain
|
||||
|
||||
|
||||
@@ -21,7 +22,38 @@ class Command(BaseCommand):
|
||||
if not path.is_file():
|
||||
raise CommandError(f"Seedbestand ontbreekt: {path}")
|
||||
payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
created = updated = 0
|
||||
created = updated = reviews_created = 0
|
||||
if payload.get("disable_demo_sources") is True:
|
||||
demo_sources = Source.objects.filter(domain="jobs.example.org")
|
||||
for demo_source in demo_sources:
|
||||
demo_source.status = Source.Status.DISABLED
|
||||
demo_source.policy = Source.Policy.DENY
|
||||
demo_source.policy_reason = "Demobron verborgen na installatie van livebronnen"
|
||||
demo_source.metadata = {**demo_source.metadata, "demo": True}
|
||||
demo_source.save(
|
||||
update_fields=[
|
||||
"status",
|
||||
"policy",
|
||||
"policy_reason",
|
||||
"metadata",
|
||||
"updated_at",
|
||||
]
|
||||
)
|
||||
retired = 0
|
||||
for retired_url in payload.get("retire_source_urls", []):
|
||||
for retired_source in Source.objects.filter(base_url=str(retired_url).strip()):
|
||||
retired_source.status = Source.Status.DISABLED
|
||||
retired_source.next_run_at = None
|
||||
retired_source.metadata = {
|
||||
**retired_source.metadata,
|
||||
"retired": True,
|
||||
"retired_reason": "Buiten de ingestelde regionale dekking",
|
||||
"hidden_from_source_list": True,
|
||||
}
|
||||
retired_source.save(
|
||||
update_fields=["status", "next_run_at", "metadata", "updated_at"]
|
||||
)
|
||||
retired += 1
|
||||
for item in payload.get("sources", []):
|
||||
url = str(item.get("url") or "").strip()
|
||||
domain = (urlsplit(url).hostname or "").lower()
|
||||
@@ -30,11 +62,11 @@ class Command(BaseCommand):
|
||||
continue
|
||||
deny = is_denied_domain(domain)
|
||||
source, was_created = Source.objects.update_or_create(
|
||||
domain=domain,
|
||||
base_url=url,
|
||||
source_type=item.get("type", Source.Type.EMPLOYER),
|
||||
defaults={
|
||||
"domain": domain,
|
||||
"name": item.get("name") or domain,
|
||||
"base_url": url,
|
||||
"status": Source.Status.PAUSED
|
||||
if deny
|
||||
else item.get("status", Source.Status.TRIAL),
|
||||
@@ -46,10 +78,39 @@ class Command(BaseCommand):
|
||||
else item.get("policy_reason", ""),
|
||||
"parser_key": item.get("parser", "auto"),
|
||||
"crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)),
|
||||
"minimum_interval_seconds": int(item.get("minimum_interval_seconds", 30)),
|
||||
"allow_public_endpoint": bool(item.get("allow_public_endpoint", False)),
|
||||
"metadata": item.get("metadata") or {},
|
||||
},
|
||||
)
|
||||
review = item.get("review") or {}
|
||||
if review:
|
||||
expires_at = parse_datetime(str(review.get("expires_at") or ""))
|
||||
if expires_at is None:
|
||||
raise CommandError(
|
||||
f"Bron {source.name}: review.expires_at moet een ISO-datetime zijn"
|
||||
)
|
||||
decision = str(review.get("decision") or "")
|
||||
if decision not in SourcePolicyReview.Decision.values:
|
||||
raise CommandError(f"Bron {source.name}: ongeldige reviewbeslissing")
|
||||
_, review_created = SourcePolicyReview.objects.get_or_create(
|
||||
source=source,
|
||||
decision=decision,
|
||||
evidence_link=str(review.get("evidence_link") or ""),
|
||||
expires_at=expires_at,
|
||||
defaults={
|
||||
"scope": SourcePolicyReview.Scope.SOURCE,
|
||||
"reason": str(review.get("reason") or "Bronseedreview"),
|
||||
"notes": str(review.get("notes") or ""),
|
||||
"metadata": review.get("metadata") or {},
|
||||
},
|
||||
)
|
||||
reviews_created += int(review_created)
|
||||
created += int(was_created)
|
||||
updated += int(not was_created)
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f"Bronnen: {created} aangemaakt, {updated} bijgewerkt")
|
||||
self.style.SUCCESS(
|
||||
f"Bronnen: {created} aangemaakt, {updated} bijgewerkt, "
|
||||
f"{reviews_created} reviews geregistreerd, {retired} buiten regio uitgeschakeld"
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# Generated by Django 5.2.16 on 2026-07-21 22:59
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sources', '0003_source_lease_and_origin_state'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveConstraint(
|
||||
model_name='source',
|
||||
name='unique_domain_source_type',
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='source',
|
||||
constraint=models.UniqueConstraint(condition=models.Q(('base_url', ''), _negated=True), fields=('source_type', 'base_url'), name='unique_source_type_base_url_nonempty'),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,67 @@
|
||||
# Generated by Django 5.2.16 on 2026-07-22 00:12
|
||||
|
||||
import django.db.models.deletion
|
||||
import django.utils.timezone
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sources', '0004_remove_source_unique_domain_source_type_and_more'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='emailmessagerecord',
|
||||
name='message_id',
|
||||
field=models.CharField(max_length=998),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='MailboxConnection',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('platform', models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed')], max_length=24)),
|
||||
('provider', models.CharField(choices=[('gmail', 'Gmail'), ('outlook', 'Outlook / Microsoft 365'), ('custom', 'Andere IMAP-provider')], max_length=24)),
|
||||
('custom_host', models.CharField(blank=True, max_length=255)),
|
||||
('port', models.PositiveIntegerField(default=993)),
|
||||
('username', models.CharField(max_length=320)),
|
||||
('encrypted_password', models.TextField()),
|
||||
('mailbox', models.CharField(default='INBOX', max_length=255)),
|
||||
('enabled', models.BooleanField(default=True)),
|
||||
('poll_interval_minutes', models.PositiveIntegerField(choices=[(5, 'Elke 5 minuten'), (15, 'Elke 15 minuten'), (30, 'Elke 30 minuten'), (60, 'Elk uur'), (180, 'Elke 3 uur'), (360, 'Elke 6 uur')], default=30)),
|
||||
('next_poll_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
|
||||
('last_polled_at', models.DateTimeField(blank=True, null=True)),
|
||||
('last_success_at', models.DateTimeField(blank=True, null=True)),
|
||||
('last_error_category', models.CharField(blank=True, max_length=80)),
|
||||
('last_error_message', models.CharField(blank=True, max_length=500)),
|
||||
('lease_token', models.CharField(blank=True, max_length=64)),
|
||||
('lease_expires_at', models.DateTimeField(blank=True, db_index=True, null=True)),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mailbox_connections', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['platform'],
|
||||
},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='emailmessagerecord',
|
||||
name='mailbox_connection',
|
||||
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='sources.mailboxconnection'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='emailmessagerecord',
|
||||
constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', False)), fields=('mailbox_connection', 'message_id'), name='unique_message_per_mailbox_connection'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='emailmessagerecord',
|
||||
constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', True)), fields=('message_id',), name='unique_legacy_email_message_id'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='mailboxconnection',
|
||||
constraint=models.UniqueConstraint(fields=('user', 'platform'), name='unique_mailbox_platform_per_user'),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,18 @@
|
||||
# Generated by Django 5.2.16 on 2026-07-22 00:48
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sources', '0005_alter_emailmessagerecord_message_id_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='mailboxconnection',
|
||||
name='platform',
|
||||
field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('eurobrussels', 'EuroBrussels'), ('brusselsjobs', 'BrusselsJobs')], max_length=24),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,26 @@
|
||||
# Generated by Django 5.2.16 on 2026-07-22 01:25
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
def disable_retired_brussels_mailboxes(apps, schema_editor):
|
||||
mailbox_connection = apps.get_model("sources", "MailboxConnection")
|
||||
mailbox_connection.objects.filter(
|
||||
platform__in=("eurobrussels", "brusselsjobs")
|
||||
).update(enabled=False)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sources', '0006_alter_mailboxconnection_platform'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(disable_retired_brussels_mailboxes, migrations.RunPython.noop),
|
||||
migrations.AlterField(
|
||||
model_name='mailboxconnection',
|
||||
name='platform',
|
||||
field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('careerjet', 'Careerjet'), ('randstad', 'Randstad'), ('roberthalf', 'Robert Half')], max_length=24),
|
||||
),
|
||||
]
|
||||
+109
-2
@@ -62,7 +62,9 @@ class Source(TimeStampedModel):
|
||||
ordering = ["name"]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["domain", "source_type"], name="unique_domain_source_type"
|
||||
fields=["source_type", "base_url"],
|
||||
condition=~models.Q(base_url=""),
|
||||
name="unique_source_type_base_url_nonempty",
|
||||
)
|
||||
]
|
||||
|
||||
@@ -315,8 +317,101 @@ class RawDocument(TimeStampedModel):
|
||||
return self.final_url or self.url or f"Document {self.pk}"
|
||||
|
||||
|
||||
class MailboxConnection(TimeStampedModel):
|
||||
class Platform(models.TextChoices):
|
||||
VDAB = "vdab", "VDAB"
|
||||
INDEED = "indeed", "Indeed"
|
||||
LINKEDIN = "linkedin", "LinkedIn"
|
||||
ICTJOB = "ictjob", "ictjob.be"
|
||||
JOBAT = "jobat", "Jobat"
|
||||
STEPSTONE = "stepstone", "StepStone"
|
||||
CAREERJET = "careerjet", "Careerjet"
|
||||
RANDSTAD = "randstad", "Randstad"
|
||||
ROBERTHALF = "roberthalf", "Robert Half"
|
||||
|
||||
class Provider(models.TextChoices):
|
||||
GMAIL = "gmail", "Gmail"
|
||||
OUTLOOK = "outlook", "Outlook / Microsoft 365"
|
||||
CUSTOM = "custom", "Andere IMAP-provider"
|
||||
|
||||
class PollInterval(models.IntegerChoices):
|
||||
FIVE_MINUTES = 5, "Elke 5 minuten"
|
||||
FIFTEEN_MINUTES = 15, "Elke 15 minuten"
|
||||
THIRTY_MINUTES = 30, "Elke 30 minuten"
|
||||
HOURLY = 60, "Elk uur"
|
||||
THREE_HOURS = 180, "Elke 3 uur"
|
||||
SIX_HOURS = 360, "Elke 6 uur"
|
||||
|
||||
PROVIDER_HOSTS = {
|
||||
Provider.GMAIL: "imap.gmail.com",
|
||||
Provider.OUTLOOK: "outlook.office365.com",
|
||||
}
|
||||
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="mailbox_connections",
|
||||
)
|
||||
platform = models.CharField(max_length=24, choices=Platform.choices)
|
||||
provider = models.CharField(max_length=24, choices=Provider.choices)
|
||||
custom_host = models.CharField(max_length=255, blank=True)
|
||||
port = models.PositiveIntegerField(default=993)
|
||||
username = models.CharField(max_length=320)
|
||||
encrypted_password = models.TextField()
|
||||
mailbox = models.CharField(max_length=255, default="INBOX")
|
||||
enabled = models.BooleanField(default=True)
|
||||
poll_interval_minutes = models.PositiveIntegerField(
|
||||
choices=PollInterval.choices,
|
||||
default=PollInterval.THIRTY_MINUTES,
|
||||
)
|
||||
next_poll_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
last_polled_at = models.DateTimeField(null=True, blank=True)
|
||||
last_success_at = models.DateTimeField(null=True, blank=True)
|
||||
last_error_category = models.CharField(max_length=80, blank=True)
|
||||
last_error_message = models.CharField(max_length=500, blank=True)
|
||||
lease_token = models.CharField(max_length=64, blank=True)
|
||||
lease_expires_at = models.DateTimeField(null=True, blank=True, db_index=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["platform"]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["user", "platform"], name="unique_mailbox_platform_per_user"
|
||||
)
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.get_platform_display()} · {self.username}"
|
||||
|
||||
@property
|
||||
def imap_host(self) -> str:
|
||||
return self.PROVIDER_HOSTS.get(self.provider, self.custom_host).strip().lower()
|
||||
|
||||
def clean(self) -> None:
|
||||
host = self.imap_host
|
||||
if (
|
||||
not host
|
||||
or "://" in host
|
||||
or "/" in host
|
||||
or "@" in host
|
||||
or any(c.isspace() for c in host)
|
||||
):
|
||||
raise ValidationError({"custom_host": "Vul een geldige IMAP-hostnaam in."})
|
||||
if not 1 <= self.port <= 65535:
|
||||
raise ValidationError({"port": "De IMAP-poort moet tussen 1 en 65535 liggen."})
|
||||
if "\n" in self.mailbox or "\r" in self.mailbox:
|
||||
raise ValidationError({"mailbox": "De mailboxnaam bevat ongeldige tekens."})
|
||||
|
||||
|
||||
class EmailMessageRecord(TimeStampedModel):
|
||||
message_id = models.CharField(max_length=998, unique=True)
|
||||
mailbox_connection = models.ForeignKey(
|
||||
MailboxConnection,
|
||||
on_delete=models.CASCADE,
|
||||
null=True,
|
||||
blank=True,
|
||||
related_name="messages",
|
||||
)
|
||||
message_id = models.CharField(max_length=998)
|
||||
mailbox = models.CharField(max_length=255, default="INBOX")
|
||||
sender = models.CharField(max_length=500, blank=True)
|
||||
subject = models.CharField(max_length=998, blank=True)
|
||||
@@ -330,6 +425,18 @@ class EmailMessageRecord(TimeStampedModel):
|
||||
|
||||
class Meta:
|
||||
ordering = ["-received_at", "-created_at"]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["mailbox_connection", "message_id"],
|
||||
condition=models.Q(mailbox_connection__isnull=False),
|
||||
name="unique_message_per_mailbox_connection",
|
||||
),
|
||||
models.UniqueConstraint(
|
||||
fields=["message_id"],
|
||||
condition=models.Q(mailbox_connection__isnull=True),
|
||||
name="unique_legacy_email_message_id",
|
||||
),
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.subject or self.message_id
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlatformAlert:
|
||||
key: str
|
||||
label: str
|
||||
domains: tuple[str, ...]
|
||||
alert_url: str
|
||||
help_text: str
|
||||
|
||||
|
||||
PLATFORM_ALERTS = {
|
||||
alert.key: alert
|
||||
for alert in (
|
||||
PlatformAlert(
|
||||
"vdab",
|
||||
"VDAB",
|
||||
("vdab.be",),
|
||||
"https://www.vdab.be/jobs/job-alerts",
|
||||
"Brede Vlaamse vacaturealerts.",
|
||||
),
|
||||
PlatformAlert(
|
||||
"indeed",
|
||||
"Indeed",
|
||||
("indeed.com", "indeed.be"),
|
||||
"https://support.indeed.com/hc/nl/articles/204488890-"
|
||||
"Vacature-alerts-activeren-stopzetten-en-beheren",
|
||||
"Algemene en regionale vacaturealerts.",
|
||||
),
|
||||
PlatformAlert(
|
||||
"linkedin",
|
||||
"LinkedIn",
|
||||
("linkedin.com",),
|
||||
"https://www.linkedin.com/help/linkedin/answer/a511279",
|
||||
"Dagelijkse of wekelijkse alerts, maximaal twintig per LinkedIn-account.",
|
||||
),
|
||||
PlatformAlert(
|
||||
"ictjob",
|
||||
"ictjob.be",
|
||||
("ictjob.be",),
|
||||
"https://www.ictjob.be/nl/it-job-alert",
|
||||
"Gespecialiseerde Belgische IT- en telecomvacatures.",
|
||||
),
|
||||
PlatformAlert(
|
||||
"jobat",
|
||||
"Jobat",
|
||||
("jobat.be",),
|
||||
"https://www.jobat.be/nl/jobalert",
|
||||
"Belgische jobs per functie, regio en contracttype.",
|
||||
),
|
||||
PlatformAlert(
|
||||
"stepstone",
|
||||
"StepStone",
|
||||
("stepstone.be",),
|
||||
"https://www.stepstone.be/vacatures",
|
||||
"Dagelijkse vergelijkbare vacatures vanuit een zoekopdracht.",
|
||||
),
|
||||
PlatformAlert(
|
||||
"careerjet",
|
||||
"Careerjet",
|
||||
("careerjet.be",),
|
||||
"https://www.careerjet.be/",
|
||||
"Brede Belgische zoekresultaten met regio- en postcodegebonden e-mailalerts.",
|
||||
),
|
||||
PlatformAlert(
|
||||
"randstad",
|
||||
"Randstad",
|
||||
("randstad.be",),
|
||||
"https://www.randstad.be/jobalert-aanmaken/",
|
||||
"Belgische jobs met instelbare specialisatie, locatie, contract en frequentie.",
|
||||
),
|
||||
PlatformAlert(
|
||||
"roberthalf",
|
||||
"Robert Half",
|
||||
("roberthalf.com",),
|
||||
"https://www.roberthalf.com/be/en/find-jobs/job-alerts",
|
||||
"Belgische IT- en digitaljobs, waaronder systeem-, netwerk- en supportfuncties.",
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def platform_label(key: str) -> str:
|
||||
alert = PLATFORM_ALERTS.get(key)
|
||||
return alert.label if alert else key
|
||||
@@ -140,23 +140,28 @@ def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int
|
||||
for candidate in ordered:
|
||||
provenance = _provenance_entry(candidate)
|
||||
now = _utc_now()
|
||||
source, was_created = Source.objects.get_or_create(
|
||||
domain=candidate.domain,
|
||||
source_type=candidate.source_type,
|
||||
defaults={
|
||||
"name": _candidate_name(candidate),
|
||||
"base_url": candidate.url,
|
||||
"status": Source.Status.CANDIDATE,
|
||||
"policy": Source.Policy.REVIEW,
|
||||
"policy_reason": "Automatisch ontdekt",
|
||||
"parser_key": "auto",
|
||||
"strict_mode": True,
|
||||
"metadata": {
|
||||
"discovery": [provenance],
|
||||
"discovered_at": now,
|
||||
},
|
||||
},
|
||||
source = (
|
||||
Source.objects.filter(
|
||||
domain=candidate.domain,
|
||||
source_type=candidate.source_type,
|
||||
)
|
||||
.order_by("id")
|
||||
.first()
|
||||
)
|
||||
was_created = source is None
|
||||
if source is None:
|
||||
source = Source.objects.create(
|
||||
domain=candidate.domain,
|
||||
source_type=candidate.source_type,
|
||||
name=_candidate_name(candidate),
|
||||
base_url=candidate.url,
|
||||
status=Source.Status.CANDIDATE,
|
||||
policy=Source.Policy.REVIEW,
|
||||
policy_reason="Automatisch ontdekt",
|
||||
parser_key="auto",
|
||||
strict_mode=True,
|
||||
metadata={"discovery": [provenance], "discovered_at": now},
|
||||
)
|
||||
if was_created:
|
||||
created += 1
|
||||
continue
|
||||
|
||||
@@ -13,7 +13,8 @@ from django.utils import timezone
|
||||
from apps.jobs.services.normalization import normalize_extracted_job
|
||||
from apps.jobs.services.pipeline import persist_draft
|
||||
from apps.sources.adapters.email_alert import EmailAlertAdapter
|
||||
from apps.sources.models import EmailMessageRecord, RawDocument, Source
|
||||
from apps.sources.models import EmailMessageRecord, MailboxConnection, RawDocument, Source
|
||||
from apps.sources.platforms import platform_label
|
||||
|
||||
|
||||
def message_identity(raw_message: bytes) -> str:
|
||||
@@ -24,22 +25,32 @@ def message_identity(raw_message: bytes) -> str:
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageRecord:
|
||||
def ingest_email(
|
||||
raw_message: bytes,
|
||||
*,
|
||||
mailbox: str = "INBOX",
|
||||
mailbox_connection: MailboxConnection | None = None,
|
||||
platform: str | None = None,
|
||||
) -> EmailMessageRecord:
|
||||
parsed = BytesParser(policy=policy.default).parsebytes(raw_message)
|
||||
message_id = message_identity(raw_message)
|
||||
existing = EmailMessageRecord.objects.filter(message_id=message_id).first()
|
||||
existing = EmailMessageRecord.objects.filter(
|
||||
message_id=message_id, mailbox_connection=mailbox_connection
|
||||
).first()
|
||||
if existing:
|
||||
return existing
|
||||
|
||||
source_domain = f"{platform}.mailbox.local" if platform else "mailbox.local"
|
||||
source_name = f"{platform_label(platform)} vacaturemailbox" if platform else "Vacaturemailbox"
|
||||
source, _ = Source.objects.get_or_create(
|
||||
domain="mailbox.local",
|
||||
domain=source_domain,
|
||||
source_type=Source.Type.EMAIL,
|
||||
defaults={
|
||||
"name": "Vacaturemailbox",
|
||||
"name": source_name,
|
||||
"status": Source.Status.ACTIVE,
|
||||
"policy": Source.Policy.ALLOW,
|
||||
"parser_key": "email-alert",
|
||||
"crawl_interval_minutes": 10,
|
||||
"crawl_interval_minutes": 1440,
|
||||
},
|
||||
)
|
||||
content_hash = hashlib.sha256(raw_message).hexdigest()
|
||||
@@ -52,7 +63,11 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
|
||||
body_text=raw_message.decode("utf-8", errors="replace"),
|
||||
byte_length=len(raw_message),
|
||||
retain_until=retain_until,
|
||||
metadata={"mailbox": mailbox},
|
||||
metadata={
|
||||
"mailbox": mailbox,
|
||||
"mailbox_connection_id": mailbox_connection.pk if mailbox_connection else None,
|
||||
"alert_provider": platform or "unknown",
|
||||
},
|
||||
)
|
||||
received_at = None
|
||||
if parsed.get("date"):
|
||||
@@ -64,12 +79,28 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
|
||||
received_at = None
|
||||
|
||||
adapter = EmailAlertAdapter()
|
||||
result = adapter.extract_message(raw_message)
|
||||
result = adapter.extract_message(raw_message, expected_provider=platform)
|
||||
detected_provider = next(
|
||||
(
|
||||
str(extracted.raw.get("alert_provider"))
|
||||
for extracted in result.jobs
|
||||
if extracted.raw.get("alert_provider")
|
||||
),
|
||||
"unknown",
|
||||
)
|
||||
alert_provider = platform or detected_provider
|
||||
document.parser_key = result.parser_key
|
||||
document.parser_version = result.parser_version
|
||||
document.extraction_confidence = result.confidence
|
||||
document.metadata = {**document.metadata, "alert_provider": alert_provider}
|
||||
document.save(
|
||||
update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"]
|
||||
update_fields=[
|
||||
"parser_key",
|
||||
"parser_version",
|
||||
"extraction_confidence",
|
||||
"metadata",
|
||||
"updated_at",
|
||||
]
|
||||
)
|
||||
links: list[str] = []
|
||||
errors: list[str] = []
|
||||
@@ -87,6 +118,7 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
|
||||
except Exception as exc:
|
||||
errors.append(f"{exc.__class__.__name__}: {exc}")
|
||||
return EmailMessageRecord.objects.create(
|
||||
mailbox_connection=mailbox_connection,
|
||||
message_id=message_id,
|
||||
mailbox=mailbox,
|
||||
sender=str(parsed.get("from") or "")[:500],
|
||||
|
||||
@@ -14,6 +14,7 @@ from django.conf import settings
|
||||
from apps.sources.models import Source
|
||||
|
||||
from .policy import assess_url
|
||||
from .tls import trusted_tls_context
|
||||
from .url_security import ValidatedUrl, validate_public_url
|
||||
|
||||
ALLOWED_CONTENT_TYPES = (
|
||||
@@ -123,6 +124,7 @@ def fetch_url(
|
||||
timeout=httpx.Timeout(settings.FETCHER_TIMEOUT_SECONDS),
|
||||
follow_redirects=False,
|
||||
headers=headers,
|
||||
verify=trusted_tls_context(),
|
||||
)
|
||||
current_url = url
|
||||
previous_validation: ValidatedUrl | None = None
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import imaplib
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
from apps.sources.models import EmailMessageRecord, MailboxConnection
|
||||
from apps.sources.services.email_import import ingest_email, message_identity
|
||||
from apps.sources.services.mailbox_connections import decrypt_mailbox_password
|
||||
from apps.sources.services.tls import trusted_tls_context
|
||||
from apps.sources.services.url_security import validate_public_url
|
||||
|
||||
ImapClientFactory = Callable[..., Any]
|
||||
HostValidator = Callable[[str], object]
|
||||
|
||||
|
||||
def poll_imap_mailbox(
|
||||
connection: MailboxConnection,
|
||||
*,
|
||||
client_factory: ImapClientFactory | None = None,
|
||||
host_validator: HostValidator = validate_public_url,
|
||||
) -> dict[str, int | str]:
|
||||
"""Import one bounded mailbox batch without letting one bad message abort the batch."""
|
||||
if not connection.enabled:
|
||||
return {"status": "disabled", "imported": 0}
|
||||
|
||||
if client_factory is None:
|
||||
client_factory = imaplib.IMAP4_SSL
|
||||
host_validator(f"https://{connection.imap_host}")
|
||||
password = decrypt_mailbox_password(connection)
|
||||
client_options = {"timeout": settings.IMAP_CONNECT_TIMEOUT_SECONDS}
|
||||
if client_factory is imaplib.IMAP4_SSL:
|
||||
client_options["ssl_context"] = trusted_tls_context()
|
||||
client = client_factory(connection.imap_host, connection.port, **client_options)
|
||||
counters = {
|
||||
"imported": 0,
|
||||
"duplicates": 0,
|
||||
"skipped_oversized": 0,
|
||||
"failed": 0,
|
||||
}
|
||||
try:
|
||||
client.login(connection.username, password)
|
||||
status, _ = client.select(connection.mailbox, readonly=True)
|
||||
if status != "OK":
|
||||
raise RuntimeError("IMAP-mailbox kon niet worden geopend")
|
||||
status, data = client.uid("search", None, "ALL")
|
||||
if status != "OK":
|
||||
raise RuntimeError("IMAP-zoekopdracht mislukt")
|
||||
uids = (data[0] or b"").split()[-settings.IMAP_MAX_MESSAGES_PER_POLL :]
|
||||
for uid in uids:
|
||||
try:
|
||||
status, payload = client.uid("fetch", uid, "(RFC822)")
|
||||
if status != "OK" or not payload:
|
||||
counters["failed"] += 1
|
||||
continue
|
||||
raw = next((part[1] for part in payload if isinstance(part, tuple)), None)
|
||||
if not isinstance(raw, bytes):
|
||||
counters["failed"] += 1
|
||||
continue
|
||||
if len(raw) > settings.IMAP_MAX_MESSAGE_BYTES:
|
||||
counters["skipped_oversized"] += 1
|
||||
continue
|
||||
identity = message_identity(raw)
|
||||
if EmailMessageRecord.objects.filter(
|
||||
mailbox_connection=connection, message_id=identity
|
||||
).exists():
|
||||
counters["duplicates"] += 1
|
||||
continue
|
||||
ingest_email(
|
||||
raw,
|
||||
mailbox=connection.mailbox,
|
||||
mailbox_connection=connection,
|
||||
platform=connection.platform,
|
||||
)
|
||||
counters["imported"] += 1
|
||||
except Exception:
|
||||
counters["failed"] += 1
|
||||
return {"status": "ok", **counters}
|
||||
finally:
|
||||
with suppress(Exception):
|
||||
client.logout()
|
||||
@@ -0,0 +1,148 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken, MultiFernet
|
||||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.sources.models import MailboxConnection
|
||||
|
||||
|
||||
class MailboxCredentialError(RuntimeError):
|
||||
"""Safe credential configuration error without secret material."""
|
||||
|
||||
|
||||
def _cipher() -> MultiFernet:
|
||||
keys = settings.MAILBOX_CREDENTIAL_KEYS
|
||||
if not keys:
|
||||
raise MailboxCredentialError(
|
||||
"Credentialopslag is niet geconfigureerd. Stel MAILBOX_CREDENTIAL_KEYS in."
|
||||
)
|
||||
try:
|
||||
return MultiFernet([Fernet(key.encode("ascii")) for key in keys])
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise MailboxCredentialError(
|
||||
"MAILBOX_CREDENTIAL_KEYS bevat een ongeldige sleutel."
|
||||
) from exc
|
||||
|
||||
|
||||
def encrypt_mailbox_password(password: str) -> str:
|
||||
if not password:
|
||||
raise MailboxCredentialError("Een app-wachtwoord is verplicht.")
|
||||
return _cipher().encrypt(password.encode("utf-8")).decode("ascii")
|
||||
|
||||
|
||||
def decrypt_mailbox_password(connection: MailboxConnection) -> str:
|
||||
try:
|
||||
return _cipher().decrypt(connection.encrypted_password.encode("ascii")).decode("utf-8")
|
||||
except (InvalidToken, ValueError, UnicodeError) as exc:
|
||||
raise MailboxCredentialError(
|
||||
"Het opgeslagen app-wachtwoord kan niet worden ontsleuteld. Sla het opnieuw op."
|
||||
) from exc
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def rotate_mailbox_credentials() -> int:
|
||||
"""Re-encrypt every mailbox password with the first configured key."""
|
||||
rotated = 0
|
||||
for connection in MailboxConnection.objects.select_for_update().all():
|
||||
password = decrypt_mailbox_password(connection)
|
||||
connection.encrypted_password = encrypt_mailbox_password(password)
|
||||
connection.save(update_fields=["encrypted_password", "updated_at"])
|
||||
rotated += 1
|
||||
return rotated
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def save_mailbox_connection(
|
||||
*,
|
||||
user,
|
||||
cleaned_data: dict[str, object],
|
||||
instance: MailboxConnection | None = None,
|
||||
) -> MailboxConnection:
|
||||
connection = instance or MailboxConnection(user=user)
|
||||
if connection.pk and connection.user_id != user.pk:
|
||||
raise PermissionError("Mailboxkoppeling behoort niet tot deze gebruiker.")
|
||||
|
||||
password = str(cleaned_data.pop("password", "") or "")
|
||||
for field in (
|
||||
"platform",
|
||||
"provider",
|
||||
"custom_host",
|
||||
"port",
|
||||
"username",
|
||||
"mailbox",
|
||||
"enabled",
|
||||
"poll_interval_minutes",
|
||||
):
|
||||
setattr(connection, field, cleaned_data[field])
|
||||
if password:
|
||||
connection.encrypted_password = encrypt_mailbox_password(password)
|
||||
elif not connection.encrypted_password:
|
||||
raise MailboxCredentialError("Een app-wachtwoord is verplicht.")
|
||||
connection.next_poll_at = timezone.now()
|
||||
connection.last_error_category = ""
|
||||
connection.last_error_message = ""
|
||||
connection.full_clean()
|
||||
connection.save()
|
||||
return connection
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def claim_mailbox_connection(
|
||||
*, connection_id: int, worker_token: str | None = None, force: bool = False
|
||||
) -> tuple[MailboxConnection, str] | None:
|
||||
now = timezone.now()
|
||||
connection = MailboxConnection.objects.select_for_update().get(pk=connection_id)
|
||||
if not connection.enabled:
|
||||
return None
|
||||
if connection.lease_expires_at and connection.lease_expires_at > now:
|
||||
return None
|
||||
if not force and connection.next_poll_at > now:
|
||||
return None
|
||||
token = worker_token or uuid4().hex
|
||||
connection.lease_token = token
|
||||
connection.lease_expires_at = now + timedelta(minutes=5)
|
||||
connection.save(update_fields=["lease_token", "lease_expires_at", "updated_at"])
|
||||
return connection, token
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def finish_mailbox_poll(
|
||||
*,
|
||||
connection_id: int,
|
||||
worker_token: str,
|
||||
success: bool,
|
||||
error_category: str = "",
|
||||
error_message: str = "",
|
||||
) -> None:
|
||||
connection = MailboxConnection.objects.select_for_update().get(pk=connection_id)
|
||||
if connection.lease_token != worker_token:
|
||||
return
|
||||
now = timezone.now()
|
||||
connection.last_polled_at = now
|
||||
connection.next_poll_at = now + timedelta(minutes=connection.poll_interval_minutes)
|
||||
connection.lease_token = ""
|
||||
connection.lease_expires_at = None
|
||||
if success:
|
||||
connection.last_success_at = now
|
||||
connection.last_error_category = ""
|
||||
connection.last_error_message = ""
|
||||
else:
|
||||
connection.last_error_category = error_category[:80]
|
||||
connection.last_error_message = error_message[:500]
|
||||
connection.save(
|
||||
update_fields=[
|
||||
"last_polled_at",
|
||||
"next_poll_at",
|
||||
"lease_token",
|
||||
"lease_expires_at",
|
||||
"last_success_at",
|
||||
"last_error_category",
|
||||
"last_error_message",
|
||||
"updated_at",
|
||||
]
|
||||
)
|
||||
@@ -19,6 +19,10 @@ DEFAULT_DENYLIST = {
|
||||
"stepstone.be",
|
||||
"jobat.be",
|
||||
"vdab.be",
|
||||
"ictjob.be",
|
||||
"careerjet.be",
|
||||
"randstad.be",
|
||||
"roberthalf.com",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
from urllib.parse import urljoin, urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
from django.conf import settings
|
||||
@@ -11,6 +11,7 @@ from django.utils import timezone
|
||||
|
||||
from apps.sources.models import Source, SourceRobotsCache
|
||||
|
||||
from .tls import trusted_tls_context
|
||||
from .url_security import UnsafeUrlError, validate_public_url
|
||||
|
||||
ALLOW = "allow"
|
||||
@@ -123,20 +124,37 @@ def _max_bytes() -> int:
|
||||
|
||||
def _fetch_robots(origin: str, *, client: httpx.Client | None = None) -> tuple[str, int, str, str]:
|
||||
robots_url = _robots_url(origin)
|
||||
validate_public_url(
|
||||
robots_url,
|
||||
allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
|
||||
)
|
||||
own_client = client is None
|
||||
http_client = client or httpx.Client(
|
||||
timeout=httpx.Timeout(getattr(settings, "ROBOTS_FETCH_TIMEOUT_SECONDS", 5)),
|
||||
follow_redirects=False,
|
||||
verify=trusted_tls_context(),
|
||||
)
|
||||
try:
|
||||
response = http_client.get(
|
||||
robots_url,
|
||||
headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
|
||||
)
|
||||
current_url = robots_url
|
||||
for _ in range(getattr(settings, "FETCHER_MAX_REDIRECTS", 5) + 1):
|
||||
validate_public_url(
|
||||
current_url,
|
||||
allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
|
||||
)
|
||||
response = http_client.get(
|
||||
current_url,
|
||||
headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
|
||||
)
|
||||
if response.status_code not in {301, 302, 303, 307, 308}:
|
||||
break
|
||||
location = response.headers.get("location")
|
||||
if not location:
|
||||
raise httpx.HTTPStatusError(
|
||||
"Robotsredirect zonder Location-header",
|
||||
request=response.request,
|
||||
response=response,
|
||||
)
|
||||
current_url = urljoin(current_url, location)
|
||||
else:
|
||||
raise httpx.TooManyRedirects(
|
||||
"Te veel redirects bij robotscontrole", request=response.request
|
||||
)
|
||||
finally:
|
||||
if own_client:
|
||||
http_client.close()
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ssl
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import certifi
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def trusted_tls_context() -> ssl.SSLContext:
|
||||
"""Build a strict TLS context from the pinned application CA bundle."""
|
||||
bundle = Path(certifi.where()).resolve(strict=True)
|
||||
if not bundle.is_file():
|
||||
raise RuntimeError("De vertrouwde CA-bundel ontbreekt.")
|
||||
return ssl.create_default_context(cafile=str(bundle))
|
||||
+63
-37
@@ -1,7 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import imaplib
|
||||
from contextlib import suppress
|
||||
from datetime import timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -11,8 +9,7 @@ from django.db.models import Q
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.jobs.services.pipeline import process_raw_document
|
||||
from apps.sources.models import EmailMessageRecord, RawDocument, Source, SourceRun
|
||||
from apps.sources.services.email_import import ingest_email, message_identity
|
||||
from apps.sources.models import MailboxConnection, RawDocument, Source, SourceRun
|
||||
from apps.sources.services.fetcher import (
|
||||
FetchError,
|
||||
FetchTimeoutError,
|
||||
@@ -25,6 +22,12 @@ from apps.sources.services.health import (
|
||||
evaluate_source_health,
|
||||
start_health_canary,
|
||||
)
|
||||
from apps.sources.services.imap_import import poll_imap_mailbox
|
||||
from apps.sources.services.mailbox_connections import (
|
||||
MailboxCredentialError,
|
||||
claim_mailbox_connection,
|
||||
finish_mailbox_poll,
|
||||
)
|
||||
from apps.sources.services.policy import assess_url
|
||||
from apps.sources.services.scheduling import (
|
||||
acquire_source_lease,
|
||||
@@ -192,40 +195,63 @@ def schedule_due_sources() -> dict[str, int]:
|
||||
return {"scheduled": len(ids)}
|
||||
|
||||
|
||||
@shared_task(name="apps.sources.tasks.poll_imap_inbox")
|
||||
def poll_imap_inbox() -> dict[str, int | str]:
|
||||
if not settings.IMAP_ENABLED:
|
||||
return {"status": "disabled", "imported": 0}
|
||||
if not settings.IMAP_HOST or not settings.IMAP_USER or not settings.IMAP_PASSWORD:
|
||||
return {"status": "not_configured", "imported": 0}
|
||||
client_cls = imaplib.IMAP4_SSL if settings.IMAP_USE_SSL else imaplib.IMAP4
|
||||
client = client_cls(settings.IMAP_HOST, settings.IMAP_PORT)
|
||||
imported = 0
|
||||
@shared_task(name="apps.sources.tasks.schedule_mailbox_polls")
|
||||
def schedule_mailbox_polls() -> dict[str, int]:
|
||||
now = timezone.now()
|
||||
due_ids = list(
|
||||
MailboxConnection.objects.filter(enabled=True, next_poll_at__lte=now)
|
||||
.filter(Q(lease_expires_at__isnull=True) | Q(lease_expires_at__lte=now))
|
||||
.values_list("id", flat=True)[:100]
|
||||
)
|
||||
for connection_id in due_ids:
|
||||
poll_mailbox.delay(connection_id)
|
||||
return {"scheduled": len(due_ids)}
|
||||
|
||||
|
||||
@shared_task(bind=True, name="apps.sources.tasks.poll_mailbox")
|
||||
def poll_mailbox(self, connection_id: int, force: bool = False) -> dict[str, int | str]:
|
||||
worker_token = str(self.request.id or uuid4())
|
||||
claimed = claim_mailbox_connection(
|
||||
connection_id=connection_id, worker_token=worker_token, force=force
|
||||
)
|
||||
if claimed is None:
|
||||
return {"status": "skipped", "imported": 0}
|
||||
connection, token = claimed
|
||||
try:
|
||||
client.login(settings.IMAP_USER, settings.IMAP_PASSWORD)
|
||||
status, _ = client.select(settings.IMAP_MAILBOX, readonly=not settings.IMAP_MARK_SEEN)
|
||||
if status != "OK":
|
||||
raise RuntimeError("IMAP-mailbox kon niet worden geopend")
|
||||
status, data = client.uid("search", None, "ALL")
|
||||
if status != "OK":
|
||||
raise RuntimeError("IMAP-zoekopdracht mislukt")
|
||||
uids = (data[0] or b"").split()[-200:]
|
||||
for uid in uids:
|
||||
status, payload = client.uid("fetch", uid, "(RFC822)")
|
||||
if status != "OK" or not payload:
|
||||
continue
|
||||
raw = next((part[1] for part in payload if isinstance(part, tuple)), None)
|
||||
if not raw:
|
||||
continue
|
||||
message_id = message_identity(raw)
|
||||
was_known = EmailMessageRecord.objects.filter(message_id=message_id).exists()
|
||||
ingest_email(raw, mailbox=settings.IMAP_MAILBOX)
|
||||
if not was_known:
|
||||
imported += 1
|
||||
return {"status": "ok", "imported": imported}
|
||||
finally:
|
||||
with suppress(Exception):
|
||||
client.logout()
|
||||
result = poll_imap_mailbox(connection)
|
||||
finish_mailbox_poll(
|
||||
connection_id=connection.pk,
|
||||
worker_token=token,
|
||||
success=result["status"] == "ok",
|
||||
)
|
||||
return result
|
||||
except MailboxCredentialError as exc:
|
||||
finish_mailbox_poll(
|
||||
connection_id=connection.pk,
|
||||
worker_token=token,
|
||||
success=False,
|
||||
error_category="credentials",
|
||||
error_message=str(exc),
|
||||
)
|
||||
return {"status": "failed", "error_category": "credentials", "imported": 0}
|
||||
except Exception as exc:
|
||||
error_category = "imap"
|
||||
if exc.__class__.__name__.lower().startswith("unsafe"):
|
||||
error_category = "host_policy"
|
||||
finish_mailbox_poll(
|
||||
connection_id=connection.pk,
|
||||
worker_token=token,
|
||||
success=False,
|
||||
error_category=error_category,
|
||||
error_message="IMAP-verbinding of authenticatie mislukt.",
|
||||
)
|
||||
return {"status": "failed", "error_category": error_category, "imported": 0}
|
||||
|
||||
|
||||
@shared_task(name="apps.sources.tasks.poll_imap_inbox")
|
||||
def poll_imap_inbox() -> dict[str, int]:
|
||||
"""Compatibility alias for old deployments; it now schedules due per-platform mailboxes."""
|
||||
return schedule_mailbox_polls()
|
||||
|
||||
|
||||
@shared_task(name="apps.sources.tasks.cleanup_raw_documents")
|
||||
|
||||
+14
-1
@@ -1,10 +1,23 @@
|
||||
from django.urls import path
|
||||
|
||||
from .views import SourceListView, bulk_candidate_action, manual_import_view, retry_source
|
||||
from .views import (
|
||||
SourceListView,
|
||||
bulk_candidate_action,
|
||||
edit_mailbox,
|
||||
manual_import_view,
|
||||
retry_source,
|
||||
save_mailbox,
|
||||
sync_platform_alerts,
|
||||
toggle_mailbox,
|
||||
)
|
||||
|
||||
urlpatterns = [
|
||||
path("", SourceListView.as_view(), name="list"),
|
||||
path("manual-import/", manual_import_view, name="manual_import"),
|
||||
path("mailboxes/add/", save_mailbox, name="mailbox_add"),
|
||||
path("mailboxes/<int:pk>/edit/", edit_mailbox, name="mailbox_edit"),
|
||||
path("mailboxes/<int:pk>/sync/", sync_platform_alerts, name="sync_platform_alerts"),
|
||||
path("mailboxes/<int:pk>/toggle/", toggle_mailbox, name="mailbox_toggle"),
|
||||
path("bulk/", bulk_candidate_action, name="bulk"),
|
||||
path("<int:pk>/retry/", retry_source, name="retry"),
|
||||
]
|
||||
|
||||
+133
-6
@@ -4,39 +4,88 @@ from django.conf import settings
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.contrib.auth.mixins import LoginRequiredMixin
|
||||
from django.db.models import Count, Q
|
||||
from django.http import HttpRequest
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.urls import reverse
|
||||
from django.utils import timezone
|
||||
from django.views.decorators.cache import never_cache
|
||||
from django.views.decorators.http import require_POST
|
||||
from django.views.generic import ListView
|
||||
|
||||
from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
|
||||
from apps.profiles.models import SearchProfile
|
||||
|
||||
from .forms import ManualImportForm
|
||||
from .models import Source, SourcePolicyReview
|
||||
from .forms import MailboxConnectionForm, ManualImportForm
|
||||
from .models import MailboxConnection, Source, SourcePolicyReview
|
||||
from .platforms import PLATFORM_ALERTS
|
||||
from .services.mailbox_connections import MailboxCredentialError, save_mailbox_connection
|
||||
from .services.manual_import import ManualImportError, ManualImportSummary, import_manual_source
|
||||
from .services.policy import create_policy_review
|
||||
from .tasks import fetch_source
|
||||
from .tasks import fetch_source, poll_mailbox
|
||||
|
||||
|
||||
def _source_list_queryset():
|
||||
return Source.objects.prefetch_related("policy_reviews").all().order_by("name")
|
||||
return (
|
||||
Source.objects.prefetch_related("policy_reviews")
|
||||
.exclude(domain="jobs.example.org", status=Source.Status.DISABLED)
|
||||
.filter(
|
||||
Q(metadata__hidden_from_source_list__isnull=True)
|
||||
| Q(metadata__hidden_from_source_list=False)
|
||||
)
|
||||
.filter(Q(metadata__watchlist__isnull=True) | Q(metadata__watchlist=False))
|
||||
.annotate(job_count=Count("job_aliases", distinct=True))
|
||||
.order_by("name")
|
||||
)
|
||||
|
||||
|
||||
def _manual_import_context(request: HttpRequest, *, form: ManualImportForm, manual_result=None):
|
||||
def _watchlist_queryset():
|
||||
return (
|
||||
Source.objects.filter(metadata__watchlist=True)
|
||||
.filter(
|
||||
Q(metadata__hidden_from_source_list__isnull=True)
|
||||
| Q(metadata__hidden_from_source_list=False)
|
||||
)
|
||||
.order_by("name")
|
||||
)
|
||||
|
||||
|
||||
def _manual_import_context(
|
||||
request: HttpRequest,
|
||||
*,
|
||||
form: ManualImportForm,
|
||||
manual_result=None,
|
||||
mailbox_form: MailboxConnectionForm | None = None,
|
||||
editing_mailbox: MailboxConnection | None = None,
|
||||
):
|
||||
base_queryset = _source_list_queryset()
|
||||
watchlist_queryset = _watchlist_queryset()
|
||||
mailbox_connections = list(
|
||||
request.user.mailbox_connections.filter(platform__in=PLATFORM_ALERTS)
|
||||
)
|
||||
active_search_profile = SearchProfile.objects.filter(user=request.user, is_active=True).first()
|
||||
return {
|
||||
"sources": base_queryset,
|
||||
"active_source_count": Source.objects.filter(status=Source.Status.ACTIVE).count(),
|
||||
"source_total_count": base_queryset.count(),
|
||||
"review_pending_count": Source.objects.filter(policy=Source.Policy.REVIEW).count(),
|
||||
"watchlist_sources": watchlist_queryset,
|
||||
"watchlist_count": watchlist_queryset.count(),
|
||||
"manual_import_form": form,
|
||||
"manual_import_bookmarklet_endpoint": request.build_absolute_uri(
|
||||
reverse("sources:manual_import")
|
||||
),
|
||||
"manual_import_paste_max_bytes": settings.MANUAL_IMPORT_PASTE_MAX_BYTES,
|
||||
"manual_import_result": manual_result,
|
||||
"platform_alert_ingress_ready": any(
|
||||
connection.enabled for connection in mailbox_connections
|
||||
),
|
||||
"mailbox_connections": mailbox_connections,
|
||||
"platform_alerts": PLATFORM_ALERTS.values(),
|
||||
"mailbox_form": mailbox_form or MailboxConnectionForm(),
|
||||
"editing_mailbox": editing_mailbox,
|
||||
"mailbox_credential_key_ready": bool(settings.MAILBOX_CREDENTIAL_KEYS),
|
||||
"active_search_profile": active_search_profile,
|
||||
}
|
||||
|
||||
|
||||
@@ -108,7 +157,7 @@ def _action_to_review(
|
||||
def bulk_candidate_action(request):
|
||||
action = request.POST.get("action")
|
||||
source_ids = [int(raw_id) for raw_id in request.POST.getlist("source_ids") if raw_id.isdigit()]
|
||||
sources = Source.objects.filter(id__in=source_ids).exclude(policy=Source.Policy.DENY)
|
||||
sources = _source_list_queryset().filter(id__in=source_ids).exclude(policy=Source.Policy.DENY)
|
||||
if not sources.exists():
|
||||
messages.info(request, "Geen bruikbare bronregels geselecteerd.")
|
||||
return redirect("sources:list")
|
||||
@@ -148,6 +197,84 @@ def retry_source(request, pk: int):
|
||||
return redirect("sources:list")
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def save_mailbox(request, pk: int | None = None):
|
||||
instance = None
|
||||
if pk is not None:
|
||||
instance = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
|
||||
form = MailboxConnectionForm(request.POST, instance=instance)
|
||||
if form.is_valid():
|
||||
try:
|
||||
connection = save_mailbox_connection(
|
||||
user=request.user,
|
||||
cleaned_data=dict(form.cleaned_data),
|
||||
instance=instance,
|
||||
)
|
||||
except MailboxCredentialError as exc:
|
||||
form.add_error("password", str(exc))
|
||||
else:
|
||||
poll_mailbox.delay(connection.pk, force=True)
|
||||
messages.success(
|
||||
request,
|
||||
f"{connection.get_platform_display()}-mailbox opgeslagen; "
|
||||
"verbindingstest ingepland.",
|
||||
)
|
||||
return redirect("sources:list")
|
||||
messages.error(request, "Controleer de mailboxinstellingen.")
|
||||
return render(
|
||||
request,
|
||||
"sources/list.html",
|
||||
_manual_import_context(
|
||||
request,
|
||||
form=ManualImportForm(),
|
||||
mailbox_form=form,
|
||||
editing_mailbox=instance,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@login_required
|
||||
def edit_mailbox(request, pk: int):
|
||||
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
|
||||
if request.method == "POST":
|
||||
return save_mailbox(request, pk=pk)
|
||||
return render(
|
||||
request,
|
||||
"sources/list.html",
|
||||
_manual_import_context(
|
||||
request,
|
||||
form=ManualImportForm(),
|
||||
mailbox_form=MailboxConnectionForm(instance=connection),
|
||||
editing_mailbox=connection,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def sync_platform_alerts(request, pk: int):
|
||||
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
|
||||
if not connection.enabled:
|
||||
messages.error(request, "Activeer deze mailbox voordat je synchroniseert.")
|
||||
else:
|
||||
poll_mailbox.delay(connection.pk, force=True)
|
||||
messages.success(request, f"{connection.get_platform_display()}-mailimport ingepland.")
|
||||
return redirect("sources:list")
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def toggle_mailbox(request, pk: int):
|
||||
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
|
||||
connection.enabled = not connection.enabled
|
||||
connection.next_poll_at = timezone.now()
|
||||
connection.save(update_fields=["enabled", "next_poll_at", "updated_at"])
|
||||
state = "geactiveerd" if connection.enabled else "gepauzeerd"
|
||||
messages.success(request, f"{connection.get_platform_display()}-mailbox {state}.")
|
||||
return redirect("sources:list")
|
||||
|
||||
|
||||
@login_required
|
||||
@never_cache
|
||||
def manual_import_view(request):
|
||||
|
||||
Reference in New Issue
Block a user