feat: release regional radar and mailbox integrations
deploy / deploy (push) Canceled after 0s

This commit is contained in:
Jens
2026-07-22 05:12:07 +02:00
parent 598d3ec18a
commit 551d0f46c2
131 changed files with 6209 additions and 336 deletions
+6
View File
@@ -8,6 +8,9 @@ from .ats import (
from .base import ExtractedJob, ExtractionResult
from .generic_html import GenericHtmlAdapter
from .jsonld import JsonLdJobPostingAdapter
from .kempen import KempenEmployerAdapter
from .mol_region import MolRegionEmployerAdapter
from .regional import LocalEmployerListingAdapter
from .rss import RssAdapter
__all__ = [
@@ -16,7 +19,10 @@ __all__ = [
"GenericHtmlAdapter",
"GreenhouseAdapter",
"JsonLdJobPostingAdapter",
"KempenEmployerAdapter",
"LeverAdapter",
"LocalEmployerListingAdapter",
"MolRegionEmployerAdapter",
"RecruiteeAdapter",
"RssAdapter",
"SmartRecruitersAdapter",
+45 -6
View File
@@ -169,16 +169,20 @@ class _AtsAdapter(ABC):
def _extract_location(self, record: dict[str, object]) -> tuple[str, str, str, str]:
location_raw = _first_text(
record,
"location",
"location.city",
"location.name",
"location.address",
"location.raw",
"categories.location",
"city",
"cityName",
"place",
"office",
"location.address",
"address",
"data.location",
"officeLocation",
"locationName",
"location",
)
if not location_raw:
location_raw = _first_text(record, "data.city", "data.location")
@@ -223,6 +227,13 @@ class _AtsAdapter(ABC):
"remote_type",
"job_type",
).lower()
if not workplace_raw:
if record.get("hybrid") is True:
return "hybrid"
if record.get("remote") is True:
return "remote"
if record.get("on_site") is True:
return "on_site"
if "remote" in workplace_raw:
return "remote" if "hybrid" not in workplace_raw else "hybrid"
if "hybrid" in workplace_raw:
@@ -256,18 +267,21 @@ class _AtsAdapter(ABC):
"url",
"jobUrl",
"job_url",
"hostedUrl",
"applyUrl",
"link",
"absoluteUrl",
"careers_url",
"data.url",
)
employer_name = _first_text(
record,
"company_name",
"company.name",
"company",
"employer",
"organization",
"organizationName",
"company.name",
"department",
"hiringOrganization",
)
@@ -290,6 +304,7 @@ class _AtsAdapter(ABC):
"published_at",
"created_at",
"jobCreated",
"releasedDate",
)
valid_through = _first_text(
record,
@@ -308,6 +323,9 @@ class _AtsAdapter(ABC):
"jobType",
"type",
"data.employmentType",
"employment_type_code",
"categories.commitment",
"typeOfEmployment.label",
)
)
workplace_type = self._extract_workplace(record)
@@ -428,13 +446,20 @@ class GreenhouseAdapter(_AtsAdapter):
class LeverAdapter(_AtsAdapter):
parser_key = "ats-lever"
source_hosts = ("jobs.lever.co",)
source_hosts = (
"jobs.lever.co",
"jobs.eu.lever.co",
"api.lever.co",
"api.eu.lever.co",
)
support_markers = ("lever", "requisition", "posting")
listing_paths = (("data",), ("jobs",), ("results",))
detail_paths = (("data",), ("job",), ("position",), ("result",))
closed_statuses = CLOSED_STATUSES | {"archived", "deleted"}
def _extract_records(self, payload):
if isinstance(payload, list):
return [item for item in payload if isinstance(item, dict)]
for path in self.listing_paths:
value = payload
for key in path:
@@ -466,7 +491,7 @@ class RecruiteeAdapter(_AtsAdapter):
parser_key = "ats-recruitee"
source_hosts = ("recruitee.com",)
support_markers = ("recruitee", "career", "vacancy")
listing_paths = (("jobs",), ("data", "jobs"), ("vacancies",))
listing_paths = (("jobs",), ("offers",), ("data", "jobs"), ("vacancies",))
detail_paths = (("job",), ("data", "job"), ("vacancy",), ("result",))
closed_statuses = CLOSED_STATUSES | {"hidden", "paused"}
@@ -494,12 +519,26 @@ class RecruiteeAdapter(_AtsAdapter):
class SmartRecruitersAdapter(_AtsAdapter):
parser_key = "ats-smartrecruiters"
parser_version = "1.1.0"
source_hosts = ("smartrecruiters.com",)
support_markers = ("smartrecruiters", "smart recruiter")
listing_paths = (("jobs",), ("data", "jobs"), ("results",))
listing_paths = (("jobs",), ("data", "jobs"), ("results",), ("content",))
detail_paths = (("job",), ("data", "job"), ("posting",), ("result",))
closed_statuses = CLOSED_STATUSES | {"unpublished", "expired"}
def _to_job(self, record: dict[str, object], base_url: str) -> ExtractedJob:
prepared = dict(record)
if not _first_text(prepared, "url", "jobUrl", "hostedUrl", "applyUrl", "link"):
company_identifier = _first_text(prepared, "company.identifier")
posting_id = _first_text(prepared, "id", "uuid")
if company_identifier and posting_id:
prepared["url"] = (
f"https://jobs.smartrecruiters.com/{company_identifier}/{posting_id}"
)
if _find_nested(prepared, "location.remote") is True:
prepared["remote"] = True
return super()._to_job(prepared, base_url)
def _extract_records(self, payload):
for path in self.listing_paths:
value = payload
+59 -3
View File
@@ -9,17 +9,58 @@ from urllib.parse import urljoin, urlsplit
from bs4 import BeautifulSoup
from apps.sources.platforms import PLATFORM_ALERTS
from apps.sources.services.canonicalize import canonicalize_url
from .base import ExtractedJob, ExtractionResult, FieldEvidence
URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I)
SKIP_TEXT = re.compile(r"unsubscribe|afmelden|uitschrijven|privacy|view in browser", re.I)
SKIP_TEXT = re.compile(
r"unsubscribe|afmelden|uitschrijven|privacy|view in browser|bekijk online|"
r"account|aanmelden|inloggen|login|voorkeuren|preferences|voorwaarden|terms|"
r"contact|help|hulp|over ons|about us",
re.I,
)
SKIP_PATH = re.compile(
r"/(?:unsubscribe|uitschrijven|afmelden|privacy|account|login|signin|preferences|"
r"settings|terms|legal|help|contact)(?:/|$)",
re.I,
)
def _matches_domain(hostname: str, domains: tuple[str, ...]) -> bool:
return any(hostname == domain or hostname.endswith(f".{domain}") for domain in domains)
class EmailAlertAdapter:
parser_key = "email-alert"
parser_version = "1.0.0"
parser_version = "1.1.0"
@staticmethod
def _provider_for_candidates(candidates: list[tuple[str, str]]) -> str:
hostnames = {
(urlsplit(href).hostname or "").lower()
for _, href in candidates
if href.lower().startswith(("http://", "https://"))
}
for provider, alert in PLATFORM_ALERTS.items():
if any(_matches_domain(hostname, alert.domains) for hostname in hostnames):
return provider
return "other"
@staticmethod
def _is_expected_platform_link(*, label: str, href: str, provider: str) -> bool:
alert = PLATFORM_ALERTS.get(provider)
if alert is None or len(label.strip()) < 4:
return False
parsed = urlsplit(href)
hostname = (parsed.hostname or "").lower()
return (
parsed.scheme.lower() == "https"
and _matches_domain(hostname, alert.domains)
and not SKIP_TEXT.search(label)
and not SKIP_PATH.search(parsed.path)
)
@staticmethod
def _decode_parts(message: Message) -> tuple[str, str]:
@@ -41,7 +82,9 @@ class EmailAlertAdapter:
html_parts.append(str(payload))
return "\n".join(plain_parts), "\n".join(html_parts)
def extract_message(self, raw_message: bytes) -> ExtractionResult:
def extract_message(
self, raw_message: bytes, *, expected_provider: str | None = None
) -> ExtractionResult:
message = BytesParser(policy=policy.default).parsebytes(raw_message)
plain, html_body = self._decode_parts(message)
candidates: list[tuple[str, str]] = []
@@ -61,6 +104,18 @@ class EmailAlertAdapter:
continue
candidates.append(("", clean_href))
alert_provider = expected_provider or self._provider_for_candidates(candidates)
if expected_provider:
candidates = [
(label, href)
for label, href in candidates
if self._is_expected_platform_link(
label=label,
href=href,
provider=expected_provider,
)
]
jobs: list[ExtractedJob] = []
seen: set[str] = set()
subject = str(message.get("subject") or "Vacature uit e-mail").strip()
@@ -84,6 +139,7 @@ class EmailAlertAdapter:
"email_subject": subject,
"email_sender": str(message.get("from") or ""),
"target_domain": hostname,
"alert_provider": alert_provider,
},
evidence=[FieldEvidence("url", "email-anchor", 0.75, label[:240])],
)
+290
View File
@@ -0,0 +1,290 @@
from __future__ import annotations
import re
from urllib.parse import parse_qs, urljoin, urlsplit
from bs4 import BeautifulSoup, Tag
from .base import ExtractedJob, ExtractionResult, FieldEvidence
class KempenEmployerAdapter:
"""Extract public employer lists around Mol from exact reviewed routes."""
parser_key = "regional-kempen-employers"
parser_version = "1.0.0"
supported_routes = {
"ziekenhuisgeel.careersite.be": {"/nl/vacatures"},
"geel.hro.be": {"/"},
"jobs.turnhout.be": {"/"},
"jobs.renotec.be": {"/nl/alle-jobs"},
"ravago.softgarden.io": {"/en/vacancies"},
"jobs.sanofi.com": {"/en/belgium"},
"www.daf.com": {"/nl-nl/werken-bij-daf/vacatures"},
}
def _route(self, url: str) -> tuple[str, str] | None:
parts = urlsplit(url)
host = (parts.hostname or "").lower()
path = parts.path.rstrip("/") or "/"
if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
return None
return host, path
def _supports_url(self, url: str) -> bool:
return self._route(url) is not None
@staticmethod
def _same_host_url(base_url: str, href: str) -> str:
job_url = urljoin(base_url, href)
parts = urlsplit(job_url)
if parts.scheme != "https":
return ""
if (parts.hostname or "").lower() != (urlsplit(base_url).hostname or "").lower():
return ""
return job_url
@staticmethod
def _job(
*,
url: str,
title: str,
employer: str,
location: str,
postal_code: str,
description: str,
external_id: str,
evidence_source: str,
) -> ExtractedJob:
return ExtractedJob(
url=url,
title=title,
employer_name=employer,
external_id=external_id,
location_text=location,
postal_code=postal_code,
region="Antwerpen",
country="BE",
description_text=description,
raw={"regional_listing": evidence_source, "region_center": "2400 Mol"},
evidence=[
FieldEvidence("title", evidence_source, 0.94, title[:240]),
FieldEvidence("employer_name", "reviewed-source", 0.97, employer),
FieldEvidence("location_text", "kempen-location-marker", 0.9, location),
],
)
def _ziekenhuis_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select(".vacature-tegel"):
link = card.select_one("a.vacature-tegel__link[href]")
title_node = card.select_one(".vacature-tegel__titel")
if not isinstance(link, Tag) or not isinstance(title_node, Tag):
continue
job_url = self._same_host_url(url, str(link.get("href") or ""))
match = re.fullmatch(r"/nl/vacature/(\d+)/[^/]+", urlsplit(job_url).path)
title = title_node.get_text(" ", strip=True)
if not title or not match:
continue
jobs.append(
self._job(
url=job_url,
title=title,
employer="Ziekenhuis Geel",
location="Geel, Antwerpen",
postal_code="2440",
description=card.get_text(" ", strip=True),
external_id=match.group(1),
evidence_source="ziekenhuis-geel-card",
)
)
return jobs
def _stad_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select(".vacatureKader[data-href]"):
href = str(card.get("data-href") or "")
match = re.fullmatch(r"vacature\.php\?id=(\d+)", href)
title_node = card.find("h4")
title = title_node.get_text(" ", strip=True) if title_node else ""
if not title or not match:
continue
jobs.append(
self._job(
url=self._same_host_url(url, href),
title=title,
employer="Lokaal bestuur Geel",
location="Geel, Antwerpen",
postal_code="2440",
description=card.get_text(" ", strip=True),
external_id=match.group(1),
evidence_source="stad-geel-hro-card",
)
)
return jobs
def _stad_turnhout(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.find_all("a", href=True):
href = str(link.get("href") or "")
match = re.fullmatch(r"vacature\.php\?id=(\d+)", href)
title_node = link.select_one(".block-update__body__title__inner")
title = title_node.get_text(" ", strip=True) if title_node else ""
if not title or not match:
continue
jobs.append(
self._job(
url=self._same_host_url(url, href),
title=title,
employer="Stad Turnhout",
location="Turnhout, Antwerpen",
postal_code="2300",
description=link.get_text(" ", strip=True),
external_id=match.group(1),
evidence_source="stad-turnhout-hro-card",
)
)
return jobs
def _renotec(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select(".s-tile.s-card"):
card_text = card.get_text(" ", strip=True)
if "Geel" not in card_text:
continue
title_node = card.find("h3")
link = card.find("a", href=True)
title = title_node.get_text(" ", strip=True) if title_node else ""
job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
parts = urlsplit(job_url)
query = parse_qs(parts.query)
external_id = (query.get("id") or [""])[0]
if parts.path != "/nl/detail/" or not external_id.isdigit() or not title:
continue
jobs.append(
self._job(
url=job_url,
title=title,
employer="Group Renotec",
location="Geel, Antwerpen (mogelijk meerdere werfregio's)",
postal_code="2440",
description=card_text,
external_id=external_id,
evidence_source="renotec-geel-card",
)
)
return jobs
def _ravago(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select(".matchElement"):
locations = {
node.get_text(" ", strip=True) for node in card.select(".location-view-item")
}
local_places = locations.intersection({"Arendonk", "Olen"})
if not local_places:
continue
link = card.find("a", href=True)
title = link.get_text(" ", strip=True) if link else ""
job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
match = re.fullmatch(r"/job/(\d+)/[^/]+/?", urlsplit(job_url).path)
if not title or not match:
continue
place = "Arendonk" if "Arendonk" in local_places else "Olen"
jobs.append(
self._job(
url=job_url,
title=title,
employer="Ravago",
location=f"{place}, Antwerpen",
postal_code="2370" if place == "Arendonk" else "2250",
description=card.get_text(" ", strip=True),
external_id=match.group(1),
evidence_source="ravago-softgarden-card",
)
)
return jobs
def _sanofi(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.select(".job-list a[data-job-id][href]"):
location_node = link.select_one(".job-location")
title_node = link.select_one(".job-title")
location = location_node.get_text(" ", strip=True) if location_node else ""
title = title_node.get_text(" ", strip=True) if title_node else ""
external_id = str(link.get("data-job-id") or "")
job_url = self._same_host_url(url, str(link.get("href") or ""))
if location != "Geel, Belgium" or not title or not external_id.isdigit():
continue
if not re.fullmatch(r"/en/job/geel/[^/]+/\d+/\d+", urlsplit(job_url).path):
continue
jobs.append(
self._job(
url=job_url,
title=title,
employer="Sanofi",
location="Geel, Antwerpen",
postal_code="2440",
description=link.get_text(" ", strip=True),
external_id=external_id,
evidence_source="sanofi-belgium-geel-card",
)
)
return jobs
def _daf(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select("li.itemlist__item"):
location_node = card.select_one(".js-vac-metalocation")
title_link = card.select_one("a.js-vac-title[href]")
location = location_node.get_text(" ", strip=True) if location_node else ""
title = title_link.get_text(" ", strip=True) if title_link else ""
job_url = (
self._same_host_url(url, str(title_link.get("href") or "")) if title_link else ""
)
path = urlsplit(job_url).path
prefix = "/nl-nl/werken-bij-daf/vacatures/"
if location != "Westerlo" or not title or not path.startswith(prefix):
continue
slug = path.removeprefix(prefix).strip("/")
if not slug or "/" in slug:
continue
jobs.append(
self._job(
url=job_url,
title=title,
employer="DAF Trucks",
location="Westerlo, Antwerpen",
postal_code="2260",
description=card.get_text(" ", strip=True),
external_id=slug,
evidence_source="daf-westerlo-card",
)
)
return jobs
def extract(self, content: str, *, url: str) -> ExtractionResult:
route = self._route(url)
if route is None:
return ExtractionResult(
[], self.parser_key, self.parser_version, 0.0, ["Onherkende Kempen-bron"]
)
host, _ = route
extractors = {
"ziekenhuisgeel.careersite.be": self._ziekenhuis_geel,
"geel.hro.be": self._stad_geel,
"jobs.turnhout.be": self._stad_turnhout,
"jobs.renotec.be": self._renotec,
"ravago.softgarden.io": self._ravago,
"jobs.sanofi.com": self._sanofi,
"www.daf.com": self._daf,
}
jobs = extractors[host](BeautifulSoup(content, "lxml"), url)
unique_jobs = list({job.url: job for job in jobs if job.url}.values())
return ExtractionResult(
unique_jobs,
self.parser_key,
self.parser_version,
0.92 if unique_jobs else 0.0,
[] if unique_jobs else ["Geen actuele regionale vacatures gevonden"],
)
+253
View File
@@ -0,0 +1,253 @@
from __future__ import annotations
import re
from urllib.parse import parse_qs, urljoin, urlsplit
from bs4 import BeautifulSoup
from .base import ExtractedJob, ExtractionResult, FieldEvidence
class MolRegionEmployerAdapter:
"""Extract reviewed employer listings around postcode 2400 without detail fetches."""
parser_key = "regional-mol-employers"
parser_version = "1.0.0"
thomas_more_company_guid = "eab9ca13-ee10-4504-8a87-a785d0b037ef"
supported_routes = {
"www.sckcen.be": {"/nl/carriere/vacatures"},
"cipalschaubroeck.teamtailor.com": {"/jobs"},
"www.vanroey.be": {"/en/job-overview"},
"netropolix.recruitee.com": {"/"},
"jobpage.cvwarehouse.com": {"/"},
}
def _route(self, url: str) -> tuple[str, str] | None:
parts = urlsplit(url)
host = (parts.hostname or "").lower()
path = parts.path.rstrip("/") or "/"
if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
return None
if host == "jobpage.cvwarehouse.com":
query = parse_qs(parts.query)
if query.get("companyGuid") != [self.thomas_more_company_guid] or "job" in query:
return None
return host, path
def _supports_url(self, url: str) -> bool:
return self._route(url) is not None
@staticmethod
def _same_host_url(base_url: str, href: str) -> str:
job_url = urljoin(base_url, href)
if urlsplit(job_url).scheme != "https":
return ""
if (urlsplit(job_url).hostname or "").lower() != (
urlsplit(base_url).hostname or ""
).lower():
return ""
return job_url
@staticmethod
def _job(
*,
url: str,
title: str,
employer: str,
location: str,
postal_code: str,
description: str,
external_id: str,
evidence_source: str,
) -> ExtractedJob:
return ExtractedJob(
url=url,
title=title,
employer_name=employer,
external_id=external_id,
location_text=location,
postal_code=postal_code,
region="Antwerpen",
country="BE",
description_text=description,
raw={"regional_listing": evidence_source, "region_center": "2400 Mol"},
evidence=[
FieldEvidence("title", evidence_source, 0.94, title[:240]),
FieldEvidence("employer_name", "reviewed-source", 0.96, employer),
FieldEvidence("location_text", "mol-region-review", 0.82, location),
],
)
def _sck_cen(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.select("article a[href]"):
job_url = self._same_host_url(url, str(link.get("href") or ""))
path = urlsplit(job_url).path if job_url else ""
if not path.startswith("/nl/carriere/vacatures/"):
continue
title = link.get_text(" ", strip=True)
if not title:
continue
card = link.find_parent("article")
description = card.get_text(" ", strip=True) if card else title
jobs.append(
self._job(
url=job_url,
title=title,
employer="SCK CEN",
location="Mol, Antwerpen",
postal_code="2400",
description=description,
external_id=path.rstrip("/").rsplit("/", 1)[-1],
evidence_source="sck-cen-vacancy-card",
)
)
return jobs
def _cipal(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select("li"):
card_text = card.get_text(" ", strip=True)
if "Westerlo" not in card_text and "Geel" not in card_text:
continue
link = card.find("a", href=True)
if not link:
continue
title = link.get_text(" ", strip=True)
job_url = self._same_host_url(url, str(link.get("href") or ""))
path = urlsplit(job_url).path if job_url else ""
match = re.fullmatch(r"/jobs/(\d+)-[^/]+", path.rstrip("/"))
if not title or not match:
continue
location = "Geel, Antwerpen" if "Geel" in card_text else "Westerlo, Antwerpen"
jobs.append(
self._job(
url=job_url,
title=title,
employer="Cipal Schaubroeck",
location=location,
postal_code="2440" if "Geel" in card_text else "2260",
description=card_text,
external_id=match.group(1),
evidence_source="cipal-teamtailor-card",
)
)
return jobs
def _vanroey(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for heading in soup.select("h3.elementor-heading-title"):
link = heading.find("a", href=True)
if not link:
continue
title = link.get_text(" ", strip=True)
job_url = self._same_host_url(url, str(link.get("href") or ""))
path = urlsplit(job_url).path if job_url else ""
if not title or not re.fullmatch(r"/en/job/[^/]+/", path):
continue
slug = path.rstrip("/").rsplit("/", 1)[-1]
if "oost-vlaanderen" in slug:
continue
card = heading.find_parent("section") or heading.parent
description = card.get_text(" ", strip=True) if card else title
jobs.append(
self._job(
url=job_url,
title=title,
employer="VanRoey",
location="Turnhout/Geel (hybride; controleer vacaturedetail)",
postal_code="",
description=description,
external_id=slug,
evidence_source="vanroey-job-card",
)
)
return jobs
def _netropolix(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.find_all("a", href=True):
title = link.get_text(" ", strip=True)
if "Geel" not in title:
continue
job_url = self._same_host_url(url, str(link.get("href") or ""))
path = urlsplit(job_url).path if job_url else ""
if not re.fullmatch(r"/o/[^/]+", path.rstrip("/")):
continue
card = link.find_parent("div")
description = card.get_text(" ", strip=True) if card else title
jobs.append(
self._job(
url=job_url,
title=title,
employer="NTX (Netropolix)",
location="Geel, Antwerpen",
postal_code="2440",
description=description,
external_id=path.rstrip("/").rsplit("/", 1)[-1],
evidence_source="netropolix-recruitee-card",
)
)
return jobs
def _thomas_more(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.select("a.jobLink[data-item='readmore'][data-jobid][href]"):
title = link.get_text(" ", strip=True)
title_casefold = title.casefold()
if not any(place in title_casefold for place in ("geel", "turnhout", "vorselaar")):
continue
external_id = str(link.get("data-jobid") or "")
if not external_id.isdigit():
continue
job_url = self._same_host_url(url, str(link.get("href") or ""))
query = parse_qs(urlsplit(job_url).query) if job_url else {}
if query.get("companyGuid") != [self.thomas_more_company_guid] or query.get("job") != [
external_id
]:
continue
if "turnhout" in title_casefold:
location, postal_code = "Turnhout, Antwerpen", "2300"
elif "vorselaar" in title_casefold:
location, postal_code = "Vorselaar, Antwerpen", "2290"
else:
location, postal_code = "Geel, Antwerpen", "2440"
jobs.append(
self._job(
url=job_url,
title=title,
employer="Thomas More",
location=location,
postal_code=postal_code,
description=f"Publieke regionale vacature bij Thomas More: {title}.",
external_id=external_id,
evidence_source="thomas-more-cvwarehouse-card",
)
)
return jobs
def extract(self, content: str, *, url: str) -> ExtractionResult:
route = self._route(url)
if route is None:
return ExtractionResult(
[], self.parser_key, self.parser_version, 0.0, ["Onherkende Mol-regiobron"]
)
host, _ = route
soup = BeautifulSoup(content, "lxml")
extractors = {
"www.sckcen.be": self._sck_cen,
"cipalschaubroeck.teamtailor.com": self._cipal,
"www.vanroey.be": self._vanroey,
"netropolix.recruitee.com": self._netropolix,
"jobpage.cvwarehouse.com": self._thomas_more,
}
jobs = extractors[host](soup, url)
unique_jobs = list({job.url: job for job in jobs}.values())
warnings = [] if unique_jobs else ["Geen actuele vacatures rond Mol gevonden"]
return ExtractionResult(
unique_jobs,
self.parser_key,
self.parser_version,
0.9 if unique_jobs else 0.0,
warnings,
)
+312
View File
@@ -0,0 +1,312 @@
from __future__ import annotations
from urllib.parse import urljoin, urlsplit
from bs4 import BeautifulSoup
from .base import ExtractedJob, ExtractionResult, FieldEvidence
class CordaCampusAdapter:
"""Extract the public job cards intentionally published by Corda Campus."""
parser_key = "regional-corda-campus"
parser_version = "1.0.0"
source_hosts = ("cordacampus.com",)
def _supports_url(self, url: str) -> bool:
parts = urlsplit(url)
host = (parts.hostname or "").lower()
return host.endswith(self.source_hosts) and parts.path.rstrip("/") == "/jobs"
def extract(self, content: str, *, url: str) -> ExtractionResult:
if not self._supports_url(url):
return ExtractionResult(
[], self.parser_key, self.parser_version, 0.0, ["Onherkenbare regiobron"]
)
source_host = (urlsplit(url).hostname or "").lower()
soup = BeautifulSoup(content, "lxml")
jobs: list[ExtractedJob] = []
seen_urls: set[str] = set()
for card in soup.select('a.event-item[href*="/job/"]'):
job_url = urljoin(url, str(card.get("href") or ""))
job_host = (urlsplit(job_url).hostname or "").lower()
if job_host != source_host or job_url in seen_urls:
continue
title_node = card.select_one(".job-title")
employer_node = card.select_one(".company-title")
title = title_node.get_text(" ", strip=True) if title_node else ""
employer = employer_node.get_text(" ", strip=True) if employer_node else ""
if not title:
continue
date_node = card.select_one(".bottom-info")
date_posted = date_node.get_text(" ", strip=True) if date_node else ""
external_id = urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1]
description = f"Vacature van {employer or 'een Corda-werkgever'} via Corda Campus."
jobs.append(
ExtractedJob(
url=job_url,
title=title,
employer_name=employer,
external_id=external_id,
location_text="Hasselt, Limburg",
region="Limburg",
country="BE",
description_text=description,
date_posted=date_posted,
raw={"regional_listing": "corda-campus"},
evidence=[
FieldEvidence("title", "corda-job-card", 0.95, title[:240]),
FieldEvidence("employer_name", "corda-job-card", 0.92, employer[:240]),
FieldEvidence(
"location_text",
"regional-source-scope",
0.72,
"Corda Campus, Hasselt",
),
],
)
)
seen_urls.add(job_url)
warnings = [] if jobs else ["Geen actuele Corda-vacatures gevonden"]
return ExtractionResult(
jobs,
self.parser_key,
self.parser_version,
0.9 if jobs else 0.0,
warnings,
)
class LocalEmployerListingAdapter:
"""Extract reviewed public listing pages of employers in the Hasselt-Genk area."""
parser_key = "regional-local-employers"
parser_version = "1.0.0"
supported_routes = {
"acagroup.be": {"/en/jobs"},
"www.xploregroup.be": {"/en/jobs"},
"www.uhasselt.be": {"/vacatures"},
"ses.pxl.be": {"/"},
"ziekenhuis-oost-limburg.cvw.io": {"/"},
}
def _supports_url(self, url: str) -> bool:
return self._route(url) is not None
def _route(self, url: str) -> tuple[str, str] | None:
parts = urlsplit(url)
host = (parts.hostname or "").lower()
path = parts.path.rstrip("/") or "/"
if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
return None
return host, path
@staticmethod
def _job(
*,
url: str,
title: str,
employer: str,
location: str,
description: str,
external_id: str,
evidence_source: str,
) -> ExtractedJob:
return ExtractedJob(
url=url,
title=title,
employer_name=employer,
external_id=external_id,
location_text=location,
region="Limburg",
country="BE",
description_text=description,
raw={"regional_listing": evidence_source},
evidence=[
FieldEvidence("title", evidence_source, 0.94, title[:240]),
FieldEvidence("employer_name", "reviewed-source", 0.96, employer),
FieldEvidence("location_text", "regional-source-scope", 0.78, location),
],
)
@staticmethod
def _same_host_url(base_url: str, href: str) -> str:
job_url = urljoin(base_url, href)
if (urlsplit(job_url).hostname or "").lower() != (
urlsplit(base_url).hostname or ""
).lower():
return ""
return job_url
def _aca(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.find_all("a", href=True):
href = str(link.get("href") or "")
if not href.startswith("/en/jobs/"):
continue
title_node = link.find("h3")
title = title_node.get_text(" ", strip=True) if title_node else ""
job_url = self._same_host_url(url, href)
if not title or not job_url:
continue
description_node = link.find("p")
description = (
description_node.get_text(" ", strip=True)
if description_node
else f"Vacature bij ACA Group: {title}."
)
jobs.append(
self._job(
url=job_url,
title=title,
employer="ACA Group",
location="Hasselt (hybride; kantoorselectie per vacature)",
description=description,
external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1],
evidence_source="aca-job-card",
)
)
return jobs
def _xplore(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for heading in soup.find_all("h3"):
link = heading.find("a", href=True)
if not link or not str(link.get("href") or "").startswith("/en/jobs/"):
continue
card = heading.parent
location_node = next(
(
node
for node in card.find_all("p")
if "Hasselt" in node.get_text(" ", strip=True)
),
None,
)
if not location_node:
continue
title = link.get_text(" ", strip=True)
job_url = self._same_host_url(url, str(link.get("href") or ""))
if not title or not job_url:
continue
location = location_node.get_text(" ", strip=True)
jobs.append(
self._job(
url=job_url,
title=title,
employer="Xplore Group",
location=location,
description=(
f"Vacature bij Xplore Group met Hasselt als mogelijke werklocatie: {title}."
),
external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1],
evidence_source="xplore-job-card",
)
)
return jobs
def _uhasselt(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select("section.vacancy-item"):
title_node = card.find("h3")
link = card.find("a", href=True)
title = title_node.get_text(" ", strip=True) if title_node else ""
job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
if not title or not job_url or "/vacatures/detail/" not in urlsplit(job_url).path:
continue
external_id = urlsplit(job_url).path.split("/detail/", 1)[-1].split("-", 1)[0]
jobs.append(
self._job(
url=job_url,
title=title,
employer="Universiteit Hasselt",
location="Hasselt/Diepenbeek, Limburg",
description=card.get_text(" ", strip=True),
external_id=external_id,
evidence_source="uhasselt-vacancy-card",
)
)
return jobs
def _pxl(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select(".vacature-card[id^='Vacature_']"):
external_id = str(card.get("id") or "").removeprefix("Vacature_")
title_node = card.select_one(".vacature-card-titel")
title = title_node.get_text(" ", strip=True) if title_node else ""
if not title or not external_id.isdigit():
continue
cells = [
node.get_text(" ", strip=True) for node in card.select("td.vacature-card-td-text")
]
campus = next((value for value in cells if value.startswith("Campus ")), "Hasselt")
job_url = f"{url.rstrip('/')}?vacature_id={external_id}"
jobs.append(
self._job(
url=job_url,
title=title,
employer="Hogeschool PXL",
location=f"{campus}, Limburg",
description=" · ".join(cells),
external_id=external_id,
evidence_source="pxl-vacancy-card",
)
)
return jobs
def _zol(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.select("a[data-item='readmore'][data-jobid][href]"):
external_id = str(link.get("data-jobid") or "")
title_node = link.select_one(".job-title")
title = title_node.get_text(" ", strip=True) if title_node else ""
job_url = self._same_host_url(url, str(link.get("href") or ""))
if not title or not external_id.isdigit() or not job_url:
continue
jobs.append(
self._job(
url=job_url,
title=title,
employer="Ziekenhuis Oost-Limburg",
location="Genk/Lanaken/Maaseik, Limburg",
description=f"Publieke vacature van Ziekenhuis Oost-Limburg: {title}.",
external_id=external_id,
evidence_source="zol-cvwarehouse-card",
)
)
return jobs
def extract(self, content: str, *, url: str) -> ExtractionResult:
route = self._route(url)
if route is None:
return ExtractionResult(
[],
self.parser_key,
self.parser_version,
0.0,
["Onherkenbare lokale werkgeversbron"],
)
host, _ = route
soup = BeautifulSoup(content, "lxml")
extractors = {
"acagroup.be": self._aca,
"www.xploregroup.be": self._xplore,
"www.uhasselt.be": self._uhasselt,
"ses.pxl.be": self._pxl,
"ziekenhuis-oost-limburg.cvw.io": self._zol,
}
jobs = extractors[host](soup, url)
unique_jobs = list({job.url: job for job in jobs}.values())
warnings = [] if unique_jobs else ["Geen actuele lokale vacatures gevonden"]
return ExtractionResult(
unique_jobs,
self.parser_key,
self.parser_version,
0.9 if unique_jobs else 0.0,
warnings,
)
+11
View File
@@ -12,6 +12,9 @@ from .ats import (
from .base import ExtractionResult
from .generic_html import GenericHtmlAdapter
from .jsonld import JsonLdJobPostingAdapter
from .kempen import KempenEmployerAdapter
from .mol_region import MolRegionEmployerAdapter
from .regional import CordaCampusAdapter, LocalEmployerListingAdapter
from .rss import RssAdapter
@@ -22,10 +25,18 @@ class AdapterRegistry:
self.recruitee = RecruiteeAdapter()
self.smartrecruiters = SmartRecruitersAdapter()
self.workable = WorkableAdapter()
self.corda_campus = CordaCampusAdapter()
self.local_employers = LocalEmployerListingAdapter()
self.mol_region_employers = MolRegionEmployerAdapter()
self.kempen_employers = KempenEmployerAdapter()
self.jsonld = JsonLdJobPostingAdapter()
self.generic = GenericHtmlAdapter()
self.rss = RssAdapter()
self.providers = [
self.corda_campus,
self.local_employers,
self.mol_region_employers,
self.kempen_employers,
self.greenhouse,
self.lever,
self.recruitee,
+17
View File
@@ -2,6 +2,7 @@ from django.contrib import admin
from .models import (
EmailMessageRecord,
MailboxConnection,
RawDocument,
Source,
SourceLease,
@@ -12,6 +13,22 @@ from .models import (
)
@admin.register(MailboxConnection)
class MailboxConnectionAdmin(admin.ModelAdmin):
list_display = (
"platform",
"provider",
"username",
"enabled",
"poll_interval_minutes",
"last_success_at",
"next_poll_at",
)
list_filter = ("platform", "provider", "enabled")
search_fields = ("username",)
exclude = ("encrypted_password", "lease_token")
@admin.register(Source)
class SourceAdmin(admin.ModelAdmin):
list_display = (
+52
View File
@@ -2,6 +2,58 @@ from __future__ import annotations
from django import forms
from .models import MailboxConnection
class MailboxConnectionForm(forms.ModelForm):
password = forms.CharField(
required=False,
label="App-wachtwoord",
help_text="Laat leeg bij wijzigen om het bestaande app-wachtwoord te behouden.",
widget=forms.PasswordInput(
attrs={"autocomplete": "new-password", "placeholder": "App-wachtwoord"},
render_value=False,
),
)
class Meta:
model = MailboxConnection
fields = (
"platform",
"provider",
"custom_host",
"port",
"username",
"mailbox",
"poll_interval_minutes",
"enabled",
)
labels = {
"platform": "Vacatureplatform",
"provider": "Mailboxprovider",
"custom_host": "Aangepaste IMAP-host",
"port": "IMAP-poort",
"username": "Mailboxaccount",
"mailbox": "Map",
"poll_interval_minutes": "Controlefrequentie",
"enabled": "Automatisch synchroniseren",
}
widgets = {
"username": forms.EmailInput(attrs={"autocomplete": "username"}),
"custom_host": forms.TextInput(
attrs={"autocomplete": "off", "placeholder": "imap.provider.be"}
),
"mailbox": forms.TextInput(attrs={"autocomplete": "off"}),
}
def clean(self):
data = super().clean()
if data.get("provider") != MailboxConnection.Provider.CUSTOM:
data["custom_host"] = ""
if not self.instance.pk and not data.get("password"):
self.add_error("password", "Een app-wachtwoord is verplicht voor een nieuwe koppeling.")
return data
class ManualImportForm(forms.Form):
source_url = forms.URLField(
@@ -31,11 +31,11 @@ class Command(BaseCommand):
if not hostname:
raise CommandError("--url bevat geen geldig domein")
source, _ = Source.objects.get_or_create(
domain=hostname,
base_url=options["url"],
source_type=options["source_type"],
defaults={
"domain": hostname,
"name": options["source_name"],
"base_url": options["url"],
"status": Source.Status.ACTIVE,
"policy": Source.Policy.ALLOW,
"parser_key": "auto",
@@ -0,0 +1,17 @@
from django.core.management.base import BaseCommand, CommandError
from apps.sources.services.mailbox_connections import (
MailboxCredentialError,
rotate_mailbox_credentials,
)
class Command(BaseCommand):
help = "Versleutel alle mailbox-app-wachtwoorden opnieuw met de eerste ingestelde sleutel."
def handle(self, *args, **options):
try:
count = rotate_mailbox_credentials()
except MailboxCredentialError as exc:
raise CommandError(str(exc)) from exc
self.stdout.write(self.style.SUCCESS(f"{count} mailboxcredential(s) geroteerd."))
@@ -5,8 +5,9 @@ from urllib.parse import urlsplit
import yaml
from django.core.management.base import BaseCommand, CommandError
from django.utils.dateparse import parse_datetime
from apps.sources.models import Source
from apps.sources.models import Source, SourcePolicyReview
from apps.sources.services.policy import is_denied_domain
@@ -21,7 +22,38 @@ class Command(BaseCommand):
if not path.is_file():
raise CommandError(f"Seedbestand ontbreekt: {path}")
payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
created = updated = 0
created = updated = reviews_created = 0
if payload.get("disable_demo_sources") is True:
demo_sources = Source.objects.filter(domain="jobs.example.org")
for demo_source in demo_sources:
demo_source.status = Source.Status.DISABLED
demo_source.policy = Source.Policy.DENY
demo_source.policy_reason = "Demobron verborgen na installatie van livebronnen"
demo_source.metadata = {**demo_source.metadata, "demo": True}
demo_source.save(
update_fields=[
"status",
"policy",
"policy_reason",
"metadata",
"updated_at",
]
)
retired = 0
for retired_url in payload.get("retire_source_urls", []):
for retired_source in Source.objects.filter(base_url=str(retired_url).strip()):
retired_source.status = Source.Status.DISABLED
retired_source.next_run_at = None
retired_source.metadata = {
**retired_source.metadata,
"retired": True,
"retired_reason": "Buiten de ingestelde regionale dekking",
"hidden_from_source_list": True,
}
retired_source.save(
update_fields=["status", "next_run_at", "metadata", "updated_at"]
)
retired += 1
for item in payload.get("sources", []):
url = str(item.get("url") or "").strip()
domain = (urlsplit(url).hostname or "").lower()
@@ -30,11 +62,11 @@ class Command(BaseCommand):
continue
deny = is_denied_domain(domain)
source, was_created = Source.objects.update_or_create(
domain=domain,
base_url=url,
source_type=item.get("type", Source.Type.EMPLOYER),
defaults={
"domain": domain,
"name": item.get("name") or domain,
"base_url": url,
"status": Source.Status.PAUSED
if deny
else item.get("status", Source.Status.TRIAL),
@@ -46,10 +78,39 @@ class Command(BaseCommand):
else item.get("policy_reason", ""),
"parser_key": item.get("parser", "auto"),
"crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)),
"minimum_interval_seconds": int(item.get("minimum_interval_seconds", 30)),
"allow_public_endpoint": bool(item.get("allow_public_endpoint", False)),
"metadata": item.get("metadata") or {},
},
)
review = item.get("review") or {}
if review:
expires_at = parse_datetime(str(review.get("expires_at") or ""))
if expires_at is None:
raise CommandError(
f"Bron {source.name}: review.expires_at moet een ISO-datetime zijn"
)
decision = str(review.get("decision") or "")
if decision not in SourcePolicyReview.Decision.values:
raise CommandError(f"Bron {source.name}: ongeldige reviewbeslissing")
_, review_created = SourcePolicyReview.objects.get_or_create(
source=source,
decision=decision,
evidence_link=str(review.get("evidence_link") or ""),
expires_at=expires_at,
defaults={
"scope": SourcePolicyReview.Scope.SOURCE,
"reason": str(review.get("reason") or "Bronseedreview"),
"notes": str(review.get("notes") or ""),
"metadata": review.get("metadata") or {},
},
)
reviews_created += int(review_created)
created += int(was_created)
updated += int(not was_created)
self.stdout.write(
self.style.SUCCESS(f"Bronnen: {created} aangemaakt, {updated} bijgewerkt")
self.style.SUCCESS(
f"Bronnen: {created} aangemaakt, {updated} bijgewerkt, "
f"{reviews_created} reviews geregistreerd, {retired} buiten regio uitgeschakeld"
)
)
@@ -0,0 +1,21 @@
# Generated by Django 5.2.16 on 2026-07-21 22:59
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sources', '0003_source_lease_and_origin_state'),
]
operations = [
migrations.RemoveConstraint(
model_name='source',
name='unique_domain_source_type',
),
migrations.AddConstraint(
model_name='source',
constraint=models.UniqueConstraint(condition=models.Q(('base_url', ''), _negated=True), fields=('source_type', 'base_url'), name='unique_source_type_base_url_nonempty'),
),
]
@@ -0,0 +1,67 @@
# Generated by Django 5.2.16 on 2026-07-22 00:12
import django.db.models.deletion
import django.utils.timezone
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sources', '0004_remove_source_unique_domain_source_type_and_more'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AlterField(
model_name='emailmessagerecord',
name='message_id',
field=models.CharField(max_length=998),
),
migrations.CreateModel(
name='MailboxConnection',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('platform', models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed')], max_length=24)),
('provider', models.CharField(choices=[('gmail', 'Gmail'), ('outlook', 'Outlook / Microsoft 365'), ('custom', 'Andere IMAP-provider')], max_length=24)),
('custom_host', models.CharField(blank=True, max_length=255)),
('port', models.PositiveIntegerField(default=993)),
('username', models.CharField(max_length=320)),
('encrypted_password', models.TextField()),
('mailbox', models.CharField(default='INBOX', max_length=255)),
('enabled', models.BooleanField(default=True)),
('poll_interval_minutes', models.PositiveIntegerField(choices=[(5, 'Elke 5 minuten'), (15, 'Elke 15 minuten'), (30, 'Elke 30 minuten'), (60, 'Elk uur'), (180, 'Elke 3 uur'), (360, 'Elke 6 uur')], default=30)),
('next_poll_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
('last_polled_at', models.DateTimeField(blank=True, null=True)),
('last_success_at', models.DateTimeField(blank=True, null=True)),
('last_error_category', models.CharField(blank=True, max_length=80)),
('last_error_message', models.CharField(blank=True, max_length=500)),
('lease_token', models.CharField(blank=True, max_length=64)),
('lease_expires_at', models.DateTimeField(blank=True, db_index=True, null=True)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mailbox_connections', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['platform'],
},
),
migrations.AddField(
model_name='emailmessagerecord',
name='mailbox_connection',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='sources.mailboxconnection'),
),
migrations.AddConstraint(
model_name='emailmessagerecord',
constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', False)), fields=('mailbox_connection', 'message_id'), name='unique_message_per_mailbox_connection'),
),
migrations.AddConstraint(
model_name='emailmessagerecord',
constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', True)), fields=('message_id',), name='unique_legacy_email_message_id'),
),
migrations.AddConstraint(
model_name='mailboxconnection',
constraint=models.UniqueConstraint(fields=('user', 'platform'), name='unique_mailbox_platform_per_user'),
),
]
@@ -0,0 +1,18 @@
# Generated by Django 5.2.16 on 2026-07-22 00:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sources', '0005_alter_emailmessagerecord_message_id_and_more'),
]
operations = [
migrations.AlterField(
model_name='mailboxconnection',
name='platform',
field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('eurobrussels', 'EuroBrussels'), ('brusselsjobs', 'BrusselsJobs')], max_length=24),
),
]
@@ -0,0 +1,26 @@
# Generated by Django 5.2.16 on 2026-07-22 01:25
from django.db import migrations, models
def disable_retired_brussels_mailboxes(apps, schema_editor):
mailbox_connection = apps.get_model("sources", "MailboxConnection")
mailbox_connection.objects.filter(
platform__in=("eurobrussels", "brusselsjobs")
).update(enabled=False)
class Migration(migrations.Migration):
dependencies = [
('sources', '0006_alter_mailboxconnection_platform'),
]
operations = [
migrations.RunPython(disable_retired_brussels_mailboxes, migrations.RunPython.noop),
migrations.AlterField(
model_name='mailboxconnection',
name='platform',
field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('careerjet', 'Careerjet'), ('randstad', 'Randstad'), ('roberthalf', 'Robert Half')], max_length=24),
),
]
+109 -2
View File
@@ -62,7 +62,9 @@ class Source(TimeStampedModel):
ordering = ["name"]
constraints = [
models.UniqueConstraint(
fields=["domain", "source_type"], name="unique_domain_source_type"
fields=["source_type", "base_url"],
condition=~models.Q(base_url=""),
name="unique_source_type_base_url_nonempty",
)
]
@@ -315,8 +317,101 @@ class RawDocument(TimeStampedModel):
return self.final_url or self.url or f"Document {self.pk}"
class MailboxConnection(TimeStampedModel):
class Platform(models.TextChoices):
VDAB = "vdab", "VDAB"
INDEED = "indeed", "Indeed"
LINKEDIN = "linkedin", "LinkedIn"
ICTJOB = "ictjob", "ictjob.be"
JOBAT = "jobat", "Jobat"
STEPSTONE = "stepstone", "StepStone"
CAREERJET = "careerjet", "Careerjet"
RANDSTAD = "randstad", "Randstad"
ROBERTHALF = "roberthalf", "Robert Half"
class Provider(models.TextChoices):
GMAIL = "gmail", "Gmail"
OUTLOOK = "outlook", "Outlook / Microsoft 365"
CUSTOM = "custom", "Andere IMAP-provider"
class PollInterval(models.IntegerChoices):
FIVE_MINUTES = 5, "Elke 5 minuten"
FIFTEEN_MINUTES = 15, "Elke 15 minuten"
THIRTY_MINUTES = 30, "Elke 30 minuten"
HOURLY = 60, "Elk uur"
THREE_HOURS = 180, "Elke 3 uur"
SIX_HOURS = 360, "Elke 6 uur"
PROVIDER_HOSTS = {
Provider.GMAIL: "imap.gmail.com",
Provider.OUTLOOK: "outlook.office365.com",
}
user = models.ForeignKey(
settings.AUTH_USER_MODEL,
on_delete=models.CASCADE,
related_name="mailbox_connections",
)
platform = models.CharField(max_length=24, choices=Platform.choices)
provider = models.CharField(max_length=24, choices=Provider.choices)
custom_host = models.CharField(max_length=255, blank=True)
port = models.PositiveIntegerField(default=993)
username = models.CharField(max_length=320)
encrypted_password = models.TextField()
mailbox = models.CharField(max_length=255, default="INBOX")
enabled = models.BooleanField(default=True)
poll_interval_minutes = models.PositiveIntegerField(
choices=PollInterval.choices,
default=PollInterval.THIRTY_MINUTES,
)
next_poll_at = models.DateTimeField(default=timezone.now, db_index=True)
last_polled_at = models.DateTimeField(null=True, blank=True)
last_success_at = models.DateTimeField(null=True, blank=True)
last_error_category = models.CharField(max_length=80, blank=True)
last_error_message = models.CharField(max_length=500, blank=True)
lease_token = models.CharField(max_length=64, blank=True)
lease_expires_at = models.DateTimeField(null=True, blank=True, db_index=True)
class Meta:
ordering = ["platform"]
constraints = [
models.UniqueConstraint(
fields=["user", "platform"], name="unique_mailbox_platform_per_user"
)
]
def __str__(self) -> str:
return f"{self.get_platform_display()} · {self.username}"
@property
def imap_host(self) -> str:
return self.PROVIDER_HOSTS.get(self.provider, self.custom_host).strip().lower()
def clean(self) -> None:
host = self.imap_host
if (
not host
or "://" in host
or "/" in host
or "@" in host
or any(c.isspace() for c in host)
):
raise ValidationError({"custom_host": "Vul een geldige IMAP-hostnaam in."})
if not 1 <= self.port <= 65535:
raise ValidationError({"port": "De IMAP-poort moet tussen 1 en 65535 liggen."})
if "\n" in self.mailbox or "\r" in self.mailbox:
raise ValidationError({"mailbox": "De mailboxnaam bevat ongeldige tekens."})
class EmailMessageRecord(TimeStampedModel):
message_id = models.CharField(max_length=998, unique=True)
mailbox_connection = models.ForeignKey(
MailboxConnection,
on_delete=models.CASCADE,
null=True,
blank=True,
related_name="messages",
)
message_id = models.CharField(max_length=998)
mailbox = models.CharField(max_length=255, default="INBOX")
sender = models.CharField(max_length=500, blank=True)
subject = models.CharField(max_length=998, blank=True)
@@ -330,6 +425,18 @@ class EmailMessageRecord(TimeStampedModel):
class Meta:
ordering = ["-received_at", "-created_at"]
constraints = [
models.UniqueConstraint(
fields=["mailbox_connection", "message_id"],
condition=models.Q(mailbox_connection__isnull=False),
name="unique_message_per_mailbox_connection",
),
models.UniqueConstraint(
fields=["message_id"],
condition=models.Q(mailbox_connection__isnull=True),
name="unique_legacy_email_message_id",
),
]
def __str__(self) -> str:
return self.subject or self.message_id
+88
View File
@@ -0,0 +1,88 @@
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True)
class PlatformAlert:
key: str
label: str
domains: tuple[str, ...]
alert_url: str
help_text: str
PLATFORM_ALERTS = {
alert.key: alert
for alert in (
PlatformAlert(
"vdab",
"VDAB",
("vdab.be",),
"https://www.vdab.be/jobs/job-alerts",
"Brede Vlaamse vacaturealerts.",
),
PlatformAlert(
"indeed",
"Indeed",
("indeed.com", "indeed.be"),
"https://support.indeed.com/hc/nl/articles/204488890-"
"Vacature-alerts-activeren-stopzetten-en-beheren",
"Algemene en regionale vacaturealerts.",
),
PlatformAlert(
"linkedin",
"LinkedIn",
("linkedin.com",),
"https://www.linkedin.com/help/linkedin/answer/a511279",
"Dagelijkse of wekelijkse alerts, maximaal twintig per LinkedIn-account.",
),
PlatformAlert(
"ictjob",
"ictjob.be",
("ictjob.be",),
"https://www.ictjob.be/nl/it-job-alert",
"Gespecialiseerde Belgische IT- en telecomvacatures.",
),
PlatformAlert(
"jobat",
"Jobat",
("jobat.be",),
"https://www.jobat.be/nl/jobalert",
"Belgische jobs per functie, regio en contracttype.",
),
PlatformAlert(
"stepstone",
"StepStone",
("stepstone.be",),
"https://www.stepstone.be/vacatures",
"Dagelijkse vergelijkbare vacatures vanuit een zoekopdracht.",
),
PlatformAlert(
"careerjet",
"Careerjet",
("careerjet.be",),
"https://www.careerjet.be/",
"Brede Belgische zoekresultaten met regio- en postcodegebonden e-mailalerts.",
),
PlatformAlert(
"randstad",
"Randstad",
("randstad.be",),
"https://www.randstad.be/jobalert-aanmaken/",
"Belgische jobs met instelbare specialisatie, locatie, contract en frequentie.",
),
PlatformAlert(
"roberthalf",
"Robert Half",
("roberthalf.com",),
"https://www.roberthalf.com/be/en/find-jobs/job-alerts",
"Belgische IT- en digitaljobs, waaronder systeem-, netwerk- en supportfuncties.",
),
)
}
def platform_label(key: str) -> str:
alert = PLATFORM_ALERTS.get(key)
return alert.label if alert else key
+21 -16
View File
@@ -140,23 +140,28 @@ def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int
for candidate in ordered:
provenance = _provenance_entry(candidate)
now = _utc_now()
source, was_created = Source.objects.get_or_create(
domain=candidate.domain,
source_type=candidate.source_type,
defaults={
"name": _candidate_name(candidate),
"base_url": candidate.url,
"status": Source.Status.CANDIDATE,
"policy": Source.Policy.REVIEW,
"policy_reason": "Automatisch ontdekt",
"parser_key": "auto",
"strict_mode": True,
"metadata": {
"discovery": [provenance],
"discovered_at": now,
},
},
source = (
Source.objects.filter(
domain=candidate.domain,
source_type=candidate.source_type,
)
.order_by("id")
.first()
)
was_created = source is None
if source is None:
source = Source.objects.create(
domain=candidate.domain,
source_type=candidate.source_type,
name=_candidate_name(candidate),
base_url=candidate.url,
status=Source.Status.CANDIDATE,
policy=Source.Policy.REVIEW,
policy_reason="Automatisch ontdekt",
parser_key="auto",
strict_mode=True,
metadata={"discovery": [provenance], "discovered_at": now},
)
if was_created:
created += 1
continue
+41 -9
View File
@@ -13,7 +13,8 @@ from django.utils import timezone
from apps.jobs.services.normalization import normalize_extracted_job
from apps.jobs.services.pipeline import persist_draft
from apps.sources.adapters.email_alert import EmailAlertAdapter
from apps.sources.models import EmailMessageRecord, RawDocument, Source
from apps.sources.models import EmailMessageRecord, MailboxConnection, RawDocument, Source
from apps.sources.platforms import platform_label
def message_identity(raw_message: bytes) -> str:
@@ -24,22 +25,32 @@ def message_identity(raw_message: bytes) -> str:
@transaction.atomic
def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageRecord:
def ingest_email(
raw_message: bytes,
*,
mailbox: str = "INBOX",
mailbox_connection: MailboxConnection | None = None,
platform: str | None = None,
) -> EmailMessageRecord:
parsed = BytesParser(policy=policy.default).parsebytes(raw_message)
message_id = message_identity(raw_message)
existing = EmailMessageRecord.objects.filter(message_id=message_id).first()
existing = EmailMessageRecord.objects.filter(
message_id=message_id, mailbox_connection=mailbox_connection
).first()
if existing:
return existing
source_domain = f"{platform}.mailbox.local" if platform else "mailbox.local"
source_name = f"{platform_label(platform)} vacaturemailbox" if platform else "Vacaturemailbox"
source, _ = Source.objects.get_or_create(
domain="mailbox.local",
domain=source_domain,
source_type=Source.Type.EMAIL,
defaults={
"name": "Vacaturemailbox",
"name": source_name,
"status": Source.Status.ACTIVE,
"policy": Source.Policy.ALLOW,
"parser_key": "email-alert",
"crawl_interval_minutes": 10,
"crawl_interval_minutes": 1440,
},
)
content_hash = hashlib.sha256(raw_message).hexdigest()
@@ -52,7 +63,11 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
body_text=raw_message.decode("utf-8", errors="replace"),
byte_length=len(raw_message),
retain_until=retain_until,
metadata={"mailbox": mailbox},
metadata={
"mailbox": mailbox,
"mailbox_connection_id": mailbox_connection.pk if mailbox_connection else None,
"alert_provider": platform or "unknown",
},
)
received_at = None
if parsed.get("date"):
@@ -64,12 +79,28 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
received_at = None
adapter = EmailAlertAdapter()
result = adapter.extract_message(raw_message)
result = adapter.extract_message(raw_message, expected_provider=platform)
detected_provider = next(
(
str(extracted.raw.get("alert_provider"))
for extracted in result.jobs
if extracted.raw.get("alert_provider")
),
"unknown",
)
alert_provider = platform or detected_provider
document.parser_key = result.parser_key
document.parser_version = result.parser_version
document.extraction_confidence = result.confidence
document.metadata = {**document.metadata, "alert_provider": alert_provider}
document.save(
update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"]
update_fields=[
"parser_key",
"parser_version",
"extraction_confidence",
"metadata",
"updated_at",
]
)
links: list[str] = []
errors: list[str] = []
@@ -87,6 +118,7 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
except Exception as exc:
errors.append(f"{exc.__class__.__name__}: {exc}")
return EmailMessageRecord.objects.create(
mailbox_connection=mailbox_connection,
message_id=message_id,
mailbox=mailbox,
sender=str(parsed.get("from") or "")[:500],
+2
View File
@@ -14,6 +14,7 @@ from django.conf import settings
from apps.sources.models import Source
from .policy import assess_url
from .tls import trusted_tls_context
from .url_security import ValidatedUrl, validate_public_url
ALLOWED_CONTENT_TYPES = (
@@ -123,6 +124,7 @@ def fetch_url(
timeout=httpx.Timeout(settings.FETCHER_TIMEOUT_SECONDS),
follow_redirects=False,
headers=headers,
verify=trusted_tls_context(),
)
current_url = url
previous_validation: ValidatedUrl | None = None
+84
View File
@@ -0,0 +1,84 @@
from __future__ import annotations
import imaplib
from collections.abc import Callable
from contextlib import suppress
from typing import Any
from django.conf import settings
from apps.sources.models import EmailMessageRecord, MailboxConnection
from apps.sources.services.email_import import ingest_email, message_identity
from apps.sources.services.mailbox_connections import decrypt_mailbox_password
from apps.sources.services.tls import trusted_tls_context
from apps.sources.services.url_security import validate_public_url
ImapClientFactory = Callable[..., Any]
HostValidator = Callable[[str], object]
def poll_imap_mailbox(
connection: MailboxConnection,
*,
client_factory: ImapClientFactory | None = None,
host_validator: HostValidator = validate_public_url,
) -> dict[str, int | str]:
"""Import one bounded mailbox batch without letting one bad message abort the batch."""
if not connection.enabled:
return {"status": "disabled", "imported": 0}
if client_factory is None:
client_factory = imaplib.IMAP4_SSL
host_validator(f"https://{connection.imap_host}")
password = decrypt_mailbox_password(connection)
client_options = {"timeout": settings.IMAP_CONNECT_TIMEOUT_SECONDS}
if client_factory is imaplib.IMAP4_SSL:
client_options["ssl_context"] = trusted_tls_context()
client = client_factory(connection.imap_host, connection.port, **client_options)
counters = {
"imported": 0,
"duplicates": 0,
"skipped_oversized": 0,
"failed": 0,
}
try:
client.login(connection.username, password)
status, _ = client.select(connection.mailbox, readonly=True)
if status != "OK":
raise RuntimeError("IMAP-mailbox kon niet worden geopend")
status, data = client.uid("search", None, "ALL")
if status != "OK":
raise RuntimeError("IMAP-zoekopdracht mislukt")
uids = (data[0] or b"").split()[-settings.IMAP_MAX_MESSAGES_PER_POLL :]
for uid in uids:
try:
status, payload = client.uid("fetch", uid, "(RFC822)")
if status != "OK" or not payload:
counters["failed"] += 1
continue
raw = next((part[1] for part in payload if isinstance(part, tuple)), None)
if not isinstance(raw, bytes):
counters["failed"] += 1
continue
if len(raw) > settings.IMAP_MAX_MESSAGE_BYTES:
counters["skipped_oversized"] += 1
continue
identity = message_identity(raw)
if EmailMessageRecord.objects.filter(
mailbox_connection=connection, message_id=identity
).exists():
counters["duplicates"] += 1
continue
ingest_email(
raw,
mailbox=connection.mailbox,
mailbox_connection=connection,
platform=connection.platform,
)
counters["imported"] += 1
except Exception:
counters["failed"] += 1
return {"status": "ok", **counters}
finally:
with suppress(Exception):
client.logout()
@@ -0,0 +1,148 @@
from __future__ import annotations
from datetime import timedelta
from uuid import uuid4
from cryptography.fernet import Fernet, InvalidToken, MultiFernet
from django.conf import settings
from django.db import transaction
from django.utils import timezone
from apps.sources.models import MailboxConnection
class MailboxCredentialError(RuntimeError):
"""Safe credential configuration error without secret material."""
def _cipher() -> MultiFernet:
keys = settings.MAILBOX_CREDENTIAL_KEYS
if not keys:
raise MailboxCredentialError(
"Credentialopslag is niet geconfigureerd. Stel MAILBOX_CREDENTIAL_KEYS in."
)
try:
return MultiFernet([Fernet(key.encode("ascii")) for key in keys])
except (ValueError, TypeError) as exc:
raise MailboxCredentialError(
"MAILBOX_CREDENTIAL_KEYS bevat een ongeldige sleutel."
) from exc
def encrypt_mailbox_password(password: str) -> str:
if not password:
raise MailboxCredentialError("Een app-wachtwoord is verplicht.")
return _cipher().encrypt(password.encode("utf-8")).decode("ascii")
def decrypt_mailbox_password(connection: MailboxConnection) -> str:
try:
return _cipher().decrypt(connection.encrypted_password.encode("ascii")).decode("utf-8")
except (InvalidToken, ValueError, UnicodeError) as exc:
raise MailboxCredentialError(
"Het opgeslagen app-wachtwoord kan niet worden ontsleuteld. Sla het opnieuw op."
) from exc
@transaction.atomic
def rotate_mailbox_credentials() -> int:
"""Re-encrypt every mailbox password with the first configured key."""
rotated = 0
for connection in MailboxConnection.objects.select_for_update().all():
password = decrypt_mailbox_password(connection)
connection.encrypted_password = encrypt_mailbox_password(password)
connection.save(update_fields=["encrypted_password", "updated_at"])
rotated += 1
return rotated
@transaction.atomic
def save_mailbox_connection(
*,
user,
cleaned_data: dict[str, object],
instance: MailboxConnection | None = None,
) -> MailboxConnection:
connection = instance or MailboxConnection(user=user)
if connection.pk and connection.user_id != user.pk:
raise PermissionError("Mailboxkoppeling behoort niet tot deze gebruiker.")
password = str(cleaned_data.pop("password", "") or "")
for field in (
"platform",
"provider",
"custom_host",
"port",
"username",
"mailbox",
"enabled",
"poll_interval_minutes",
):
setattr(connection, field, cleaned_data[field])
if password:
connection.encrypted_password = encrypt_mailbox_password(password)
elif not connection.encrypted_password:
raise MailboxCredentialError("Een app-wachtwoord is verplicht.")
connection.next_poll_at = timezone.now()
connection.last_error_category = ""
connection.last_error_message = ""
connection.full_clean()
connection.save()
return connection
@transaction.atomic
def claim_mailbox_connection(
*, connection_id: int, worker_token: str | None = None, force: bool = False
) -> tuple[MailboxConnection, str] | None:
now = timezone.now()
connection = MailboxConnection.objects.select_for_update().get(pk=connection_id)
if not connection.enabled:
return None
if connection.lease_expires_at and connection.lease_expires_at > now:
return None
if not force and connection.next_poll_at > now:
return None
token = worker_token or uuid4().hex
connection.lease_token = token
connection.lease_expires_at = now + timedelta(minutes=5)
connection.save(update_fields=["lease_token", "lease_expires_at", "updated_at"])
return connection, token
@transaction.atomic
def finish_mailbox_poll(
*,
connection_id: int,
worker_token: str,
success: bool,
error_category: str = "",
error_message: str = "",
) -> None:
connection = MailboxConnection.objects.select_for_update().get(pk=connection_id)
if connection.lease_token != worker_token:
return
now = timezone.now()
connection.last_polled_at = now
connection.next_poll_at = now + timedelta(minutes=connection.poll_interval_minutes)
connection.lease_token = ""
connection.lease_expires_at = None
if success:
connection.last_success_at = now
connection.last_error_category = ""
connection.last_error_message = ""
else:
connection.last_error_category = error_category[:80]
connection.last_error_message = error_message[:500]
connection.save(
update_fields=[
"last_polled_at",
"next_poll_at",
"lease_token",
"lease_expires_at",
"last_success_at",
"last_error_category",
"last_error_message",
"updated_at",
]
)
+4
View File
@@ -19,6 +19,10 @@ DEFAULT_DENYLIST = {
"stepstone.be",
"jobat.be",
"vdab.be",
"ictjob.be",
"careerjet.be",
"randstad.be",
"roberthalf.com",
}
+27 -9
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
from collections.abc import Iterable
from dataclasses import dataclass
from datetime import datetime, timedelta
from urllib.parse import urlsplit, urlunsplit
from urllib.parse import urljoin, urlsplit, urlunsplit
import httpx
from django.conf import settings
@@ -11,6 +11,7 @@ from django.utils import timezone
from apps.sources.models import Source, SourceRobotsCache
from .tls import trusted_tls_context
from .url_security import UnsafeUrlError, validate_public_url
ALLOW = "allow"
@@ -123,20 +124,37 @@ def _max_bytes() -> int:
def _fetch_robots(origin: str, *, client: httpx.Client | None = None) -> tuple[str, int, str, str]:
robots_url = _robots_url(origin)
validate_public_url(
robots_url,
allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
)
own_client = client is None
http_client = client or httpx.Client(
timeout=httpx.Timeout(getattr(settings, "ROBOTS_FETCH_TIMEOUT_SECONDS", 5)),
follow_redirects=False,
verify=trusted_tls_context(),
)
try:
response = http_client.get(
robots_url,
headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
)
current_url = robots_url
for _ in range(getattr(settings, "FETCHER_MAX_REDIRECTS", 5) + 1):
validate_public_url(
current_url,
allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
)
response = http_client.get(
current_url,
headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
)
if response.status_code not in {301, 302, 303, 307, 308}:
break
location = response.headers.get("location")
if not location:
raise httpx.HTTPStatusError(
"Robotsredirect zonder Location-header",
request=response.request,
response=response,
)
current_url = urljoin(current_url, location)
else:
raise httpx.TooManyRedirects(
"Te veel redirects bij robotscontrole", request=response.request
)
finally:
if own_client:
http_client.close()
+16
View File
@@ -0,0 +1,16 @@
from __future__ import annotations
import ssl
from functools import lru_cache
from pathlib import Path
import certifi
@lru_cache(maxsize=1)
def trusted_tls_context() -> ssl.SSLContext:
"""Build a strict TLS context from the pinned application CA bundle."""
bundle = Path(certifi.where()).resolve(strict=True)
if not bundle.is_file():
raise RuntimeError("De vertrouwde CA-bundel ontbreekt.")
return ssl.create_default_context(cafile=str(bundle))
+63 -37
View File
@@ -1,7 +1,5 @@
from __future__ import annotations
import imaplib
from contextlib import suppress
from datetime import timedelta
from uuid import uuid4
@@ -11,8 +9,7 @@ from django.db.models import Q
from django.utils import timezone
from apps.jobs.services.pipeline import process_raw_document
from apps.sources.models import EmailMessageRecord, RawDocument, Source, SourceRun
from apps.sources.services.email_import import ingest_email, message_identity
from apps.sources.models import MailboxConnection, RawDocument, Source, SourceRun
from apps.sources.services.fetcher import (
FetchError,
FetchTimeoutError,
@@ -25,6 +22,12 @@ from apps.sources.services.health import (
evaluate_source_health,
start_health_canary,
)
from apps.sources.services.imap_import import poll_imap_mailbox
from apps.sources.services.mailbox_connections import (
MailboxCredentialError,
claim_mailbox_connection,
finish_mailbox_poll,
)
from apps.sources.services.policy import assess_url
from apps.sources.services.scheduling import (
acquire_source_lease,
@@ -192,40 +195,63 @@ def schedule_due_sources() -> dict[str, int]:
return {"scheduled": len(ids)}
@shared_task(name="apps.sources.tasks.poll_imap_inbox")
def poll_imap_inbox() -> dict[str, int | str]:
if not settings.IMAP_ENABLED:
return {"status": "disabled", "imported": 0}
if not settings.IMAP_HOST or not settings.IMAP_USER or not settings.IMAP_PASSWORD:
return {"status": "not_configured", "imported": 0}
client_cls = imaplib.IMAP4_SSL if settings.IMAP_USE_SSL else imaplib.IMAP4
client = client_cls(settings.IMAP_HOST, settings.IMAP_PORT)
imported = 0
@shared_task(name="apps.sources.tasks.schedule_mailbox_polls")
def schedule_mailbox_polls() -> dict[str, int]:
now = timezone.now()
due_ids = list(
MailboxConnection.objects.filter(enabled=True, next_poll_at__lte=now)
.filter(Q(lease_expires_at__isnull=True) | Q(lease_expires_at__lte=now))
.values_list("id", flat=True)[:100]
)
for connection_id in due_ids:
poll_mailbox.delay(connection_id)
return {"scheduled": len(due_ids)}
@shared_task(bind=True, name="apps.sources.tasks.poll_mailbox")
def poll_mailbox(self, connection_id: int, force: bool = False) -> dict[str, int | str]:
worker_token = str(self.request.id or uuid4())
claimed = claim_mailbox_connection(
connection_id=connection_id, worker_token=worker_token, force=force
)
if claimed is None:
return {"status": "skipped", "imported": 0}
connection, token = claimed
try:
client.login(settings.IMAP_USER, settings.IMAP_PASSWORD)
status, _ = client.select(settings.IMAP_MAILBOX, readonly=not settings.IMAP_MARK_SEEN)
if status != "OK":
raise RuntimeError("IMAP-mailbox kon niet worden geopend")
status, data = client.uid("search", None, "ALL")
if status != "OK":
raise RuntimeError("IMAP-zoekopdracht mislukt")
uids = (data[0] or b"").split()[-200:]
for uid in uids:
status, payload = client.uid("fetch", uid, "(RFC822)")
if status != "OK" or not payload:
continue
raw = next((part[1] for part in payload if isinstance(part, tuple)), None)
if not raw:
continue
message_id = message_identity(raw)
was_known = EmailMessageRecord.objects.filter(message_id=message_id).exists()
ingest_email(raw, mailbox=settings.IMAP_MAILBOX)
if not was_known:
imported += 1
return {"status": "ok", "imported": imported}
finally:
with suppress(Exception):
client.logout()
result = poll_imap_mailbox(connection)
finish_mailbox_poll(
connection_id=connection.pk,
worker_token=token,
success=result["status"] == "ok",
)
return result
except MailboxCredentialError as exc:
finish_mailbox_poll(
connection_id=connection.pk,
worker_token=token,
success=False,
error_category="credentials",
error_message=str(exc),
)
return {"status": "failed", "error_category": "credentials", "imported": 0}
except Exception as exc:
error_category = "imap"
if exc.__class__.__name__.lower().startswith("unsafe"):
error_category = "host_policy"
finish_mailbox_poll(
connection_id=connection.pk,
worker_token=token,
success=False,
error_category=error_category,
error_message="IMAP-verbinding of authenticatie mislukt.",
)
return {"status": "failed", "error_category": error_category, "imported": 0}
@shared_task(name="apps.sources.tasks.poll_imap_inbox")
def poll_imap_inbox() -> dict[str, int]:
"""Compatibility alias for old deployments; it now schedules due per-platform mailboxes."""
return schedule_mailbox_polls()
@shared_task(name="apps.sources.tasks.cleanup_raw_documents")
+14 -1
View File
@@ -1,10 +1,23 @@
from django.urls import path
from .views import SourceListView, bulk_candidate_action, manual_import_view, retry_source
from .views import (
SourceListView,
bulk_candidate_action,
edit_mailbox,
manual_import_view,
retry_source,
save_mailbox,
sync_platform_alerts,
toggle_mailbox,
)
urlpatterns = [
path("", SourceListView.as_view(), name="list"),
path("manual-import/", manual_import_view, name="manual_import"),
path("mailboxes/add/", save_mailbox, name="mailbox_add"),
path("mailboxes/<int:pk>/edit/", edit_mailbox, name="mailbox_edit"),
path("mailboxes/<int:pk>/sync/", sync_platform_alerts, name="sync_platform_alerts"),
path("mailboxes/<int:pk>/toggle/", toggle_mailbox, name="mailbox_toggle"),
path("bulk/", bulk_candidate_action, name="bulk"),
path("<int:pk>/retry/", retry_source, name="retry"),
]
+133 -6
View File
@@ -4,39 +4,88 @@ from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin
from django.db.models import Count, Q
from django.http import HttpRequest
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.utils import timezone
from django.views.decorators.cache import never_cache
from django.views.decorators.http import require_POST
from django.views.generic import ListView
from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
from apps.profiles.models import SearchProfile
from .forms import ManualImportForm
from .models import Source, SourcePolicyReview
from .forms import MailboxConnectionForm, ManualImportForm
from .models import MailboxConnection, Source, SourcePolicyReview
from .platforms import PLATFORM_ALERTS
from .services.mailbox_connections import MailboxCredentialError, save_mailbox_connection
from .services.manual_import import ManualImportError, ManualImportSummary, import_manual_source
from .services.policy import create_policy_review
from .tasks import fetch_source
from .tasks import fetch_source, poll_mailbox
def _source_list_queryset():
return Source.objects.prefetch_related("policy_reviews").all().order_by("name")
return (
Source.objects.prefetch_related("policy_reviews")
.exclude(domain="jobs.example.org", status=Source.Status.DISABLED)
.filter(
Q(metadata__hidden_from_source_list__isnull=True)
| Q(metadata__hidden_from_source_list=False)
)
.filter(Q(metadata__watchlist__isnull=True) | Q(metadata__watchlist=False))
.annotate(job_count=Count("job_aliases", distinct=True))
.order_by("name")
)
def _manual_import_context(request: HttpRequest, *, form: ManualImportForm, manual_result=None):
def _watchlist_queryset():
return (
Source.objects.filter(metadata__watchlist=True)
.filter(
Q(metadata__hidden_from_source_list__isnull=True)
| Q(metadata__hidden_from_source_list=False)
)
.order_by("name")
)
def _manual_import_context(
request: HttpRequest,
*,
form: ManualImportForm,
manual_result=None,
mailbox_form: MailboxConnectionForm | None = None,
editing_mailbox: MailboxConnection | None = None,
):
base_queryset = _source_list_queryset()
watchlist_queryset = _watchlist_queryset()
mailbox_connections = list(
request.user.mailbox_connections.filter(platform__in=PLATFORM_ALERTS)
)
active_search_profile = SearchProfile.objects.filter(user=request.user, is_active=True).first()
return {
"sources": base_queryset,
"active_source_count": Source.objects.filter(status=Source.Status.ACTIVE).count(),
"source_total_count": base_queryset.count(),
"review_pending_count": Source.objects.filter(policy=Source.Policy.REVIEW).count(),
"watchlist_sources": watchlist_queryset,
"watchlist_count": watchlist_queryset.count(),
"manual_import_form": form,
"manual_import_bookmarklet_endpoint": request.build_absolute_uri(
reverse("sources:manual_import")
),
"manual_import_paste_max_bytes": settings.MANUAL_IMPORT_PASTE_MAX_BYTES,
"manual_import_result": manual_result,
"platform_alert_ingress_ready": any(
connection.enabled for connection in mailbox_connections
),
"mailbox_connections": mailbox_connections,
"platform_alerts": PLATFORM_ALERTS.values(),
"mailbox_form": mailbox_form or MailboxConnectionForm(),
"editing_mailbox": editing_mailbox,
"mailbox_credential_key_ready": bool(settings.MAILBOX_CREDENTIAL_KEYS),
"active_search_profile": active_search_profile,
}
@@ -108,7 +157,7 @@ def _action_to_review(
def bulk_candidate_action(request):
action = request.POST.get("action")
source_ids = [int(raw_id) for raw_id in request.POST.getlist("source_ids") if raw_id.isdigit()]
sources = Source.objects.filter(id__in=source_ids).exclude(policy=Source.Policy.DENY)
sources = _source_list_queryset().filter(id__in=source_ids).exclude(policy=Source.Policy.DENY)
if not sources.exists():
messages.info(request, "Geen bruikbare bronregels geselecteerd.")
return redirect("sources:list")
@@ -148,6 +197,84 @@ def retry_source(request, pk: int):
return redirect("sources:list")
@login_required
@require_POST
def save_mailbox(request, pk: int | None = None):
instance = None
if pk is not None:
instance = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
form = MailboxConnectionForm(request.POST, instance=instance)
if form.is_valid():
try:
connection = save_mailbox_connection(
user=request.user,
cleaned_data=dict(form.cleaned_data),
instance=instance,
)
except MailboxCredentialError as exc:
form.add_error("password", str(exc))
else:
poll_mailbox.delay(connection.pk, force=True)
messages.success(
request,
f"{connection.get_platform_display()}-mailbox opgeslagen; "
"verbindingstest ingepland.",
)
return redirect("sources:list")
messages.error(request, "Controleer de mailboxinstellingen.")
return render(
request,
"sources/list.html",
_manual_import_context(
request,
form=ManualImportForm(),
mailbox_form=form,
editing_mailbox=instance,
),
)
@login_required
def edit_mailbox(request, pk: int):
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
if request.method == "POST":
return save_mailbox(request, pk=pk)
return render(
request,
"sources/list.html",
_manual_import_context(
request,
form=ManualImportForm(),
mailbox_form=MailboxConnectionForm(instance=connection),
editing_mailbox=connection,
),
)
@login_required
@require_POST
def sync_platform_alerts(request, pk: int):
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
if not connection.enabled:
messages.error(request, "Activeer deze mailbox voordat je synchroniseert.")
else:
poll_mailbox.delay(connection.pk, force=True)
messages.success(request, f"{connection.get_platform_display()}-mailimport ingepland.")
return redirect("sources:list")
@login_required
@require_POST
def toggle_mailbox(request, pk: int):
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
connection.enabled = not connection.enabled
connection.next_poll_at = timezone.now()
connection.save(update_fields=["enabled", "next_poll_at", "updated_at"])
state = "geactiveerd" if connection.enabled else "gepauzeerd"
messages.success(request, f"{connection.get_platform_display()}-mailbox {state}.")
return redirect("sources:list")
@login_required
@never_cache
def manual_import_view(request):