feat: release regional radar and mailbox integrations
deploy / deploy (push) Canceled after 0s

This commit is contained in:
Jens
2026-07-22 05:12:07 +02:00
parent 598d3ec18a
commit 551d0f46c2
131 changed files with 6209 additions and 336 deletions
+11 -2
View File
@@ -4,9 +4,18 @@ from typing import Any
from django.http import HttpRequest
from apps.profiles.models import SearchProfile
def navigation_context(request: HttpRequest) -> dict[str, Any]:
return {
context = {
"app_name": "VacatureRadar",
"app_version": "0.2.1",
"app_version": "0.2.2",
}
if request.user.is_authenticated:
context["navigation_profile"] = (
SearchProfile.objects.filter(user=request.user, is_active=True)
.only("id", "name")
.first()
)
return context
+7 -1
View File
@@ -102,6 +102,10 @@ class TodayView(LoginRequiredMixin, TemplateView):
context.update(
{
"score_cards": cards,
"priority_applications": Application.objects.filter(user=self.request.user)
.exclude(status__in=[Application.Status.REJECTED, Application.Status.WITHDRAWN])
.select_related("job", "job__employer")
.order_by("follow_up_date", "-updated_at")[:3],
"active_jobs": JobPosting.objects.filter(status=JobPosting.Status.ACTIVE).count(),
"new_today": JobPosting.objects.filter(
first_seen__date=timezone.localdate()
@@ -109,7 +113,9 @@ class TodayView(LoginRequiredMixin, TemplateView):
"applications_open": Application.objects.exclude(
status__in=[Application.Status.REJECTED, Application.Status.WITHDRAWN]
).count(),
"source_counts": Source.objects.aggregate(
"source_counts": Source.objects.exclude(
domain="jobs.example.org", status=Source.Status.DISABLED
).aggregate(
total=Count("id"),
unhealthy=Count("id", filter=~Q(status=Source.Status.ACTIVE)),
),
+21 -1
View File
@@ -8,6 +8,7 @@ from django.utils import timezone
from apps.jobs.models import Employer, FieldProvenance, JobPosting, JobSourceAlias, JobVersion
from apps.profiles.models import SearchProfile
from apps.sources.adapters.base import FieldEvidence
from apps.sources.adapters.registry import registry
from apps.sources.models import RawDocument, Source
from apps.sources.services.policy import is_denied_domain
@@ -89,7 +90,10 @@ def _source_is_direct(source: Source | None, draft: CanonicalJobDraft) -> bool:
host = (urlsplit(draft.canonical_url).hostname or "").lower()
if is_denied_domain(host):
return False
return bool(source and source.source_type == Source.Type.EMPLOYER)
if not source:
return False
metadata = source.metadata if isinstance(source.metadata, dict) else {}
return source.source_type == Source.Type.EMPLOYER or metadata.get("direct_employer") is True
def _alias_payload(
@@ -314,6 +318,22 @@ def process_raw_document(document: RawDocument) -> dict[str, int | str | list[st
)
created = updated = duplicates = 0
for extracted in result.jobs:
source_metadata = (
document.source.metadata
if document.source and isinstance(document.source.metadata, dict)
else {}
)
configured_employer = str(source_metadata.get("employer_name") or "").strip()
if configured_employer:
extracted.employer_name = configured_employer
extracted.evidence.append(
FieldEvidence(
"employer_name",
"reviewed-source-config",
0.99,
configured_employer,
)
)
draft = normalize_extracted_job(extracted)
_, decision, was_created = persist_draft(
draft,
+19 -1
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin
from django.db.models import Q
from django.db.models import CharField, DecimalField, OuterRef, Q, Subquery
from django.http import Http404, HttpResponse
from django.shortcuts import get_object_or_404, redirect
from django.urls import reverse
@@ -31,6 +31,24 @@ class JobListView(LoginRequiredMixin, ListView):
def get_queryset(self):
queryset = JobPosting.objects.select_related("employer").all()
profile = SearchProfile.objects.filter(user=self.request.user, is_active=True).first()
if profile:
latest_score = ScoreRun.objects.filter(job=OuterRef("pk"), profile=profile).order_by(
"-created_at"
)
queryset = queryset.annotate(
match_score=Subquery(
latest_score.values("score")[:1],
output_field=DecimalField(max_digits=5, decimal_places=2),
),
match_confidence=Subquery(
latest_score.values("confidence")[:1],
output_field=DecimalField(max_digits=4, decimal_places=3),
),
match_recommendation=Subquery(
latest_score.values("recommendation")[:1], output_field=CharField()
),
)
query = self.request.GET.get("q", "").strip()
status = self.request.GET.get("status", "active").strip()
workplace = self.request.GET.get("workplace", "").strip()
+194 -36
View File
@@ -1,30 +1,141 @@
from __future__ import annotations
import re
from django import forms
from .models import SearchProfile
TITLE_CHOICES = (
("system engineer", "System engineer"),
("infrastructure engineer", "Infrastructure engineer"),
("cloud engineer", "Cloud engineer"),
("devops engineer", "DevOps engineer"),
("network engineer", "Network engineer"),
("security engineer", "Security engineer"),
("workplace engineer", "Workplace engineer"),
("support engineer", "Support engineer"),
("software engineer", "Software engineer"),
("data engineer", "Data engineer"),
("solution architect", "Solution architect"),
("project manager", "Projectmanager"),
)
EXCLUDED_TITLE_CHOICES = (
("sales", "Sales"),
("recruiter", "Recruiter"),
("account manager", "Accountmanager"),
("callcenter", "Callcenter"),
("stage", "Stage"),
("student", "Studentenjob"),
)
SKILL_CHOICES = (
("azure", "Microsoft Azure"),
("aws", "AWS"),
("microsoft 365", "Microsoft 365"),
("active directory", "Active Directory"),
("entra id", "Entra ID"),
("intune", "Microsoft Intune"),
("linux", "Linux"),
("windows server", "Windows Server"),
("networking", "Netwerken"),
("security", "Security"),
("python", "Python"),
("powershell", "PowerShell"),
("terraform", "Terraform"),
("docker", "Docker"),
("kubernetes", "Kubernetes"),
("ci/cd", "CI/CD"),
)
EXCLUDED_SKILL_CHOICES = (
("cold calling", "Cold calling"),
("door to door", "Deur-aan-deurverkoop"),
("commission only", "Alleen commissieloon"),
("night shift", "Nachtwerk"),
)
EMPLOYMENT_CHOICES = (
("full_time", "Voltijds"),
("part_time", "Deeltijds"),
("permanent", "Vast contract"),
("fixed_term", "Tijdelijk contract"),
("freelance", "Freelance"),
("internship", "Stage"),
)
WORKPLACE_CHOICES = (
("hybrid", "Hybride"),
("remote", "Volledig op afstand"),
("on_site", "Op locatie"),
)
REGION_CHOICES = (
("Antwerpen", "Antwerpen"),
("Brussels Hoofdstedelijk Gewest", "Brussel"),
("Henegouwen", "Henegouwen"),
("Limburg", "Limburg"),
("Luik", "Luik"),
("Luxemburg", "Luxemburg"),
("Namen", "Namen"),
("Oost-Vlaanderen", "Oost-Vlaanderen"),
("Vlaams-Brabant", "Vlaams-Brabant"),
("Waals-Brabant", "Waals-Brabant"),
("West-Vlaanderen", "West-Vlaanderen"),
)
class SearchProfileForm(forms.ModelForm):
desired_titles_text = forms.CharField(
desired_titles = forms.MultipleChoiceField(
label="Gewenste functietitels",
required=False,
widget=forms.Textarea(attrs={"rows": 4, "placeholder": "Eén titel per regel"}),
choices=TITLE_CHOICES,
widget=forms.CheckboxSelectMultiple,
help_text="Selecteer alle rollen die bij je zoekrichting passen.",
)
excluded_titles_text = forms.CharField(
label="Uitgesloten titelwoorden",
excluded_titles = forms.MultipleChoiceField(
label="Uitgesloten functietitels",
required=False,
widget=forms.Textarea(attrs={"rows": 3, "placeholder": "Eén term per regel"}),
choices=EXCLUDED_TITLE_CHOICES,
widget=forms.CheckboxSelectMultiple,
help_text="Vacatures met deze titelwoorden worden hard uitgesloten.",
)
desired_skills_text = forms.CharField(
desired_skills = forms.MultipleChoiceField(
label="Gewenste skills",
required=False,
widget=forms.Textarea(attrs={"rows": 4, "placeholder": "Eén skill per regel"}),
choices=SKILL_CHOICES,
widget=forms.CheckboxSelectMultiple,
)
excluded_skills_text = forms.CharField(
label="Uitgesloten skills",
excluded_skills = forms.MultipleChoiceField(
label="Uitgesloten kenmerken",
required=False,
widget=forms.Textarea(attrs={"rows": 3, "placeholder": "Eén skill per regel"}),
choices=EXCLUDED_SKILL_CHOICES,
widget=forms.CheckboxSelectMultiple,
)
allowed_employment_types = forms.MultipleChoiceField(
label="Toegestane contractvormen",
required=False,
choices=EMPLOYMENT_CHOICES,
widget=forms.CheckboxSelectMultiple,
)
preferred_workplace = forms.MultipleChoiceField(
label="Voorkeurswerkmodel",
required=False,
choices=WORKPLACE_CHOICES,
widget=forms.CheckboxSelectMultiple,
)
preferred_regions = forms.MultipleChoiceField(
label="Voorkeursregio's",
required=False,
choices=REGION_CHOICES,
widget=forms.CheckboxSelectMultiple,
)
excluded_regions = forms.MultipleChoiceField(
label="Uitgesloten regio's",
required=False,
choices=REGION_CHOICES,
widget=forms.CheckboxSelectMultiple,
)
class Meta:
@@ -33,10 +144,11 @@ class SearchProfileForm(forms.ModelForm):
"name",
"is_active",
"home_postal_code",
"home_municipality",
"home_latitude",
"home_longitude",
"max_distance_km",
"desired_titles",
"excluded_titles",
"desired_skills",
"excluded_skills",
"allowed_employment_types",
"preferred_workplace",
"preferred_regions",
@@ -46,35 +158,81 @@ class SearchProfileForm(forms.ModelForm):
"digest_time",
"learning_enabled",
]
labels = {
"name": "Naam",
"is_active": "Actief profiel",
"home_postal_code": "Thuispostcode",
"max_distance_km": "Maximale afstand (km)",
"recommendation_threshold": "Aanbevelingsdrempel",
"top_match_threshold": "Topmatchdrempel",
"digest_time": "Tijdstip dagelijkse samenvatting",
"learning_enabled": "Gecontroleerd leren",
}
help_texts = {
"home_postal_code": (
"Vier cijfers volstaan. Gemeente en coördinaten worden veilig afgeleid wanneer "
"lokale geodata beschikbaar is."
),
"learning_enabled": "Past alleen zachte voorkeuren aan; harde regels blijven vast.",
}
widgets = {
"allowed_employment_types": forms.TextInput(
attrs={"placeholder": '["full_time", "part_time"]'}
"home_postal_code": forms.TextInput(
attrs={"inputmode": "numeric", "pattern": "[0-9]{4}", "maxlength": 4}
),
"preferred_workplace": forms.TextInput(attrs={"placeholder": '["hybrid", "on_site"]'}),
"preferred_regions": forms.TextInput(attrs={"placeholder": '["Limburg"]'}),
"excluded_regions": forms.TextInput(attrs={"placeholder": '["Brussel"]'}),
"digest_time": forms.TimeInput(attrs={"type": "time"}),
}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.instance and self.instance.pk:
self.fields["desired_titles_text"].initial = "\n".join(self.instance.desired_titles)
self.fields["excluded_titles_text"].initial = "\n".join(self.instance.excluded_titles)
self.fields["desired_skills_text"].initial = "\n".join(self.instance.desired_skills)
self.fields["excluded_skills_text"].initial = "\n".join(self.instance.excluded_skills)
self.fields["home_postal_code"].widget.attrs["maxlength"] = 4
if not self.instance or not self.instance.pk:
return
for field_name in (
"desired_titles",
"excluded_titles",
"desired_skills",
"excluded_skills",
"allowed_employment_types",
"preferred_workplace",
"preferred_regions",
"excluded_regions",
):
self._include_existing_choices(field_name)
@staticmethod
def _lines(value: str) -> list[str]:
return [line.strip() for line in value.splitlines() if line.strip()]
def _include_existing_choices(self, field_name: str) -> None:
field = self.fields[field_name]
stored = getattr(self.instance, field_name, [])
existing_by_casefold = {
str(value).casefold(): str(value) for value, _label in field.choices
}
extra = [
(str(value), str(value))
for value in stored
if str(value).casefold() not in existing_by_casefold
]
field.choices = [*field.choices, *extra]
field.initial = [
existing_by_casefold.get(str(value).casefold(), str(value)) for value in stored
]
def save(self, commit=True):
instance = super().save(commit=False)
instance.desired_titles = self._lines(self.cleaned_data["desired_titles_text"])
instance.excluded_titles = self._lines(self.cleaned_data["excluded_titles_text"])
instance.desired_skills = self._lines(self.cleaned_data["desired_skills_text"])
instance.excluded_skills = self._lines(self.cleaned_data["excluded_skills_text"])
if commit:
instance.save()
self.save_m2m()
return instance
def clean_home_postal_code(self) -> str:
value = str(self.cleaned_data.get("home_postal_code") or "").strip()
if value and not re.fullmatch(r"\d{4}", value):
raise forms.ValidationError("Gebruik een Belgische postcode van vier cijfers.")
return value
def clean(self):
cleaned = super().clean()
overlap_checks = (
("desired_titles", "excluded_titles", "functietitels"),
("desired_skills", "excluded_skills", "skills"),
("preferred_regions", "excluded_regions", "regio's"),
)
for desired_field, excluded_field, label in overlap_checks:
overlap = set(cleaned.get(desired_field) or []) & set(cleaned.get(excluded_field) or [])
if overlap:
self.add_error(
excluded_field,
f"Dezelfde {label} kunnen niet tegelijk gewenst en uitgesloten zijn.",
)
return cleaned
+24
View File
@@ -2,9 +2,33 @@ from __future__ import annotations
from django.db import transaction
from apps.jobs.services.geocoding import LocationMatchResult, resolve_cached_location
from .models import ProfileRevision, SearchProfile
@transaction.atomic
def resolve_profile_home_location(profile: SearchProfile) -> LocationMatchResult:
"""Resolve a user-entered postal code without exposing coordinate fields in the UI."""
query = profile.home_postal_code.strip()
result = (
resolve_cached_location(query) if query else LocationMatchResult(query=query, location=None)
)
location = result.location
profile.home_municipality = location.municipality or "" if location else ""
profile.home_latitude = location.point.latitude if location and location.point else None
profile.home_longitude = location.point.longitude if location and location.point else None
profile.save(
update_fields=[
"home_municipality",
"home_latitude",
"home_longitude",
"updated_at",
]
)
return result
@transaction.atomic
def save_profile_revision(profile: SearchProfile, *, reason: str) -> ProfileRevision:
latest = profile.revisions.order_by("-version").first()
+16 -2
View File
@@ -8,7 +8,7 @@ from django.views.generic import ListView, UpdateView
from .forms import SearchProfileForm
from .models import SearchProfile
from .services import save_profile_revision
from .services import resolve_profile_home_location, save_profile_revision
class ProfileListView(LoginRequiredMixin, ListView):
@@ -31,10 +31,24 @@ class ProfileUpdateView(LoginRequiredMixin, UpdateView):
def form_valid(self, form):
response = super().form_valid(form)
location_result = resolve_profile_home_location(self.object)
if self.object.is_active:
self.object.activate()
save_profile_revision(self.object, reason="user_edit")
messages.success(self.request, "Zoekprofiel opgeslagen.")
if self.object.home_postal_code and location_result.ambiguous:
messages.warning(
self.request,
"Zoekprofiel opgeslagen. Deze postcode omvat meerdere gemeenten; afstand blijft "
"onbekend tot de lokale geodata eenduidig is.",
)
elif self.object.home_postal_code and location_result.location is None:
messages.warning(
self.request,
"Zoekprofiel opgeslagen. De postcode is bewaard; lokale geodata ontbreekt nog "
"voor een afstandsberekening.",
)
else:
messages.success(self.request, "Zoekprofiel opgeslagen.")
return response
+6
View File
@@ -8,6 +8,9 @@ from .ats import (
from .base import ExtractedJob, ExtractionResult
from .generic_html import GenericHtmlAdapter
from .jsonld import JsonLdJobPostingAdapter
from .kempen import KempenEmployerAdapter
from .mol_region import MolRegionEmployerAdapter
from .regional import LocalEmployerListingAdapter
from .rss import RssAdapter
__all__ = [
@@ -16,7 +19,10 @@ __all__ = [
"GenericHtmlAdapter",
"GreenhouseAdapter",
"JsonLdJobPostingAdapter",
"KempenEmployerAdapter",
"LeverAdapter",
"LocalEmployerListingAdapter",
"MolRegionEmployerAdapter",
"RecruiteeAdapter",
"RssAdapter",
"SmartRecruitersAdapter",
+45 -6
View File
@@ -169,16 +169,20 @@ class _AtsAdapter(ABC):
def _extract_location(self, record: dict[str, object]) -> tuple[str, str, str, str]:
location_raw = _first_text(
record,
"location",
"location.city",
"location.name",
"location.address",
"location.raw",
"categories.location",
"city",
"cityName",
"place",
"office",
"location.address",
"address",
"data.location",
"officeLocation",
"locationName",
"location",
)
if not location_raw:
location_raw = _first_text(record, "data.city", "data.location")
@@ -223,6 +227,13 @@ class _AtsAdapter(ABC):
"remote_type",
"job_type",
).lower()
if not workplace_raw:
if record.get("hybrid") is True:
return "hybrid"
if record.get("remote") is True:
return "remote"
if record.get("on_site") is True:
return "on_site"
if "remote" in workplace_raw:
return "remote" if "hybrid" not in workplace_raw else "hybrid"
if "hybrid" in workplace_raw:
@@ -256,18 +267,21 @@ class _AtsAdapter(ABC):
"url",
"jobUrl",
"job_url",
"hostedUrl",
"applyUrl",
"link",
"absoluteUrl",
"careers_url",
"data.url",
)
employer_name = _first_text(
record,
"company_name",
"company.name",
"company",
"employer",
"organization",
"organizationName",
"company.name",
"department",
"hiringOrganization",
)
@@ -290,6 +304,7 @@ class _AtsAdapter(ABC):
"published_at",
"created_at",
"jobCreated",
"releasedDate",
)
valid_through = _first_text(
record,
@@ -308,6 +323,9 @@ class _AtsAdapter(ABC):
"jobType",
"type",
"data.employmentType",
"employment_type_code",
"categories.commitment",
"typeOfEmployment.label",
)
)
workplace_type = self._extract_workplace(record)
@@ -428,13 +446,20 @@ class GreenhouseAdapter(_AtsAdapter):
class LeverAdapter(_AtsAdapter):
parser_key = "ats-lever"
source_hosts = ("jobs.lever.co",)
source_hosts = (
"jobs.lever.co",
"jobs.eu.lever.co",
"api.lever.co",
"api.eu.lever.co",
)
support_markers = ("lever", "requisition", "posting")
listing_paths = (("data",), ("jobs",), ("results",))
detail_paths = (("data",), ("job",), ("position",), ("result",))
closed_statuses = CLOSED_STATUSES | {"archived", "deleted"}
def _extract_records(self, payload):
if isinstance(payload, list):
return [item for item in payload if isinstance(item, dict)]
for path in self.listing_paths:
value = payload
for key in path:
@@ -466,7 +491,7 @@ class RecruiteeAdapter(_AtsAdapter):
parser_key = "ats-recruitee"
source_hosts = ("recruitee.com",)
support_markers = ("recruitee", "career", "vacancy")
listing_paths = (("jobs",), ("data", "jobs"), ("vacancies",))
listing_paths = (("jobs",), ("offers",), ("data", "jobs"), ("vacancies",))
detail_paths = (("job",), ("data", "job"), ("vacancy",), ("result",))
closed_statuses = CLOSED_STATUSES | {"hidden", "paused"}
@@ -494,12 +519,26 @@ class RecruiteeAdapter(_AtsAdapter):
class SmartRecruitersAdapter(_AtsAdapter):
parser_key = "ats-smartrecruiters"
parser_version = "1.1.0"
source_hosts = ("smartrecruiters.com",)
support_markers = ("smartrecruiters", "smart recruiter")
listing_paths = (("jobs",), ("data", "jobs"), ("results",))
listing_paths = (("jobs",), ("data", "jobs"), ("results",), ("content",))
detail_paths = (("job",), ("data", "job"), ("posting",), ("result",))
closed_statuses = CLOSED_STATUSES | {"unpublished", "expired"}
def _to_job(self, record: dict[str, object], base_url: str) -> ExtractedJob:
prepared = dict(record)
if not _first_text(prepared, "url", "jobUrl", "hostedUrl", "applyUrl", "link"):
company_identifier = _first_text(prepared, "company.identifier")
posting_id = _first_text(prepared, "id", "uuid")
if company_identifier and posting_id:
prepared["url"] = (
f"https://jobs.smartrecruiters.com/{company_identifier}/{posting_id}"
)
if _find_nested(prepared, "location.remote") is True:
prepared["remote"] = True
return super()._to_job(prepared, base_url)
def _extract_records(self, payload):
for path in self.listing_paths:
value = payload
+59 -3
View File
@@ -9,17 +9,58 @@ from urllib.parse import urljoin, urlsplit
from bs4 import BeautifulSoup
from apps.sources.platforms import PLATFORM_ALERTS
from apps.sources.services.canonicalize import canonicalize_url
from .base import ExtractedJob, ExtractionResult, FieldEvidence
URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I)
SKIP_TEXT = re.compile(r"unsubscribe|afmelden|uitschrijven|privacy|view in browser", re.I)
SKIP_TEXT = re.compile(
r"unsubscribe|afmelden|uitschrijven|privacy|view in browser|bekijk online|"
r"account|aanmelden|inloggen|login|voorkeuren|preferences|voorwaarden|terms|"
r"contact|help|hulp|over ons|about us",
re.I,
)
SKIP_PATH = re.compile(
r"/(?:unsubscribe|uitschrijven|afmelden|privacy|account|login|signin|preferences|"
r"settings|terms|legal|help|contact)(?:/|$)",
re.I,
)
def _matches_domain(hostname: str, domains: tuple[str, ...]) -> bool:
return any(hostname == domain or hostname.endswith(f".{domain}") for domain in domains)
class EmailAlertAdapter:
parser_key = "email-alert"
parser_version = "1.0.0"
parser_version = "1.1.0"
@staticmethod
def _provider_for_candidates(candidates: list[tuple[str, str]]) -> str:
hostnames = {
(urlsplit(href).hostname or "").lower()
for _, href in candidates
if href.lower().startswith(("http://", "https://"))
}
for provider, alert in PLATFORM_ALERTS.items():
if any(_matches_domain(hostname, alert.domains) for hostname in hostnames):
return provider
return "other"
@staticmethod
def _is_expected_platform_link(*, label: str, href: str, provider: str) -> bool:
alert = PLATFORM_ALERTS.get(provider)
if alert is None or len(label.strip()) < 4:
return False
parsed = urlsplit(href)
hostname = (parsed.hostname or "").lower()
return (
parsed.scheme.lower() == "https"
and _matches_domain(hostname, alert.domains)
and not SKIP_TEXT.search(label)
and not SKIP_PATH.search(parsed.path)
)
@staticmethod
def _decode_parts(message: Message) -> tuple[str, str]:
@@ -41,7 +82,9 @@ class EmailAlertAdapter:
html_parts.append(str(payload))
return "\n".join(plain_parts), "\n".join(html_parts)
def extract_message(self, raw_message: bytes) -> ExtractionResult:
def extract_message(
self, raw_message: bytes, *, expected_provider: str | None = None
) -> ExtractionResult:
message = BytesParser(policy=policy.default).parsebytes(raw_message)
plain, html_body = self._decode_parts(message)
candidates: list[tuple[str, str]] = []
@@ -61,6 +104,18 @@ class EmailAlertAdapter:
continue
candidates.append(("", clean_href))
alert_provider = expected_provider or self._provider_for_candidates(candidates)
if expected_provider:
candidates = [
(label, href)
for label, href in candidates
if self._is_expected_platform_link(
label=label,
href=href,
provider=expected_provider,
)
]
jobs: list[ExtractedJob] = []
seen: set[str] = set()
subject = str(message.get("subject") or "Vacature uit e-mail").strip()
@@ -84,6 +139,7 @@ class EmailAlertAdapter:
"email_subject": subject,
"email_sender": str(message.get("from") or ""),
"target_domain": hostname,
"alert_provider": alert_provider,
},
evidence=[FieldEvidence("url", "email-anchor", 0.75, label[:240])],
)
+290
View File
@@ -0,0 +1,290 @@
from __future__ import annotations
import re
from urllib.parse import parse_qs, urljoin, urlsplit
from bs4 import BeautifulSoup, Tag
from .base import ExtractedJob, ExtractionResult, FieldEvidence
class KempenEmployerAdapter:
"""Extract public employer lists around Mol from exact reviewed routes."""
parser_key = "regional-kempen-employers"
parser_version = "1.0.0"
supported_routes = {
"ziekenhuisgeel.careersite.be": {"/nl/vacatures"},
"geel.hro.be": {"/"},
"jobs.turnhout.be": {"/"},
"jobs.renotec.be": {"/nl/alle-jobs"},
"ravago.softgarden.io": {"/en/vacancies"},
"jobs.sanofi.com": {"/en/belgium"},
"www.daf.com": {"/nl-nl/werken-bij-daf/vacatures"},
}
def _route(self, url: str) -> tuple[str, str] | None:
parts = urlsplit(url)
host = (parts.hostname or "").lower()
path = parts.path.rstrip("/") or "/"
if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
return None
return host, path
def _supports_url(self, url: str) -> bool:
return self._route(url) is not None
@staticmethod
def _same_host_url(base_url: str, href: str) -> str:
job_url = urljoin(base_url, href)
parts = urlsplit(job_url)
if parts.scheme != "https":
return ""
if (parts.hostname or "").lower() != (urlsplit(base_url).hostname or "").lower():
return ""
return job_url
@staticmethod
def _job(
*,
url: str,
title: str,
employer: str,
location: str,
postal_code: str,
description: str,
external_id: str,
evidence_source: str,
) -> ExtractedJob:
return ExtractedJob(
url=url,
title=title,
employer_name=employer,
external_id=external_id,
location_text=location,
postal_code=postal_code,
region="Antwerpen",
country="BE",
description_text=description,
raw={"regional_listing": evidence_source, "region_center": "2400 Mol"},
evidence=[
FieldEvidence("title", evidence_source, 0.94, title[:240]),
FieldEvidence("employer_name", "reviewed-source", 0.97, employer),
FieldEvidence("location_text", "kempen-location-marker", 0.9, location),
],
)
def _ziekenhuis_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select(".vacature-tegel"):
link = card.select_one("a.vacature-tegel__link[href]")
title_node = card.select_one(".vacature-tegel__titel")
if not isinstance(link, Tag) or not isinstance(title_node, Tag):
continue
job_url = self._same_host_url(url, str(link.get("href") or ""))
match = re.fullmatch(r"/nl/vacature/(\d+)/[^/]+", urlsplit(job_url).path)
title = title_node.get_text(" ", strip=True)
if not title or not match:
continue
jobs.append(
self._job(
url=job_url,
title=title,
employer="Ziekenhuis Geel",
location="Geel, Antwerpen",
postal_code="2440",
description=card.get_text(" ", strip=True),
external_id=match.group(1),
evidence_source="ziekenhuis-geel-card",
)
)
return jobs
def _stad_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select(".vacatureKader[data-href]"):
href = str(card.get("data-href") or "")
match = re.fullmatch(r"vacature\.php\?id=(\d+)", href)
title_node = card.find("h4")
title = title_node.get_text(" ", strip=True) if title_node else ""
if not title or not match:
continue
jobs.append(
self._job(
url=self._same_host_url(url, href),
title=title,
employer="Lokaal bestuur Geel",
location="Geel, Antwerpen",
postal_code="2440",
description=card.get_text(" ", strip=True),
external_id=match.group(1),
evidence_source="stad-geel-hro-card",
)
)
return jobs
def _stad_turnhout(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.find_all("a", href=True):
href = str(link.get("href") or "")
match = re.fullmatch(r"vacature\.php\?id=(\d+)", href)
title_node = link.select_one(".block-update__body__title__inner")
title = title_node.get_text(" ", strip=True) if title_node else ""
if not title or not match:
continue
jobs.append(
self._job(
url=self._same_host_url(url, href),
title=title,
employer="Stad Turnhout",
location="Turnhout, Antwerpen",
postal_code="2300",
description=link.get_text(" ", strip=True),
external_id=match.group(1),
evidence_source="stad-turnhout-hro-card",
)
)
return jobs
def _renotec(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select(".s-tile.s-card"):
card_text = card.get_text(" ", strip=True)
if "Geel" not in card_text:
continue
title_node = card.find("h3")
link = card.find("a", href=True)
title = title_node.get_text(" ", strip=True) if title_node else ""
job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
parts = urlsplit(job_url)
query = parse_qs(parts.query)
external_id = (query.get("id") or [""])[0]
if parts.path != "/nl/detail/" or not external_id.isdigit() or not title:
continue
jobs.append(
self._job(
url=job_url,
title=title,
employer="Group Renotec",
location="Geel, Antwerpen (mogelijk meerdere werfregio's)",
postal_code="2440",
description=card_text,
external_id=external_id,
evidence_source="renotec-geel-card",
)
)
return jobs
def _ravago(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select(".matchElement"):
locations = {
node.get_text(" ", strip=True) for node in card.select(".location-view-item")
}
local_places = locations.intersection({"Arendonk", "Olen"})
if not local_places:
continue
link = card.find("a", href=True)
title = link.get_text(" ", strip=True) if link else ""
job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
match = re.fullmatch(r"/job/(\d+)/[^/]+/?", urlsplit(job_url).path)
if not title or not match:
continue
place = "Arendonk" if "Arendonk" in local_places else "Olen"
jobs.append(
self._job(
url=job_url,
title=title,
employer="Ravago",
location=f"{place}, Antwerpen",
postal_code="2370" if place == "Arendonk" else "2250",
description=card.get_text(" ", strip=True),
external_id=match.group(1),
evidence_source="ravago-softgarden-card",
)
)
return jobs
def _sanofi(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.select(".job-list a[data-job-id][href]"):
location_node = link.select_one(".job-location")
title_node = link.select_one(".job-title")
location = location_node.get_text(" ", strip=True) if location_node else ""
title = title_node.get_text(" ", strip=True) if title_node else ""
external_id = str(link.get("data-job-id") or "")
job_url = self._same_host_url(url, str(link.get("href") or ""))
if location != "Geel, Belgium" or not title or not external_id.isdigit():
continue
if not re.fullmatch(r"/en/job/geel/[^/]+/\d+/\d+", urlsplit(job_url).path):
continue
jobs.append(
self._job(
url=job_url,
title=title,
employer="Sanofi",
location="Geel, Antwerpen",
postal_code="2440",
description=link.get_text(" ", strip=True),
external_id=external_id,
evidence_source="sanofi-belgium-geel-card",
)
)
return jobs
def _daf(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select("li.itemlist__item"):
location_node = card.select_one(".js-vac-metalocation")
title_link = card.select_one("a.js-vac-title[href]")
location = location_node.get_text(" ", strip=True) if location_node else ""
title = title_link.get_text(" ", strip=True) if title_link else ""
job_url = (
self._same_host_url(url, str(title_link.get("href") or "")) if title_link else ""
)
path = urlsplit(job_url).path
prefix = "/nl-nl/werken-bij-daf/vacatures/"
if location != "Westerlo" or not title or not path.startswith(prefix):
continue
slug = path.removeprefix(prefix).strip("/")
if not slug or "/" in slug:
continue
jobs.append(
self._job(
url=job_url,
title=title,
employer="DAF Trucks",
location="Westerlo, Antwerpen",
postal_code="2260",
description=card.get_text(" ", strip=True),
external_id=slug,
evidence_source="daf-westerlo-card",
)
)
return jobs
def extract(self, content: str, *, url: str) -> ExtractionResult:
route = self._route(url)
if route is None:
return ExtractionResult(
[], self.parser_key, self.parser_version, 0.0, ["Onherkende Kempen-bron"]
)
host, _ = route
extractors = {
"ziekenhuisgeel.careersite.be": self._ziekenhuis_geel,
"geel.hro.be": self._stad_geel,
"jobs.turnhout.be": self._stad_turnhout,
"jobs.renotec.be": self._renotec,
"ravago.softgarden.io": self._ravago,
"jobs.sanofi.com": self._sanofi,
"www.daf.com": self._daf,
}
jobs = extractors[host](BeautifulSoup(content, "lxml"), url)
unique_jobs = list({job.url: job for job in jobs if job.url}.values())
return ExtractionResult(
unique_jobs,
self.parser_key,
self.parser_version,
0.92 if unique_jobs else 0.0,
[] if unique_jobs else ["Geen actuele regionale vacatures gevonden"],
)
+253
View File
@@ -0,0 +1,253 @@
from __future__ import annotations
import re
from urllib.parse import parse_qs, urljoin, urlsplit
from bs4 import BeautifulSoup
from .base import ExtractedJob, ExtractionResult, FieldEvidence
class MolRegionEmployerAdapter:
"""Extract reviewed employer listings around postcode 2400 without detail fetches."""
parser_key = "regional-mol-employers"
parser_version = "1.0.0"
thomas_more_company_guid = "eab9ca13-ee10-4504-8a87-a785d0b037ef"
supported_routes = {
"www.sckcen.be": {"/nl/carriere/vacatures"},
"cipalschaubroeck.teamtailor.com": {"/jobs"},
"www.vanroey.be": {"/en/job-overview"},
"netropolix.recruitee.com": {"/"},
"jobpage.cvwarehouse.com": {"/"},
}
def _route(self, url: str) -> tuple[str, str] | None:
parts = urlsplit(url)
host = (parts.hostname or "").lower()
path = parts.path.rstrip("/") or "/"
if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
return None
if host == "jobpage.cvwarehouse.com":
query = parse_qs(parts.query)
if query.get("companyGuid") != [self.thomas_more_company_guid] or "job" in query:
return None
return host, path
def _supports_url(self, url: str) -> bool:
return self._route(url) is not None
@staticmethod
def _same_host_url(base_url: str, href: str) -> str:
job_url = urljoin(base_url, href)
if urlsplit(job_url).scheme != "https":
return ""
if (urlsplit(job_url).hostname or "").lower() != (
urlsplit(base_url).hostname or ""
).lower():
return ""
return job_url
@staticmethod
def _job(
*,
url: str,
title: str,
employer: str,
location: str,
postal_code: str,
description: str,
external_id: str,
evidence_source: str,
) -> ExtractedJob:
return ExtractedJob(
url=url,
title=title,
employer_name=employer,
external_id=external_id,
location_text=location,
postal_code=postal_code,
region="Antwerpen",
country="BE",
description_text=description,
raw={"regional_listing": evidence_source, "region_center": "2400 Mol"},
evidence=[
FieldEvidence("title", evidence_source, 0.94, title[:240]),
FieldEvidence("employer_name", "reviewed-source", 0.96, employer),
FieldEvidence("location_text", "mol-region-review", 0.82, location),
],
)
def _sck_cen(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.select("article a[href]"):
job_url = self._same_host_url(url, str(link.get("href") or ""))
path = urlsplit(job_url).path if job_url else ""
if not path.startswith("/nl/carriere/vacatures/"):
continue
title = link.get_text(" ", strip=True)
if not title:
continue
card = link.find_parent("article")
description = card.get_text(" ", strip=True) if card else title
jobs.append(
self._job(
url=job_url,
title=title,
employer="SCK CEN",
location="Mol, Antwerpen",
postal_code="2400",
description=description,
external_id=path.rstrip("/").rsplit("/", 1)[-1],
evidence_source="sck-cen-vacancy-card",
)
)
return jobs
def _cipal(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select("li"):
card_text = card.get_text(" ", strip=True)
if "Westerlo" not in card_text and "Geel" not in card_text:
continue
link = card.find("a", href=True)
if not link:
continue
title = link.get_text(" ", strip=True)
job_url = self._same_host_url(url, str(link.get("href") or ""))
path = urlsplit(job_url).path if job_url else ""
match = re.fullmatch(r"/jobs/(\d+)-[^/]+", path.rstrip("/"))
if not title or not match:
continue
location = "Geel, Antwerpen" if "Geel" in card_text else "Westerlo, Antwerpen"
jobs.append(
self._job(
url=job_url,
title=title,
employer="Cipal Schaubroeck",
location=location,
postal_code="2440" if "Geel" in card_text else "2260",
description=card_text,
external_id=match.group(1),
evidence_source="cipal-teamtailor-card",
)
)
return jobs
def _vanroey(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for heading in soup.select("h3.elementor-heading-title"):
link = heading.find("a", href=True)
if not link:
continue
title = link.get_text(" ", strip=True)
job_url = self._same_host_url(url, str(link.get("href") or ""))
path = urlsplit(job_url).path if job_url else ""
if not title or not re.fullmatch(r"/en/job/[^/]+/", path):
continue
slug = path.rstrip("/").rsplit("/", 1)[-1]
if "oost-vlaanderen" in slug:
continue
card = heading.find_parent("section") or heading.parent
description = card.get_text(" ", strip=True) if card else title
jobs.append(
self._job(
url=job_url,
title=title,
employer="VanRoey",
location="Turnhout/Geel (hybride; controleer vacaturedetail)",
postal_code="",
description=description,
external_id=slug,
evidence_source="vanroey-job-card",
)
)
return jobs
def _netropolix(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.find_all("a", href=True):
title = link.get_text(" ", strip=True)
if "Geel" not in title:
continue
job_url = self._same_host_url(url, str(link.get("href") or ""))
path = urlsplit(job_url).path if job_url else ""
if not re.fullmatch(r"/o/[^/]+", path.rstrip("/")):
continue
card = link.find_parent("div")
description = card.get_text(" ", strip=True) if card else title
jobs.append(
self._job(
url=job_url,
title=title,
employer="NTX (Netropolix)",
location="Geel, Antwerpen",
postal_code="2440",
description=description,
external_id=path.rstrip("/").rsplit("/", 1)[-1],
evidence_source="netropolix-recruitee-card",
)
)
return jobs
def _thomas_more(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.select("a.jobLink[data-item='readmore'][data-jobid][href]"):
title = link.get_text(" ", strip=True)
title_casefold = title.casefold()
if not any(place in title_casefold for place in ("geel", "turnhout", "vorselaar")):
continue
external_id = str(link.get("data-jobid") or "")
if not external_id.isdigit():
continue
job_url = self._same_host_url(url, str(link.get("href") or ""))
query = parse_qs(urlsplit(job_url).query) if job_url else {}
if query.get("companyGuid") != [self.thomas_more_company_guid] or query.get("job") != [
external_id
]:
continue
if "turnhout" in title_casefold:
location, postal_code = "Turnhout, Antwerpen", "2300"
elif "vorselaar" in title_casefold:
location, postal_code = "Vorselaar, Antwerpen", "2290"
else:
location, postal_code = "Geel, Antwerpen", "2440"
jobs.append(
self._job(
url=job_url,
title=title,
employer="Thomas More",
location=location,
postal_code=postal_code,
description=f"Publieke regionale vacature bij Thomas More: {title}.",
external_id=external_id,
evidence_source="thomas-more-cvwarehouse-card",
)
)
return jobs
def extract(self, content: str, *, url: str) -> ExtractionResult:
route = self._route(url)
if route is None:
return ExtractionResult(
[], self.parser_key, self.parser_version, 0.0, ["Onherkende Mol-regiobron"]
)
host, _ = route
soup = BeautifulSoup(content, "lxml")
extractors = {
"www.sckcen.be": self._sck_cen,
"cipalschaubroeck.teamtailor.com": self._cipal,
"www.vanroey.be": self._vanroey,
"netropolix.recruitee.com": self._netropolix,
"jobpage.cvwarehouse.com": self._thomas_more,
}
jobs = extractors[host](soup, url)
unique_jobs = list({job.url: job for job in jobs}.values())
warnings = [] if unique_jobs else ["Geen actuele vacatures rond Mol gevonden"]
return ExtractionResult(
unique_jobs,
self.parser_key,
self.parser_version,
0.9 if unique_jobs else 0.0,
warnings,
)
+312
View File
@@ -0,0 +1,312 @@
from __future__ import annotations
from urllib.parse import urljoin, urlsplit
from bs4 import BeautifulSoup
from .base import ExtractedJob, ExtractionResult, FieldEvidence
class CordaCampusAdapter:
"""Extract the public job cards intentionally published by Corda Campus."""
parser_key = "regional-corda-campus"
parser_version = "1.0.0"
source_hosts = ("cordacampus.com",)
def _supports_url(self, url: str) -> bool:
parts = urlsplit(url)
host = (parts.hostname or "").lower()
return host.endswith(self.source_hosts) and parts.path.rstrip("/") == "/jobs"
def extract(self, content: str, *, url: str) -> ExtractionResult:
if not self._supports_url(url):
return ExtractionResult(
[], self.parser_key, self.parser_version, 0.0, ["Onherkenbare regiobron"]
)
source_host = (urlsplit(url).hostname or "").lower()
soup = BeautifulSoup(content, "lxml")
jobs: list[ExtractedJob] = []
seen_urls: set[str] = set()
for card in soup.select('a.event-item[href*="/job/"]'):
job_url = urljoin(url, str(card.get("href") or ""))
job_host = (urlsplit(job_url).hostname or "").lower()
if job_host != source_host or job_url in seen_urls:
continue
title_node = card.select_one(".job-title")
employer_node = card.select_one(".company-title")
title = title_node.get_text(" ", strip=True) if title_node else ""
employer = employer_node.get_text(" ", strip=True) if employer_node else ""
if not title:
continue
date_node = card.select_one(".bottom-info")
date_posted = date_node.get_text(" ", strip=True) if date_node else ""
external_id = urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1]
description = f"Vacature van {employer or 'een Corda-werkgever'} via Corda Campus."
jobs.append(
ExtractedJob(
url=job_url,
title=title,
employer_name=employer,
external_id=external_id,
location_text="Hasselt, Limburg",
region="Limburg",
country="BE",
description_text=description,
date_posted=date_posted,
raw={"regional_listing": "corda-campus"},
evidence=[
FieldEvidence("title", "corda-job-card", 0.95, title[:240]),
FieldEvidence("employer_name", "corda-job-card", 0.92, employer[:240]),
FieldEvidence(
"location_text",
"regional-source-scope",
0.72,
"Corda Campus, Hasselt",
),
],
)
)
seen_urls.add(job_url)
warnings = [] if jobs else ["Geen actuele Corda-vacatures gevonden"]
return ExtractionResult(
jobs,
self.parser_key,
self.parser_version,
0.9 if jobs else 0.0,
warnings,
)
class LocalEmployerListingAdapter:
"""Extract reviewed public listing pages of employers in the Hasselt-Genk area."""
parser_key = "regional-local-employers"
parser_version = "1.0.0"
supported_routes = {
"acagroup.be": {"/en/jobs"},
"www.xploregroup.be": {"/en/jobs"},
"www.uhasselt.be": {"/vacatures"},
"ses.pxl.be": {"/"},
"ziekenhuis-oost-limburg.cvw.io": {"/"},
}
def _supports_url(self, url: str) -> bool:
return self._route(url) is not None
def _route(self, url: str) -> tuple[str, str] | None:
parts = urlsplit(url)
host = (parts.hostname or "").lower()
path = parts.path.rstrip("/") or "/"
if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
return None
return host, path
@staticmethod
def _job(
*,
url: str,
title: str,
employer: str,
location: str,
description: str,
external_id: str,
evidence_source: str,
) -> ExtractedJob:
return ExtractedJob(
url=url,
title=title,
employer_name=employer,
external_id=external_id,
location_text=location,
region="Limburg",
country="BE",
description_text=description,
raw={"regional_listing": evidence_source},
evidence=[
FieldEvidence("title", evidence_source, 0.94, title[:240]),
FieldEvidence("employer_name", "reviewed-source", 0.96, employer),
FieldEvidence("location_text", "regional-source-scope", 0.78, location),
],
)
@staticmethod
def _same_host_url(base_url: str, href: str) -> str:
job_url = urljoin(base_url, href)
if (urlsplit(job_url).hostname or "").lower() != (
urlsplit(base_url).hostname or ""
).lower():
return ""
return job_url
def _aca(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.find_all("a", href=True):
href = str(link.get("href") or "")
if not href.startswith("/en/jobs/"):
continue
title_node = link.find("h3")
title = title_node.get_text(" ", strip=True) if title_node else ""
job_url = self._same_host_url(url, href)
if not title or not job_url:
continue
description_node = link.find("p")
description = (
description_node.get_text(" ", strip=True)
if description_node
else f"Vacature bij ACA Group: {title}."
)
jobs.append(
self._job(
url=job_url,
title=title,
employer="ACA Group",
location="Hasselt (hybride; kantoorselectie per vacature)",
description=description,
external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1],
evidence_source="aca-job-card",
)
)
return jobs
def _xplore(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for heading in soup.find_all("h3"):
link = heading.find("a", href=True)
if not link or not str(link.get("href") or "").startswith("/en/jobs/"):
continue
card = heading.parent
location_node = next(
(
node
for node in card.find_all("p")
if "Hasselt" in node.get_text(" ", strip=True)
),
None,
)
if not location_node:
continue
title = link.get_text(" ", strip=True)
job_url = self._same_host_url(url, str(link.get("href") or ""))
if not title or not job_url:
continue
location = location_node.get_text(" ", strip=True)
jobs.append(
self._job(
url=job_url,
title=title,
employer="Xplore Group",
location=location,
description=(
f"Vacature bij Xplore Group met Hasselt als mogelijke werklocatie: {title}."
),
external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1],
evidence_source="xplore-job-card",
)
)
return jobs
def _uhasselt(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select("section.vacancy-item"):
title_node = card.find("h3")
link = card.find("a", href=True)
title = title_node.get_text(" ", strip=True) if title_node else ""
job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
if not title or not job_url or "/vacatures/detail/" not in urlsplit(job_url).path:
continue
external_id = urlsplit(job_url).path.split("/detail/", 1)[-1].split("-", 1)[0]
jobs.append(
self._job(
url=job_url,
title=title,
employer="Universiteit Hasselt",
location="Hasselt/Diepenbeek, Limburg",
description=card.get_text(" ", strip=True),
external_id=external_id,
evidence_source="uhasselt-vacancy-card",
)
)
return jobs
def _pxl(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for card in soup.select(".vacature-card[id^='Vacature_']"):
external_id = str(card.get("id") or "").removeprefix("Vacature_")
title_node = card.select_one(".vacature-card-titel")
title = title_node.get_text(" ", strip=True) if title_node else ""
if not title or not external_id.isdigit():
continue
cells = [
node.get_text(" ", strip=True) for node in card.select("td.vacature-card-td-text")
]
campus = next((value for value in cells if value.startswith("Campus ")), "Hasselt")
job_url = f"{url.rstrip('/')}?vacature_id={external_id}"
jobs.append(
self._job(
url=job_url,
title=title,
employer="Hogeschool PXL",
location=f"{campus}, Limburg",
description=" · ".join(cells),
external_id=external_id,
evidence_source="pxl-vacancy-card",
)
)
return jobs
def _zol(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
jobs = []
for link in soup.select("a[data-item='readmore'][data-jobid][href]"):
external_id = str(link.get("data-jobid") or "")
title_node = link.select_one(".job-title")
title = title_node.get_text(" ", strip=True) if title_node else ""
job_url = self._same_host_url(url, str(link.get("href") or ""))
if not title or not external_id.isdigit() or not job_url:
continue
jobs.append(
self._job(
url=job_url,
title=title,
employer="Ziekenhuis Oost-Limburg",
location="Genk/Lanaken/Maaseik, Limburg",
description=f"Publieke vacature van Ziekenhuis Oost-Limburg: {title}.",
external_id=external_id,
evidence_source="zol-cvwarehouse-card",
)
)
return jobs
def extract(self, content: str, *, url: str) -> ExtractionResult:
route = self._route(url)
if route is None:
return ExtractionResult(
[],
self.parser_key,
self.parser_version,
0.0,
["Onherkenbare lokale werkgeversbron"],
)
host, _ = route
soup = BeautifulSoup(content, "lxml")
extractors = {
"acagroup.be": self._aca,
"www.xploregroup.be": self._xplore,
"www.uhasselt.be": self._uhasselt,
"ses.pxl.be": self._pxl,
"ziekenhuis-oost-limburg.cvw.io": self._zol,
}
jobs = extractors[host](soup, url)
unique_jobs = list({job.url: job for job in jobs}.values())
warnings = [] if unique_jobs else ["Geen actuele lokale vacatures gevonden"]
return ExtractionResult(
unique_jobs,
self.parser_key,
self.parser_version,
0.9 if unique_jobs else 0.0,
warnings,
)
+11
View File
@@ -12,6 +12,9 @@ from .ats import (
from .base import ExtractionResult
from .generic_html import GenericHtmlAdapter
from .jsonld import JsonLdJobPostingAdapter
from .kempen import KempenEmployerAdapter
from .mol_region import MolRegionEmployerAdapter
from .regional import CordaCampusAdapter, LocalEmployerListingAdapter
from .rss import RssAdapter
@@ -22,10 +25,18 @@ class AdapterRegistry:
self.recruitee = RecruiteeAdapter()
self.smartrecruiters = SmartRecruitersAdapter()
self.workable = WorkableAdapter()
self.corda_campus = CordaCampusAdapter()
self.local_employers = LocalEmployerListingAdapter()
self.mol_region_employers = MolRegionEmployerAdapter()
self.kempen_employers = KempenEmployerAdapter()
self.jsonld = JsonLdJobPostingAdapter()
self.generic = GenericHtmlAdapter()
self.rss = RssAdapter()
self.providers = [
self.corda_campus,
self.local_employers,
self.mol_region_employers,
self.kempen_employers,
self.greenhouse,
self.lever,
self.recruitee,
+17
View File
@@ -2,6 +2,7 @@ from django.contrib import admin
from .models import (
EmailMessageRecord,
MailboxConnection,
RawDocument,
Source,
SourceLease,
@@ -12,6 +13,22 @@ from .models import (
)
@admin.register(MailboxConnection)
class MailboxConnectionAdmin(admin.ModelAdmin):
list_display = (
"platform",
"provider",
"username",
"enabled",
"poll_interval_minutes",
"last_success_at",
"next_poll_at",
)
list_filter = ("platform", "provider", "enabled")
search_fields = ("username",)
exclude = ("encrypted_password", "lease_token")
@admin.register(Source)
class SourceAdmin(admin.ModelAdmin):
list_display = (
+52
View File
@@ -2,6 +2,58 @@ from __future__ import annotations
from django import forms
from .models import MailboxConnection
class MailboxConnectionForm(forms.ModelForm):
password = forms.CharField(
required=False,
label="App-wachtwoord",
help_text="Laat leeg bij wijzigen om het bestaande app-wachtwoord te behouden.",
widget=forms.PasswordInput(
attrs={"autocomplete": "new-password", "placeholder": "App-wachtwoord"},
render_value=False,
),
)
class Meta:
model = MailboxConnection
fields = (
"platform",
"provider",
"custom_host",
"port",
"username",
"mailbox",
"poll_interval_minutes",
"enabled",
)
labels = {
"platform": "Vacatureplatform",
"provider": "Mailboxprovider",
"custom_host": "Aangepaste IMAP-host",
"port": "IMAP-poort",
"username": "Mailboxaccount",
"mailbox": "Map",
"poll_interval_minutes": "Controlefrequentie",
"enabled": "Automatisch synchroniseren",
}
widgets = {
"username": forms.EmailInput(attrs={"autocomplete": "username"}),
"custom_host": forms.TextInput(
attrs={"autocomplete": "off", "placeholder": "imap.provider.be"}
),
"mailbox": forms.TextInput(attrs={"autocomplete": "off"}),
}
def clean(self):
data = super().clean()
if data.get("provider") != MailboxConnection.Provider.CUSTOM:
data["custom_host"] = ""
if not self.instance.pk and not data.get("password"):
self.add_error("password", "Een app-wachtwoord is verplicht voor een nieuwe koppeling.")
return data
class ManualImportForm(forms.Form):
source_url = forms.URLField(
@@ -31,11 +31,11 @@ class Command(BaseCommand):
if not hostname:
raise CommandError("--url bevat geen geldig domein")
source, _ = Source.objects.get_or_create(
domain=hostname,
base_url=options["url"],
source_type=options["source_type"],
defaults={
"domain": hostname,
"name": options["source_name"],
"base_url": options["url"],
"status": Source.Status.ACTIVE,
"policy": Source.Policy.ALLOW,
"parser_key": "auto",
@@ -0,0 +1,17 @@
from django.core.management.base import BaseCommand, CommandError
from apps.sources.services.mailbox_connections import (
MailboxCredentialError,
rotate_mailbox_credentials,
)
class Command(BaseCommand):
help = "Versleutel alle mailbox-app-wachtwoorden opnieuw met de eerste ingestelde sleutel."
def handle(self, *args, **options):
try:
count = rotate_mailbox_credentials()
except MailboxCredentialError as exc:
raise CommandError(str(exc)) from exc
self.stdout.write(self.style.SUCCESS(f"{count} mailboxcredential(s) geroteerd."))
@@ -5,8 +5,9 @@ from urllib.parse import urlsplit
import yaml
from django.core.management.base import BaseCommand, CommandError
from django.utils.dateparse import parse_datetime
from apps.sources.models import Source
from apps.sources.models import Source, SourcePolicyReview
from apps.sources.services.policy import is_denied_domain
@@ -21,7 +22,38 @@ class Command(BaseCommand):
if not path.is_file():
raise CommandError(f"Seedbestand ontbreekt: {path}")
payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
created = updated = 0
created = updated = reviews_created = 0
if payload.get("disable_demo_sources") is True:
demo_sources = Source.objects.filter(domain="jobs.example.org")
for demo_source in demo_sources:
demo_source.status = Source.Status.DISABLED
demo_source.policy = Source.Policy.DENY
demo_source.policy_reason = "Demobron verborgen na installatie van livebronnen"
demo_source.metadata = {**demo_source.metadata, "demo": True}
demo_source.save(
update_fields=[
"status",
"policy",
"policy_reason",
"metadata",
"updated_at",
]
)
retired = 0
for retired_url in payload.get("retire_source_urls", []):
for retired_source in Source.objects.filter(base_url=str(retired_url).strip()):
retired_source.status = Source.Status.DISABLED
retired_source.next_run_at = None
retired_source.metadata = {
**retired_source.metadata,
"retired": True,
"retired_reason": "Buiten de ingestelde regionale dekking",
"hidden_from_source_list": True,
}
retired_source.save(
update_fields=["status", "next_run_at", "metadata", "updated_at"]
)
retired += 1
for item in payload.get("sources", []):
url = str(item.get("url") or "").strip()
domain = (urlsplit(url).hostname or "").lower()
@@ -30,11 +62,11 @@ class Command(BaseCommand):
continue
deny = is_denied_domain(domain)
source, was_created = Source.objects.update_or_create(
domain=domain,
base_url=url,
source_type=item.get("type", Source.Type.EMPLOYER),
defaults={
"domain": domain,
"name": item.get("name") or domain,
"base_url": url,
"status": Source.Status.PAUSED
if deny
else item.get("status", Source.Status.TRIAL),
@@ -46,10 +78,39 @@ class Command(BaseCommand):
else item.get("policy_reason", ""),
"parser_key": item.get("parser", "auto"),
"crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)),
"minimum_interval_seconds": int(item.get("minimum_interval_seconds", 30)),
"allow_public_endpoint": bool(item.get("allow_public_endpoint", False)),
"metadata": item.get("metadata") or {},
},
)
review = item.get("review") or {}
if review:
expires_at = parse_datetime(str(review.get("expires_at") or ""))
if expires_at is None:
raise CommandError(
f"Bron {source.name}: review.expires_at moet een ISO-datetime zijn"
)
decision = str(review.get("decision") or "")
if decision not in SourcePolicyReview.Decision.values:
raise CommandError(f"Bron {source.name}: ongeldige reviewbeslissing")
_, review_created = SourcePolicyReview.objects.get_or_create(
source=source,
decision=decision,
evidence_link=str(review.get("evidence_link") or ""),
expires_at=expires_at,
defaults={
"scope": SourcePolicyReview.Scope.SOURCE,
"reason": str(review.get("reason") or "Bronseedreview"),
"notes": str(review.get("notes") or ""),
"metadata": review.get("metadata") or {},
},
)
reviews_created += int(review_created)
created += int(was_created)
updated += int(not was_created)
self.stdout.write(
self.style.SUCCESS(f"Bronnen: {created} aangemaakt, {updated} bijgewerkt")
self.style.SUCCESS(
f"Bronnen: {created} aangemaakt, {updated} bijgewerkt, "
f"{reviews_created} reviews geregistreerd, {retired} buiten regio uitgeschakeld"
)
)
@@ -0,0 +1,21 @@
# Generated by Django 5.2.16 on 2026-07-21 22:59
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sources', '0003_source_lease_and_origin_state'),
]
operations = [
migrations.RemoveConstraint(
model_name='source',
name='unique_domain_source_type',
),
migrations.AddConstraint(
model_name='source',
constraint=models.UniqueConstraint(condition=models.Q(('base_url', ''), _negated=True), fields=('source_type', 'base_url'), name='unique_source_type_base_url_nonempty'),
),
]
@@ -0,0 +1,67 @@
# Generated by Django 5.2.16 on 2026-07-22 00:12
import django.db.models.deletion
import django.utils.timezone
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sources', '0004_remove_source_unique_domain_source_type_and_more'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AlterField(
model_name='emailmessagerecord',
name='message_id',
field=models.CharField(max_length=998),
),
migrations.CreateModel(
name='MailboxConnection',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('platform', models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed')], max_length=24)),
('provider', models.CharField(choices=[('gmail', 'Gmail'), ('outlook', 'Outlook / Microsoft 365'), ('custom', 'Andere IMAP-provider')], max_length=24)),
('custom_host', models.CharField(blank=True, max_length=255)),
('port', models.PositiveIntegerField(default=993)),
('username', models.CharField(max_length=320)),
('encrypted_password', models.TextField()),
('mailbox', models.CharField(default='INBOX', max_length=255)),
('enabled', models.BooleanField(default=True)),
('poll_interval_minutes', models.PositiveIntegerField(choices=[(5, 'Elke 5 minuten'), (15, 'Elke 15 minuten'), (30, 'Elke 30 minuten'), (60, 'Elk uur'), (180, 'Elke 3 uur'), (360, 'Elke 6 uur')], default=30)),
('next_poll_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
('last_polled_at', models.DateTimeField(blank=True, null=True)),
('last_success_at', models.DateTimeField(blank=True, null=True)),
('last_error_category', models.CharField(blank=True, max_length=80)),
('last_error_message', models.CharField(blank=True, max_length=500)),
('lease_token', models.CharField(blank=True, max_length=64)),
('lease_expires_at', models.DateTimeField(blank=True, db_index=True, null=True)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mailbox_connections', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['platform'],
},
),
migrations.AddField(
model_name='emailmessagerecord',
name='mailbox_connection',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='sources.mailboxconnection'),
),
migrations.AddConstraint(
model_name='emailmessagerecord',
constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', False)), fields=('mailbox_connection', 'message_id'), name='unique_message_per_mailbox_connection'),
),
migrations.AddConstraint(
model_name='emailmessagerecord',
constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', True)), fields=('message_id',), name='unique_legacy_email_message_id'),
),
migrations.AddConstraint(
model_name='mailboxconnection',
constraint=models.UniqueConstraint(fields=('user', 'platform'), name='unique_mailbox_platform_per_user'),
),
]
@@ -0,0 +1,18 @@
# Generated by Django 5.2.16 on 2026-07-22 00:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sources', '0005_alter_emailmessagerecord_message_id_and_more'),
]
operations = [
migrations.AlterField(
model_name='mailboxconnection',
name='platform',
field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('eurobrussels', 'EuroBrussels'), ('brusselsjobs', 'BrusselsJobs')], max_length=24),
),
]
@@ -0,0 +1,26 @@
# Generated by Django 5.2.16 on 2026-07-22 01:25
from django.db import migrations, models
def disable_retired_brussels_mailboxes(apps, schema_editor):
mailbox_connection = apps.get_model("sources", "MailboxConnection")
mailbox_connection.objects.filter(
platform__in=("eurobrussels", "brusselsjobs")
).update(enabled=False)
class Migration(migrations.Migration):
dependencies = [
('sources', '0006_alter_mailboxconnection_platform'),
]
operations = [
migrations.RunPython(disable_retired_brussels_mailboxes, migrations.RunPython.noop),
migrations.AlterField(
model_name='mailboxconnection',
name='platform',
field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('careerjet', 'Careerjet'), ('randstad', 'Randstad'), ('roberthalf', 'Robert Half')], max_length=24),
),
]
+109 -2
View File
@@ -62,7 +62,9 @@ class Source(TimeStampedModel):
ordering = ["name"]
constraints = [
models.UniqueConstraint(
fields=["domain", "source_type"], name="unique_domain_source_type"
fields=["source_type", "base_url"],
condition=~models.Q(base_url=""),
name="unique_source_type_base_url_nonempty",
)
]
@@ -315,8 +317,101 @@ class RawDocument(TimeStampedModel):
return self.final_url or self.url or f"Document {self.pk}"
class MailboxConnection(TimeStampedModel):
class Platform(models.TextChoices):
VDAB = "vdab", "VDAB"
INDEED = "indeed", "Indeed"
LINKEDIN = "linkedin", "LinkedIn"
ICTJOB = "ictjob", "ictjob.be"
JOBAT = "jobat", "Jobat"
STEPSTONE = "stepstone", "StepStone"
CAREERJET = "careerjet", "Careerjet"
RANDSTAD = "randstad", "Randstad"
ROBERTHALF = "roberthalf", "Robert Half"
class Provider(models.TextChoices):
GMAIL = "gmail", "Gmail"
OUTLOOK = "outlook", "Outlook / Microsoft 365"
CUSTOM = "custom", "Andere IMAP-provider"
class PollInterval(models.IntegerChoices):
FIVE_MINUTES = 5, "Elke 5 minuten"
FIFTEEN_MINUTES = 15, "Elke 15 minuten"
THIRTY_MINUTES = 30, "Elke 30 minuten"
HOURLY = 60, "Elk uur"
THREE_HOURS = 180, "Elke 3 uur"
SIX_HOURS = 360, "Elke 6 uur"
PROVIDER_HOSTS = {
Provider.GMAIL: "imap.gmail.com",
Provider.OUTLOOK: "outlook.office365.com",
}
user = models.ForeignKey(
settings.AUTH_USER_MODEL,
on_delete=models.CASCADE,
related_name="mailbox_connections",
)
platform = models.CharField(max_length=24, choices=Platform.choices)
provider = models.CharField(max_length=24, choices=Provider.choices)
custom_host = models.CharField(max_length=255, blank=True)
port = models.PositiveIntegerField(default=993)
username = models.CharField(max_length=320)
encrypted_password = models.TextField()
mailbox = models.CharField(max_length=255, default="INBOX")
enabled = models.BooleanField(default=True)
poll_interval_minutes = models.PositiveIntegerField(
choices=PollInterval.choices,
default=PollInterval.THIRTY_MINUTES,
)
next_poll_at = models.DateTimeField(default=timezone.now, db_index=True)
last_polled_at = models.DateTimeField(null=True, blank=True)
last_success_at = models.DateTimeField(null=True, blank=True)
last_error_category = models.CharField(max_length=80, blank=True)
last_error_message = models.CharField(max_length=500, blank=True)
lease_token = models.CharField(max_length=64, blank=True)
lease_expires_at = models.DateTimeField(null=True, blank=True, db_index=True)
class Meta:
ordering = ["platform"]
constraints = [
models.UniqueConstraint(
fields=["user", "platform"], name="unique_mailbox_platform_per_user"
)
]
def __str__(self) -> str:
return f"{self.get_platform_display()} · {self.username}"
@property
def imap_host(self) -> str:
return self.PROVIDER_HOSTS.get(self.provider, self.custom_host).strip().lower()
def clean(self) -> None:
host = self.imap_host
if (
not host
or "://" in host
or "/" in host
or "@" in host
or any(c.isspace() for c in host)
):
raise ValidationError({"custom_host": "Vul een geldige IMAP-hostnaam in."})
if not 1 <= self.port <= 65535:
raise ValidationError({"port": "De IMAP-poort moet tussen 1 en 65535 liggen."})
if "\n" in self.mailbox or "\r" in self.mailbox:
raise ValidationError({"mailbox": "De mailboxnaam bevat ongeldige tekens."})
class EmailMessageRecord(TimeStampedModel):
message_id = models.CharField(max_length=998, unique=True)
mailbox_connection = models.ForeignKey(
MailboxConnection,
on_delete=models.CASCADE,
null=True,
blank=True,
related_name="messages",
)
message_id = models.CharField(max_length=998)
mailbox = models.CharField(max_length=255, default="INBOX")
sender = models.CharField(max_length=500, blank=True)
subject = models.CharField(max_length=998, blank=True)
@@ -330,6 +425,18 @@ class EmailMessageRecord(TimeStampedModel):
class Meta:
ordering = ["-received_at", "-created_at"]
constraints = [
models.UniqueConstraint(
fields=["mailbox_connection", "message_id"],
condition=models.Q(mailbox_connection__isnull=False),
name="unique_message_per_mailbox_connection",
),
models.UniqueConstraint(
fields=["message_id"],
condition=models.Q(mailbox_connection__isnull=True),
name="unique_legacy_email_message_id",
),
]
def __str__(self) -> str:
return self.subject or self.message_id
+88
View File
@@ -0,0 +1,88 @@
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True)
class PlatformAlert:
key: str
label: str
domains: tuple[str, ...]
alert_url: str
help_text: str
PLATFORM_ALERTS = {
alert.key: alert
for alert in (
PlatformAlert(
"vdab",
"VDAB",
("vdab.be",),
"https://www.vdab.be/jobs/job-alerts",
"Brede Vlaamse vacaturealerts.",
),
PlatformAlert(
"indeed",
"Indeed",
("indeed.com", "indeed.be"),
"https://support.indeed.com/hc/nl/articles/204488890-"
"Vacature-alerts-activeren-stopzetten-en-beheren",
"Algemene en regionale vacaturealerts.",
),
PlatformAlert(
"linkedin",
"LinkedIn",
("linkedin.com",),
"https://www.linkedin.com/help/linkedin/answer/a511279",
"Dagelijkse of wekelijkse alerts, maximaal twintig per LinkedIn-account.",
),
PlatformAlert(
"ictjob",
"ictjob.be",
("ictjob.be",),
"https://www.ictjob.be/nl/it-job-alert",
"Gespecialiseerde Belgische IT- en telecomvacatures.",
),
PlatformAlert(
"jobat",
"Jobat",
("jobat.be",),
"https://www.jobat.be/nl/jobalert",
"Belgische jobs per functie, regio en contracttype.",
),
PlatformAlert(
"stepstone",
"StepStone",
("stepstone.be",),
"https://www.stepstone.be/vacatures",
"Dagelijkse vergelijkbare vacatures vanuit een zoekopdracht.",
),
PlatformAlert(
"careerjet",
"Careerjet",
("careerjet.be",),
"https://www.careerjet.be/",
"Brede Belgische zoekresultaten met regio- en postcodegebonden e-mailalerts.",
),
PlatformAlert(
"randstad",
"Randstad",
("randstad.be",),
"https://www.randstad.be/jobalert-aanmaken/",
"Belgische jobs met instelbare specialisatie, locatie, contract en frequentie.",
),
PlatformAlert(
"roberthalf",
"Robert Half",
("roberthalf.com",),
"https://www.roberthalf.com/be/en/find-jobs/job-alerts",
"Belgische IT- en digitaljobs, waaronder systeem-, netwerk- en supportfuncties.",
),
)
}
def platform_label(key: str) -> str:
alert = PLATFORM_ALERTS.get(key)
return alert.label if alert else key
+21 -16
View File
@@ -140,23 +140,28 @@ def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int
for candidate in ordered:
provenance = _provenance_entry(candidate)
now = _utc_now()
source, was_created = Source.objects.get_or_create(
domain=candidate.domain,
source_type=candidate.source_type,
defaults={
"name": _candidate_name(candidate),
"base_url": candidate.url,
"status": Source.Status.CANDIDATE,
"policy": Source.Policy.REVIEW,
"policy_reason": "Automatisch ontdekt",
"parser_key": "auto",
"strict_mode": True,
"metadata": {
"discovery": [provenance],
"discovered_at": now,
},
},
source = (
Source.objects.filter(
domain=candidate.domain,
source_type=candidate.source_type,
)
.order_by("id")
.first()
)
was_created = source is None
if source is None:
source = Source.objects.create(
domain=candidate.domain,
source_type=candidate.source_type,
name=_candidate_name(candidate),
base_url=candidate.url,
status=Source.Status.CANDIDATE,
policy=Source.Policy.REVIEW,
policy_reason="Automatisch ontdekt",
parser_key="auto",
strict_mode=True,
metadata={"discovery": [provenance], "discovered_at": now},
)
if was_created:
created += 1
continue
+41 -9
View File
@@ -13,7 +13,8 @@ from django.utils import timezone
from apps.jobs.services.normalization import normalize_extracted_job
from apps.jobs.services.pipeline import persist_draft
from apps.sources.adapters.email_alert import EmailAlertAdapter
from apps.sources.models import EmailMessageRecord, RawDocument, Source
from apps.sources.models import EmailMessageRecord, MailboxConnection, RawDocument, Source
from apps.sources.platforms import platform_label
def message_identity(raw_message: bytes) -> str:
@@ -24,22 +25,32 @@ def message_identity(raw_message: bytes) -> str:
@transaction.atomic
def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageRecord:
def ingest_email(
raw_message: bytes,
*,
mailbox: str = "INBOX",
mailbox_connection: MailboxConnection | None = None,
platform: str | None = None,
) -> EmailMessageRecord:
parsed = BytesParser(policy=policy.default).parsebytes(raw_message)
message_id = message_identity(raw_message)
existing = EmailMessageRecord.objects.filter(message_id=message_id).first()
existing = EmailMessageRecord.objects.filter(
message_id=message_id, mailbox_connection=mailbox_connection
).first()
if existing:
return existing
source_domain = f"{platform}.mailbox.local" if platform else "mailbox.local"
source_name = f"{platform_label(platform)} vacaturemailbox" if platform else "Vacaturemailbox"
source, _ = Source.objects.get_or_create(
domain="mailbox.local",
domain=source_domain,
source_type=Source.Type.EMAIL,
defaults={
"name": "Vacaturemailbox",
"name": source_name,
"status": Source.Status.ACTIVE,
"policy": Source.Policy.ALLOW,
"parser_key": "email-alert",
"crawl_interval_minutes": 10,
"crawl_interval_minutes": 1440,
},
)
content_hash = hashlib.sha256(raw_message).hexdigest()
@@ -52,7 +63,11 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
body_text=raw_message.decode("utf-8", errors="replace"),
byte_length=len(raw_message),
retain_until=retain_until,
metadata={"mailbox": mailbox},
metadata={
"mailbox": mailbox,
"mailbox_connection_id": mailbox_connection.pk if mailbox_connection else None,
"alert_provider": platform or "unknown",
},
)
received_at = None
if parsed.get("date"):
@@ -64,12 +79,28 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
received_at = None
adapter = EmailAlertAdapter()
result = adapter.extract_message(raw_message)
result = adapter.extract_message(raw_message, expected_provider=platform)
detected_provider = next(
(
str(extracted.raw.get("alert_provider"))
for extracted in result.jobs
if extracted.raw.get("alert_provider")
),
"unknown",
)
alert_provider = platform or detected_provider
document.parser_key = result.parser_key
document.parser_version = result.parser_version
document.extraction_confidence = result.confidence
document.metadata = {**document.metadata, "alert_provider": alert_provider}
document.save(
update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"]
update_fields=[
"parser_key",
"parser_version",
"extraction_confidence",
"metadata",
"updated_at",
]
)
links: list[str] = []
errors: list[str] = []
@@ -87,6 +118,7 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR
except Exception as exc:
errors.append(f"{exc.__class__.__name__}: {exc}")
return EmailMessageRecord.objects.create(
mailbox_connection=mailbox_connection,
message_id=message_id,
mailbox=mailbox,
sender=str(parsed.get("from") or "")[:500],
+2
View File
@@ -14,6 +14,7 @@ from django.conf import settings
from apps.sources.models import Source
from .policy import assess_url
from .tls import trusted_tls_context
from .url_security import ValidatedUrl, validate_public_url
ALLOWED_CONTENT_TYPES = (
@@ -123,6 +124,7 @@ def fetch_url(
timeout=httpx.Timeout(settings.FETCHER_TIMEOUT_SECONDS),
follow_redirects=False,
headers=headers,
verify=trusted_tls_context(),
)
current_url = url
previous_validation: ValidatedUrl | None = None
+84
View File
@@ -0,0 +1,84 @@
from __future__ import annotations
import imaplib
from collections.abc import Callable
from contextlib import suppress
from typing import Any
from django.conf import settings
from apps.sources.models import EmailMessageRecord, MailboxConnection
from apps.sources.services.email_import import ingest_email, message_identity
from apps.sources.services.mailbox_connections import decrypt_mailbox_password
from apps.sources.services.tls import trusted_tls_context
from apps.sources.services.url_security import validate_public_url
ImapClientFactory = Callable[..., Any]
HostValidator = Callable[[str], object]
def poll_imap_mailbox(
connection: MailboxConnection,
*,
client_factory: ImapClientFactory | None = None,
host_validator: HostValidator = validate_public_url,
) -> dict[str, int | str]:
"""Import one bounded mailbox batch without letting one bad message abort the batch."""
if not connection.enabled:
return {"status": "disabled", "imported": 0}
if client_factory is None:
client_factory = imaplib.IMAP4_SSL
host_validator(f"https://{connection.imap_host}")
password = decrypt_mailbox_password(connection)
client_options = {"timeout": settings.IMAP_CONNECT_TIMEOUT_SECONDS}
if client_factory is imaplib.IMAP4_SSL:
client_options["ssl_context"] = trusted_tls_context()
client = client_factory(connection.imap_host, connection.port, **client_options)
counters = {
"imported": 0,
"duplicates": 0,
"skipped_oversized": 0,
"failed": 0,
}
try:
client.login(connection.username, password)
status, _ = client.select(connection.mailbox, readonly=True)
if status != "OK":
raise RuntimeError("IMAP-mailbox kon niet worden geopend")
status, data = client.uid("search", None, "ALL")
if status != "OK":
raise RuntimeError("IMAP-zoekopdracht mislukt")
uids = (data[0] or b"").split()[-settings.IMAP_MAX_MESSAGES_PER_POLL :]
for uid in uids:
try:
status, payload = client.uid("fetch", uid, "(RFC822)")
if status != "OK" or not payload:
counters["failed"] += 1
continue
raw = next((part[1] for part in payload if isinstance(part, tuple)), None)
if not isinstance(raw, bytes):
counters["failed"] += 1
continue
if len(raw) > settings.IMAP_MAX_MESSAGE_BYTES:
counters["skipped_oversized"] += 1
continue
identity = message_identity(raw)
if EmailMessageRecord.objects.filter(
mailbox_connection=connection, message_id=identity
).exists():
counters["duplicates"] += 1
continue
ingest_email(
raw,
mailbox=connection.mailbox,
mailbox_connection=connection,
platform=connection.platform,
)
counters["imported"] += 1
except Exception:
counters["failed"] += 1
return {"status": "ok", **counters}
finally:
with suppress(Exception):
client.logout()
@@ -0,0 +1,148 @@
from __future__ import annotations
from datetime import timedelta
from uuid import uuid4
from cryptography.fernet import Fernet, InvalidToken, MultiFernet
from django.conf import settings
from django.db import transaction
from django.utils import timezone
from apps.sources.models import MailboxConnection
class MailboxCredentialError(RuntimeError):
"""Safe credential configuration error without secret material."""
def _cipher() -> MultiFernet:
keys = settings.MAILBOX_CREDENTIAL_KEYS
if not keys:
raise MailboxCredentialError(
"Credentialopslag is niet geconfigureerd. Stel MAILBOX_CREDENTIAL_KEYS in."
)
try:
return MultiFernet([Fernet(key.encode("ascii")) for key in keys])
except (ValueError, TypeError) as exc:
raise MailboxCredentialError(
"MAILBOX_CREDENTIAL_KEYS bevat een ongeldige sleutel."
) from exc
def encrypt_mailbox_password(password: str) -> str:
if not password:
raise MailboxCredentialError("Een app-wachtwoord is verplicht.")
return _cipher().encrypt(password.encode("utf-8")).decode("ascii")
def decrypt_mailbox_password(connection: MailboxConnection) -> str:
try:
return _cipher().decrypt(connection.encrypted_password.encode("ascii")).decode("utf-8")
except (InvalidToken, ValueError, UnicodeError) as exc:
raise MailboxCredentialError(
"Het opgeslagen app-wachtwoord kan niet worden ontsleuteld. Sla het opnieuw op."
) from exc
@transaction.atomic
def rotate_mailbox_credentials() -> int:
"""Re-encrypt every mailbox password with the first configured key."""
rotated = 0
for connection in MailboxConnection.objects.select_for_update().all():
password = decrypt_mailbox_password(connection)
connection.encrypted_password = encrypt_mailbox_password(password)
connection.save(update_fields=["encrypted_password", "updated_at"])
rotated += 1
return rotated
@transaction.atomic
def save_mailbox_connection(
*,
user,
cleaned_data: dict[str, object],
instance: MailboxConnection | None = None,
) -> MailboxConnection:
connection = instance or MailboxConnection(user=user)
if connection.pk and connection.user_id != user.pk:
raise PermissionError("Mailboxkoppeling behoort niet tot deze gebruiker.")
password = str(cleaned_data.pop("password", "") or "")
for field in (
"platform",
"provider",
"custom_host",
"port",
"username",
"mailbox",
"enabled",
"poll_interval_minutes",
):
setattr(connection, field, cleaned_data[field])
if password:
connection.encrypted_password = encrypt_mailbox_password(password)
elif not connection.encrypted_password:
raise MailboxCredentialError("Een app-wachtwoord is verplicht.")
connection.next_poll_at = timezone.now()
connection.last_error_category = ""
connection.last_error_message = ""
connection.full_clean()
connection.save()
return connection
@transaction.atomic
def claim_mailbox_connection(
*, connection_id: int, worker_token: str | None = None, force: bool = False
) -> tuple[MailboxConnection, str] | None:
now = timezone.now()
connection = MailboxConnection.objects.select_for_update().get(pk=connection_id)
if not connection.enabled:
return None
if connection.lease_expires_at and connection.lease_expires_at > now:
return None
if not force and connection.next_poll_at > now:
return None
token = worker_token or uuid4().hex
connection.lease_token = token
connection.lease_expires_at = now + timedelta(minutes=5)
connection.save(update_fields=["lease_token", "lease_expires_at", "updated_at"])
return connection, token
@transaction.atomic
def finish_mailbox_poll(
*,
connection_id: int,
worker_token: str,
success: bool,
error_category: str = "",
error_message: str = "",
) -> None:
connection = MailboxConnection.objects.select_for_update().get(pk=connection_id)
if connection.lease_token != worker_token:
return
now = timezone.now()
connection.last_polled_at = now
connection.next_poll_at = now + timedelta(minutes=connection.poll_interval_minutes)
connection.lease_token = ""
connection.lease_expires_at = None
if success:
connection.last_success_at = now
connection.last_error_category = ""
connection.last_error_message = ""
else:
connection.last_error_category = error_category[:80]
connection.last_error_message = error_message[:500]
connection.save(
update_fields=[
"last_polled_at",
"next_poll_at",
"lease_token",
"lease_expires_at",
"last_success_at",
"last_error_category",
"last_error_message",
"updated_at",
]
)
+4
View File
@@ -19,6 +19,10 @@ DEFAULT_DENYLIST = {
"stepstone.be",
"jobat.be",
"vdab.be",
"ictjob.be",
"careerjet.be",
"randstad.be",
"roberthalf.com",
}
+27 -9
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
from collections.abc import Iterable
from dataclasses import dataclass
from datetime import datetime, timedelta
from urllib.parse import urlsplit, urlunsplit
from urllib.parse import urljoin, urlsplit, urlunsplit
import httpx
from django.conf import settings
@@ -11,6 +11,7 @@ from django.utils import timezone
from apps.sources.models import Source, SourceRobotsCache
from .tls import trusted_tls_context
from .url_security import UnsafeUrlError, validate_public_url
ALLOW = "allow"
@@ -123,20 +124,37 @@ def _max_bytes() -> int:
def _fetch_robots(origin: str, *, client: httpx.Client | None = None) -> tuple[str, int, str, str]:
robots_url = _robots_url(origin)
validate_public_url(
robots_url,
allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
)
own_client = client is None
http_client = client or httpx.Client(
timeout=httpx.Timeout(getattr(settings, "ROBOTS_FETCH_TIMEOUT_SECONDS", 5)),
follow_redirects=False,
verify=trusted_tls_context(),
)
try:
response = http_client.get(
robots_url,
headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
)
current_url = robots_url
for _ in range(getattr(settings, "FETCHER_MAX_REDIRECTS", 5) + 1):
validate_public_url(
current_url,
allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
)
response = http_client.get(
current_url,
headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
)
if response.status_code not in {301, 302, 303, 307, 308}:
break
location = response.headers.get("location")
if not location:
raise httpx.HTTPStatusError(
"Robotsredirect zonder Location-header",
request=response.request,
response=response,
)
current_url = urljoin(current_url, location)
else:
raise httpx.TooManyRedirects(
"Te veel redirects bij robotscontrole", request=response.request
)
finally:
if own_client:
http_client.close()
+16
View File
@@ -0,0 +1,16 @@
from __future__ import annotations
import ssl
from functools import lru_cache
from pathlib import Path
import certifi
@lru_cache(maxsize=1)
def trusted_tls_context() -> ssl.SSLContext:
"""Build a strict TLS context from the pinned application CA bundle."""
bundle = Path(certifi.where()).resolve(strict=True)
if not bundle.is_file():
raise RuntimeError("De vertrouwde CA-bundel ontbreekt.")
return ssl.create_default_context(cafile=str(bundle))
+63 -37
View File
@@ -1,7 +1,5 @@
from __future__ import annotations
import imaplib
from contextlib import suppress
from datetime import timedelta
from uuid import uuid4
@@ -11,8 +9,7 @@ from django.db.models import Q
from django.utils import timezone
from apps.jobs.services.pipeline import process_raw_document
from apps.sources.models import EmailMessageRecord, RawDocument, Source, SourceRun
from apps.sources.services.email_import import ingest_email, message_identity
from apps.sources.models import MailboxConnection, RawDocument, Source, SourceRun
from apps.sources.services.fetcher import (
FetchError,
FetchTimeoutError,
@@ -25,6 +22,12 @@ from apps.sources.services.health import (
evaluate_source_health,
start_health_canary,
)
from apps.sources.services.imap_import import poll_imap_mailbox
from apps.sources.services.mailbox_connections import (
MailboxCredentialError,
claim_mailbox_connection,
finish_mailbox_poll,
)
from apps.sources.services.policy import assess_url
from apps.sources.services.scheduling import (
acquire_source_lease,
@@ -192,40 +195,63 @@ def schedule_due_sources() -> dict[str, int]:
return {"scheduled": len(ids)}
@shared_task(name="apps.sources.tasks.poll_imap_inbox")
def poll_imap_inbox() -> dict[str, int | str]:
if not settings.IMAP_ENABLED:
return {"status": "disabled", "imported": 0}
if not settings.IMAP_HOST or not settings.IMAP_USER or not settings.IMAP_PASSWORD:
return {"status": "not_configured", "imported": 0}
client_cls = imaplib.IMAP4_SSL if settings.IMAP_USE_SSL else imaplib.IMAP4
client = client_cls(settings.IMAP_HOST, settings.IMAP_PORT)
imported = 0
@shared_task(name="apps.sources.tasks.schedule_mailbox_polls")
def schedule_mailbox_polls() -> dict[str, int]:
now = timezone.now()
due_ids = list(
MailboxConnection.objects.filter(enabled=True, next_poll_at__lte=now)
.filter(Q(lease_expires_at__isnull=True) | Q(lease_expires_at__lte=now))
.values_list("id", flat=True)[:100]
)
for connection_id in due_ids:
poll_mailbox.delay(connection_id)
return {"scheduled": len(due_ids)}
@shared_task(bind=True, name="apps.sources.tasks.poll_mailbox")
def poll_mailbox(self, connection_id: int, force: bool = False) -> dict[str, int | str]:
worker_token = str(self.request.id or uuid4())
claimed = claim_mailbox_connection(
connection_id=connection_id, worker_token=worker_token, force=force
)
if claimed is None:
return {"status": "skipped", "imported": 0}
connection, token = claimed
try:
client.login(settings.IMAP_USER, settings.IMAP_PASSWORD)
status, _ = client.select(settings.IMAP_MAILBOX, readonly=not settings.IMAP_MARK_SEEN)
if status != "OK":
raise RuntimeError("IMAP-mailbox kon niet worden geopend")
status, data = client.uid("search", None, "ALL")
if status != "OK":
raise RuntimeError("IMAP-zoekopdracht mislukt")
uids = (data[0] or b"").split()[-200:]
for uid in uids:
status, payload = client.uid("fetch", uid, "(RFC822)")
if status != "OK" or not payload:
continue
raw = next((part[1] for part in payload if isinstance(part, tuple)), None)
if not raw:
continue
message_id = message_identity(raw)
was_known = EmailMessageRecord.objects.filter(message_id=message_id).exists()
ingest_email(raw, mailbox=settings.IMAP_MAILBOX)
if not was_known:
imported += 1
return {"status": "ok", "imported": imported}
finally:
with suppress(Exception):
client.logout()
result = poll_imap_mailbox(connection)
finish_mailbox_poll(
connection_id=connection.pk,
worker_token=token,
success=result["status"] == "ok",
)
return result
except MailboxCredentialError as exc:
finish_mailbox_poll(
connection_id=connection.pk,
worker_token=token,
success=False,
error_category="credentials",
error_message=str(exc),
)
return {"status": "failed", "error_category": "credentials", "imported": 0}
except Exception as exc:
error_category = "imap"
if exc.__class__.__name__.lower().startswith("unsafe"):
error_category = "host_policy"
finish_mailbox_poll(
connection_id=connection.pk,
worker_token=token,
success=False,
error_category=error_category,
error_message="IMAP-verbinding of authenticatie mislukt.",
)
return {"status": "failed", "error_category": error_category, "imported": 0}
@shared_task(name="apps.sources.tasks.poll_imap_inbox")
def poll_imap_inbox() -> dict[str, int]:
"""Compatibility alias for old deployments; it now schedules due per-platform mailboxes."""
return schedule_mailbox_polls()
@shared_task(name="apps.sources.tasks.cleanup_raw_documents")
+14 -1
View File
@@ -1,10 +1,23 @@
from django.urls import path
from .views import SourceListView, bulk_candidate_action, manual_import_view, retry_source
from .views import (
SourceListView,
bulk_candidate_action,
edit_mailbox,
manual_import_view,
retry_source,
save_mailbox,
sync_platform_alerts,
toggle_mailbox,
)
urlpatterns = [
path("", SourceListView.as_view(), name="list"),
path("manual-import/", manual_import_view, name="manual_import"),
path("mailboxes/add/", save_mailbox, name="mailbox_add"),
path("mailboxes/<int:pk>/edit/", edit_mailbox, name="mailbox_edit"),
path("mailboxes/<int:pk>/sync/", sync_platform_alerts, name="sync_platform_alerts"),
path("mailboxes/<int:pk>/toggle/", toggle_mailbox, name="mailbox_toggle"),
path("bulk/", bulk_candidate_action, name="bulk"),
path("<int:pk>/retry/", retry_source, name="retry"),
]
+133 -6
View File
@@ -4,39 +4,88 @@ from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin
from django.db.models import Count, Q
from django.http import HttpRequest
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.utils import timezone
from django.views.decorators.cache import never_cache
from django.views.decorators.http import require_POST
from django.views.generic import ListView
from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
from apps.profiles.models import SearchProfile
from .forms import ManualImportForm
from .models import Source, SourcePolicyReview
from .forms import MailboxConnectionForm, ManualImportForm
from .models import MailboxConnection, Source, SourcePolicyReview
from .platforms import PLATFORM_ALERTS
from .services.mailbox_connections import MailboxCredentialError, save_mailbox_connection
from .services.manual_import import ManualImportError, ManualImportSummary, import_manual_source
from .services.policy import create_policy_review
from .tasks import fetch_source
from .tasks import fetch_source, poll_mailbox
def _source_list_queryset():
return Source.objects.prefetch_related("policy_reviews").all().order_by("name")
return (
Source.objects.prefetch_related("policy_reviews")
.exclude(domain="jobs.example.org", status=Source.Status.DISABLED)
.filter(
Q(metadata__hidden_from_source_list__isnull=True)
| Q(metadata__hidden_from_source_list=False)
)
.filter(Q(metadata__watchlist__isnull=True) | Q(metadata__watchlist=False))
.annotate(job_count=Count("job_aliases", distinct=True))
.order_by("name")
)
def _manual_import_context(request: HttpRequest, *, form: ManualImportForm, manual_result=None):
def _watchlist_queryset():
return (
Source.objects.filter(metadata__watchlist=True)
.filter(
Q(metadata__hidden_from_source_list__isnull=True)
| Q(metadata__hidden_from_source_list=False)
)
.order_by("name")
)
def _manual_import_context(
request: HttpRequest,
*,
form: ManualImportForm,
manual_result=None,
mailbox_form: MailboxConnectionForm | None = None,
editing_mailbox: MailboxConnection | None = None,
):
base_queryset = _source_list_queryset()
watchlist_queryset = _watchlist_queryset()
mailbox_connections = list(
request.user.mailbox_connections.filter(platform__in=PLATFORM_ALERTS)
)
active_search_profile = SearchProfile.objects.filter(user=request.user, is_active=True).first()
return {
"sources": base_queryset,
"active_source_count": Source.objects.filter(status=Source.Status.ACTIVE).count(),
"source_total_count": base_queryset.count(),
"review_pending_count": Source.objects.filter(policy=Source.Policy.REVIEW).count(),
"watchlist_sources": watchlist_queryset,
"watchlist_count": watchlist_queryset.count(),
"manual_import_form": form,
"manual_import_bookmarklet_endpoint": request.build_absolute_uri(
reverse("sources:manual_import")
),
"manual_import_paste_max_bytes": settings.MANUAL_IMPORT_PASTE_MAX_BYTES,
"manual_import_result": manual_result,
"platform_alert_ingress_ready": any(
connection.enabled for connection in mailbox_connections
),
"mailbox_connections": mailbox_connections,
"platform_alerts": PLATFORM_ALERTS.values(),
"mailbox_form": mailbox_form or MailboxConnectionForm(),
"editing_mailbox": editing_mailbox,
"mailbox_credential_key_ready": bool(settings.MAILBOX_CREDENTIAL_KEYS),
"active_search_profile": active_search_profile,
}
@@ -108,7 +157,7 @@ def _action_to_review(
def bulk_candidate_action(request):
action = request.POST.get("action")
source_ids = [int(raw_id) for raw_id in request.POST.getlist("source_ids") if raw_id.isdigit()]
sources = Source.objects.filter(id__in=source_ids).exclude(policy=Source.Policy.DENY)
sources = _source_list_queryset().filter(id__in=source_ids).exclude(policy=Source.Policy.DENY)
if not sources.exists():
messages.info(request, "Geen bruikbare bronregels geselecteerd.")
return redirect("sources:list")
@@ -148,6 +197,84 @@ def retry_source(request, pk: int):
return redirect("sources:list")
@login_required
@require_POST
def save_mailbox(request, pk: int | None = None):
instance = None
if pk is not None:
instance = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
form = MailboxConnectionForm(request.POST, instance=instance)
if form.is_valid():
try:
connection = save_mailbox_connection(
user=request.user,
cleaned_data=dict(form.cleaned_data),
instance=instance,
)
except MailboxCredentialError as exc:
form.add_error("password", str(exc))
else:
poll_mailbox.delay(connection.pk, force=True)
messages.success(
request,
f"{connection.get_platform_display()}-mailbox opgeslagen; "
"verbindingstest ingepland.",
)
return redirect("sources:list")
messages.error(request, "Controleer de mailboxinstellingen.")
return render(
request,
"sources/list.html",
_manual_import_context(
request,
form=ManualImportForm(),
mailbox_form=form,
editing_mailbox=instance,
),
)
@login_required
def edit_mailbox(request, pk: int):
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
if request.method == "POST":
return save_mailbox(request, pk=pk)
return render(
request,
"sources/list.html",
_manual_import_context(
request,
form=ManualImportForm(),
mailbox_form=MailboxConnectionForm(instance=connection),
editing_mailbox=connection,
),
)
@login_required
@require_POST
def sync_platform_alerts(request, pk: int):
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
if not connection.enabled:
messages.error(request, "Activeer deze mailbox voordat je synchroniseert.")
else:
poll_mailbox.delay(connection.pk, force=True)
messages.success(request, f"{connection.get_platform_display()}-mailimport ingepland.")
return redirect("sources:list")
@login_required
@require_POST
def toggle_mailbox(request, pk: int):
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
connection.enabled = not connection.enabled
connection.next_poll_at = timezone.now()
connection.save(update_fields=["enabled", "next_poll_at", "updated_at"])
state = "geactiveerd" if connection.enabled else "gepauzeerd"
messages.success(request, f"{connection.get_platform_display()}-mailbox {state}.")
return redirect("sources:list")
@login_required
@never_cache
def manual_import_view(request):