diff --git a/.env.example b/.env.example index bd56a71..5604a3a 100644 --- a/.env.example +++ b/.env.example @@ -30,15 +30,12 @@ SMTP_USER= SMTP_PASSWORD= SMTP_USE_TLS=1 -# Dedicated IMAP-mailbox. Geen gewone hoofdmailbox gebruiken. -IMAP_ENABLED=0 -IMAP_HOST= -IMAP_PORT=993 -IMAP_USER= -IMAP_PASSWORD= -IMAP_MAILBOX=INBOX -IMAP_USE_SSL=1 -IMAP_MARK_SEEN=0 +# Roteerbare Fernet-sleutels voor app-wachtwoorden; actieve sleutel eerst. +# Genereer met: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" +MAILBOX_CREDENTIAL_KEYS= +IMAP_CONNECT_TIMEOUT_SECONDS=15 +IMAP_MAX_MESSAGES_PER_POLL=200 +IMAP_MAX_MESSAGE_BYTES=1000000 # Fetcher FETCHER_USER_AGENT=VacatureRadar/0.1 (+local-personal-use; contact=local-admin) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ddb0d0..156976a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,138 @@ Alle betekenisvolle wijzigingen worden hier bijgehouden. Het project volgt voorlopig een pre-1.0 semantische versieaanpak. +## 0.2.10 — 2026-07-22 + +### Gewijzigd + +- De officiële Belgische GeoNames-postcodedataset 2026-04-03 is lokaal geladen onder CC BY 4.0; postcode 2400 is gecontroleerd als Mol met benaderend postcodecentrum 51.1919, 5.1166. +- Gemeente Mol, Ziekenhuis Geel, lokaal bestuur Geel, Stad Turnhout, Group Renotec, Ravago, Sanofi Geel en DAF Trucks Westerlo zijn als echte, gereviewde bronnen geactiveerd. +- Twaalf interessante regionale werkgevers zijn als afzonderlijke, niet-scanbare waaklijst toegevoegd, ook wanneer zij momenteel geen lokale vacature publiceren. +- De bronnenpagina toont 21 actieve bronnen, 12 waaklijstwerkgevers en de actieve GeoNames-afstandsstatus; waaklijstbronnen vallen buiten bulkacties en geplande scans. +- HTTP-, robots- en IMAP-verbindingen gebruiken expliciet de gepinde `certifi`-CA-bundel met verplichte hostnaam- en certificaatcontrole. Gemeente Mol en Ziekenhuis Geel werken daarmee zonder TLS-bypass. + +### Verificatie + +- De 8 nieuwe actieve bronnen leverden samen 53 kaarten per cyclus en voltooiden elk twee livecycli zonder parserwarnings; de tweede cycli maakten geen nieuwe dubbelen aan. +- GeoNames valideerde 2.781 unieke Belgische postcode-/gemeenterijen; bron- en afgeleide SHA-256-checksums zijn vastgelegd en het actieve profiel is opnieuw gescoord. +- 215 tests geslaagd, 2 optionele Playwrightvarianten overgeslagen en 84,42% branch-aware codedekking; Ruff, Django, migraties, taakledger en repositoryvalidatie zijn groen. +- De Codex-browsercontrole bevestigde 21 actieve bronnen, 12 waaklijstkaarten, 8 nieuwe actieve werkgevers, één `h1`, een lege console en geen horizontale overflow op desktop of 390 px. + +## 0.2.9 — 2026-07-22 + +### Gewijzigd + +- Het actieve zoekprofiel gebruikt postcode 2400 en een straal van 40 km; de bronnenpagina maakt ontbrekende geocodedata expliciet zonder een afstand te verzinnen. +- SCK CEN, Cipal Schaubroeck, VanRoey, NTX/Netropolix en Thomas More zijn als gereviewde werkgeversbronnen rond Mol toegevoegd. +- De nieuwe regionale adapter accepteert alleen vijf exacte HTTPS-routes en gebruikt host-, pad-, plaats- en waar nodig werkgever-GUID-filters; login-, apply- en kandidaatendpoints blijven uitgesloten. +- Gemeente Mol en Ziekenhuis Geel zijn onderzocht maar niet geactiveerd omdat hun careersites de TLS-certificaatcontrole niet doorstonden. + +### Verificatie + +- De vijf nieuwe bronnen voltooiden twee livecycli zonder parserwarnings; Cipal verwerkte de tweede cyclus correct als HTTP 304. De radar toont 13 actieve bronnen en 210 actieve vacature-aliassen. +- 206 tests geslaagd, 2 optionele Playwrightvarianten overgeslagen en 84,12% branch-aware codedekking. +- De Codex-browsercontrole bevestigde postcode 2400, 40 km, 13 bronkaarten, een lege console en geen horizontale overflow op desktop of 360 px. + +## 0.2.8 — 2026-07-22 + +### Gewijzigd + +- EuroBrussels en BrusselsJobs zijn uit de actuele platformkeuzes verwijderd; Careerjet, Randstad en Robert Half zijn toegevoegd met hun officiële vacaturealertingangen. +- Vijf gereviewde werkgeversbronnen uit de kernregio Hasselt–Genk zijn live toegevoegd: ACA Group, Xplore Group, Universiteit Hasselt, Hogeschool PXL en Ziekenhuis Oost-Limburg. +- Zeven eerder toegevoegde werkgevers buiten de regio zijn veilig uitgeschakeld en verborgen, met behoud van historische vacatures en bronprovenance. +- De regionale werkgeversadapter beperkt links tot vooraf gereviewde HTTPS-hosts en paden; vacaturedata blijft door bronbeleid, SSRF-validatie en sanitization gaan. + +### Verificatie + +- Alle vijf nieuwe werkgeversbronnen voltooiden twee geslaagde livecycli zonder parserwarnings; samen met VITO, Corda Campus en RPM Europe Logistics zijn 8 bronnen actief en 162 vacature-aliassen beschikbaar. +- 200 tests geslaagd, 2 optionele Playwrightvarianten overgeslagen en 83,90% branch-aware codedekking. +- De Codex-browsercontrole bevestigde 9 actuele platformkeuzes, 8 regionale bronkaarten, een lege console en geen horizontale overflow op desktop of 375 px. + +## 0.2.7 — 2026-07-22 + +### Gewijzigd + +- Platformmailboxen ondersteunen nu ook LinkedIn, ictjob.be, Jobat, StepStone, EuroBrussels en BrusselsJobs naast VDAB en Indeed. +- De bronnenpagina verwijst per jobboard naar de actuele officiële alertconfiguratie; elk platform behoudt een eigen mailbox, interval, status en synchronisatieactie. +- Platformgebonden mailimport accepteert uitsluitend gelabelde HTTPS-links op het gekozen jobboarddomein en verwerpt externe tracking-, account-, help-, privacy- en uitschrijflinks. +- Alle brede jobboards blijven op de directe-fetchdenylist; VacatureRadar logt alleen in op de mailbox en nooit op het jobboard. + +### Verificatie + +- 192 tests geslaagd, 2 optionele Playwrightvarianten overgeslagen en 83,61% branch-aware codedekking. +- De Codex-browsercontrole bevestigde acht platformkeuzes en alertkaarten, één `h1`, een lege console en geen horizontale overflow op desktop of 390 px. + +## 0.2.6 — 2026-07-22 + +### Gewijzigd + +- VDAB en Indeed kunnen elk aan een afzonderlijke Gmail-, Outlook- of aangepaste IMAP-mailbox worden gekoppeld vanuit de bronpagina. +- Iedere mailbox kiest een eigen interval van 5 minuten tot 6 uur en kan afzonderlijk worden gewijzigd, gepauzeerd of direct gesynchroniseerd. +- App-wachtwoorden worden met een aparte roteerbare MultiFernet-sleutel versleuteld; ciphertext, wachtwoorden en ruwe serverfouten worden niet in de UI, logs of taakresultaten getoond. +- De scheduler gebruikt per-mailboxleases en idempotentie per mailbox + Message-ID; een falende mailbox blokkeert andere platformmailboxen niet. +- Aangepaste IMAP-hosts worden op publieke DNS/IP-adressen gecontroleerd voordat een TLS-verbinding wordt gestart. + +### Verificatie + +- 178 tests geslaagd, 2 optionele Playwrightvarianten overgeslagen en 83,56% branch-aware codedekking. +- De Codex-browsercontrole bevestigde de platformmailboxformulieren en statussen, één `h1`, een lege console en geen horizontale overflow op desktop of 390 px. + +## 0.2.5 — 2026-07-22 + +### Gewijzigd + +- De officiële VDAB-jobalertingang draait standaard eenmaal per dag om 06:15 en is via omgevingsvariabelen configureerbaar. +- VDAB-mails krijgen herkenbare provenance en hun vacatures stromen idempotent door de bestaande normalisatie-, dedupe- en rankingpipeline zonder automatische platformfetch. +- Mailboxbatches zijn begrensd op aantal en berichtgrootte; een fout of oversized bericht stopt andere mails niet. +- De bronpagina toont planning, configuratiestatus, laatste VDAB-import, een handmatige inplanknop en het onderscheid met de officiële VDAB Vacature API waarvoor partnershipgoedkeuring nodig is. + +### Verificatie + +- 164 tests geslaagd, 2 optionele Playwrightvarianten overgeslagen en 83,55% branch-aware codedekking. +- De Codex-browsercontrole bevestigde de dagelijkse VDAB-sectie, officiële links, één `h1`, een lege console en geen horizontale overflow op desktop of 390 px. + +## 0.2.4 — 2026-07-22 + +### Gewijzigd + +- De livecatalogus groeit van twee naar tien gereviewde bronnen, met lokale Corda Campus-vacatures en werkgeversfeeds uit Genk, Brussel en België-breed. +- Gedeelde ATS-domeinen ondersteunen nu meerdere werkgevers op basis van hun exacte publieke feed-URL zonder records te overschrijven. +- De SmartRecruiters-adapter ondersteunt het gedocumenteerde publieke `content`-formaat, stabiele vacaturelinks, locaties, contracttype en remote-status. +- Corda Campus heeft een begrensde listingparser die werkgever, titel en regio bewaart en off-domain vacaturelinks weigert. +- De bronpagina legt VDAB- en Indeed-ingress via officiële vacaturealerts uit; directe platformcrawling blijft geblokkeerd. + +### Verificatie + +- Acht nieuwe bronnen leverden 84 actuele bronrecords in twee geslaagde livecycli zonder parserwarnings; lokaal staan 10 bronnen actief met 145 actieve vacatures. +- 159 tests geslaagd, 2 optionele Playwrightvarianten overgeslagen en 82,91% branch-aware codedekking. +- De in-app-browsercontrole vond 10 bronkaarten, 2 platformalertlinks, geen horizontale overflow en geen consolefouten. + +## 0.2.3 — 2026-07-21 + +### Gewijzigd + +- Thuislocatie vraagt alleen nog een Belgische postcode; afgeleide gemeente en coördinaten zijn geen handmatige velden meer. +- Functietitels, skills, contractvormen, werkmodellen en regio's gebruiken toegankelijke checkboxgroepen met behoud van bestaande waarden. +- VITO en Deliverect zijn als gereviewde, functionele werkgeverfeeds toegevoegd en na twee geslaagde cycli geactiveerd. +- Recruitee `offers` en de publieke Lever Postings API worden volledig verwerkt, inclusief werkgeveridentiteit, directe-bronmarkering en actuele inventaristelling na HTTP 304. +- Robotsredirects worden opnieuw op publieke adressen gevalideerd; de oude fixturebron verdwijnt uit het normale bronoverzicht. + +## 0.2.2 — 2026-07-21 + +### Gewijzigd + +- Tweede Stitch-pass (`VR-119`) maakt de aangeleverde schermcompositie leidend in plaats van alleen de kleuren en componenttokens. +- De utilitybar bevat nu de brede vacaturezoeker en directe zoekprofielactie uit Stitch. +- Dashboard en vacatureoverzicht gebruiken een dichte, enkelkoloms radarfeed met echte scores, datakwaliteit en acties; één resultaat vult niet langer slechts een halve lege grid. +- Vacaturedetail heeft een echte intelligence-strip; pipeline, profieleditor, bronnenbeheer en systeemstatus volgen de bijbehorende Stitch-hiërarchie veel directer. +- Handmatige import is een compacte disclosure zodat brongezondheid opnieuw de primaire compositie vormt. +- Alle zichtbare profieleditorlabels zijn Nederlandstalig en de lokale favicon voorkomt een ontbrekend browserasset. + +### Verificatie + +- 146 tests geslaagd, 2 optionele lokale Playwrightvarianten overgeslagen en 82,02% branch-aware codedekking. +- Browsermatrix geslaagd op 1440×900, 1280×800, 1024×768, 768×1024, 390×844 en 360×800, inclusief dark/light, mobiele drawer, één `h1` en geen pagina-overflow. + ## 0.2.1 — 2026-07-21 ### Toegevoegd diff --git a/apps/core/context_processors.py b/apps/core/context_processors.py index f0ff6ce..2f65284 100644 --- a/apps/core/context_processors.py +++ b/apps/core/context_processors.py @@ -4,9 +4,18 @@ from typing import Any from django.http import HttpRequest +from apps.profiles.models import SearchProfile + def navigation_context(request: HttpRequest) -> dict[str, Any]: - return { + context = { "app_name": "VacatureRadar", - "app_version": "0.2.1", + "app_version": "0.2.2", } + if request.user.is_authenticated: + context["navigation_profile"] = ( + SearchProfile.objects.filter(user=request.user, is_active=True) + .only("id", "name") + .first() + ) + return context diff --git a/apps/core/views.py b/apps/core/views.py index 201781b..b3d5275 100644 --- a/apps/core/views.py +++ b/apps/core/views.py @@ -102,6 +102,10 @@ class TodayView(LoginRequiredMixin, TemplateView): context.update( { "score_cards": cards, + "priority_applications": Application.objects.filter(user=self.request.user) + .exclude(status__in=[Application.Status.REJECTED, Application.Status.WITHDRAWN]) + .select_related("job", "job__employer") + .order_by("follow_up_date", "-updated_at")[:3], "active_jobs": JobPosting.objects.filter(status=JobPosting.Status.ACTIVE).count(), "new_today": JobPosting.objects.filter( first_seen__date=timezone.localdate() @@ -109,7 +113,9 @@ class TodayView(LoginRequiredMixin, TemplateView): "applications_open": Application.objects.exclude( status__in=[Application.Status.REJECTED, Application.Status.WITHDRAWN] ).count(), - "source_counts": Source.objects.aggregate( + "source_counts": Source.objects.exclude( + domain="jobs.example.org", status=Source.Status.DISABLED + ).aggregate( total=Count("id"), unhealthy=Count("id", filter=~Q(status=Source.Status.ACTIVE)), ), diff --git a/apps/jobs/services/pipeline.py b/apps/jobs/services/pipeline.py index f8a0820..0321681 100644 --- a/apps/jobs/services/pipeline.py +++ b/apps/jobs/services/pipeline.py @@ -8,6 +8,7 @@ from django.utils import timezone from apps.jobs.models import Employer, FieldProvenance, JobPosting, JobSourceAlias, JobVersion from apps.profiles.models import SearchProfile +from apps.sources.adapters.base import FieldEvidence from apps.sources.adapters.registry import registry from apps.sources.models import RawDocument, Source from apps.sources.services.policy import is_denied_domain @@ -89,7 +90,10 @@ def _source_is_direct(source: Source | None, draft: CanonicalJobDraft) -> bool: host = (urlsplit(draft.canonical_url).hostname or "").lower() if is_denied_domain(host): return False - return bool(source and source.source_type == Source.Type.EMPLOYER) + if not source: + return False + metadata = source.metadata if isinstance(source.metadata, dict) else {} + return source.source_type == Source.Type.EMPLOYER or metadata.get("direct_employer") is True def _alias_payload( @@ -314,6 +318,22 @@ def process_raw_document(document: RawDocument) -> dict[str, int | str | list[st ) created = updated = duplicates = 0 for extracted in result.jobs: + source_metadata = ( + document.source.metadata + if document.source and isinstance(document.source.metadata, dict) + else {} + ) + configured_employer = str(source_metadata.get("employer_name") or "").strip() + if configured_employer: + extracted.employer_name = configured_employer + extracted.evidence.append( + FieldEvidence( + "employer_name", + "reviewed-source-config", + 0.99, + configured_employer, + ) + ) draft = normalize_extracted_job(extracted) _, decision, was_created = persist_draft( draft, diff --git a/apps/jobs/views.py b/apps/jobs/views.py index 73611cd..2fe2670 100644 --- a/apps/jobs/views.py +++ b/apps/jobs/views.py @@ -3,7 +3,7 @@ from __future__ import annotations from django.contrib import messages from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin -from django.db.models import Q +from django.db.models import CharField, DecimalField, OuterRef, Q, Subquery from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404, redirect from django.urls import reverse @@ -31,6 +31,24 @@ class JobListView(LoginRequiredMixin, ListView): def get_queryset(self): queryset = JobPosting.objects.select_related("employer").all() + profile = SearchProfile.objects.filter(user=self.request.user, is_active=True).first() + if profile: + latest_score = ScoreRun.objects.filter(job=OuterRef("pk"), profile=profile).order_by( + "-created_at" + ) + queryset = queryset.annotate( + match_score=Subquery( + latest_score.values("score")[:1], + output_field=DecimalField(max_digits=5, decimal_places=2), + ), + match_confidence=Subquery( + latest_score.values("confidence")[:1], + output_field=DecimalField(max_digits=4, decimal_places=3), + ), + match_recommendation=Subquery( + latest_score.values("recommendation")[:1], output_field=CharField() + ), + ) query = self.request.GET.get("q", "").strip() status = self.request.GET.get("status", "active").strip() workplace = self.request.GET.get("workplace", "").strip() diff --git a/apps/profiles/forms.py b/apps/profiles/forms.py index 783a54a..1dcd167 100644 --- a/apps/profiles/forms.py +++ b/apps/profiles/forms.py @@ -1,30 +1,141 @@ from __future__ import annotations +import re + from django import forms from .models import SearchProfile +TITLE_CHOICES = ( + ("system engineer", "System engineer"), + ("infrastructure engineer", "Infrastructure engineer"), + ("cloud engineer", "Cloud engineer"), + ("devops engineer", "DevOps engineer"), + ("network engineer", "Network engineer"), + ("security engineer", "Security engineer"), + ("workplace engineer", "Workplace engineer"), + ("support engineer", "Support engineer"), + ("software engineer", "Software engineer"), + ("data engineer", "Data engineer"), + ("solution architect", "Solution architect"), + ("project manager", "Projectmanager"), +) + +EXCLUDED_TITLE_CHOICES = ( + ("sales", "Sales"), + ("recruiter", "Recruiter"), + ("account manager", "Accountmanager"), + ("callcenter", "Callcenter"), + ("stage", "Stage"), + ("student", "Studentenjob"), +) + +SKILL_CHOICES = ( + ("azure", "Microsoft Azure"), + ("aws", "AWS"), + ("microsoft 365", "Microsoft 365"), + ("active directory", "Active Directory"), + ("entra id", "Entra ID"), + ("intune", "Microsoft Intune"), + ("linux", "Linux"), + ("windows server", "Windows Server"), + ("networking", "Netwerken"), + ("security", "Security"), + ("python", "Python"), + ("powershell", "PowerShell"), + ("terraform", "Terraform"), + ("docker", "Docker"), + ("kubernetes", "Kubernetes"), + ("ci/cd", "CI/CD"), +) + +EXCLUDED_SKILL_CHOICES = ( + ("cold calling", "Cold calling"), + ("door to door", "Deur-aan-deurverkoop"), + ("commission only", "Alleen commissieloon"), + ("night shift", "Nachtwerk"), +) + +EMPLOYMENT_CHOICES = ( + ("full_time", "Voltijds"), + ("part_time", "Deeltijds"), + ("permanent", "Vast contract"), + ("fixed_term", "Tijdelijk contract"), + ("freelance", "Freelance"), + ("internship", "Stage"), +) + +WORKPLACE_CHOICES = ( + ("hybrid", "Hybride"), + ("remote", "Volledig op afstand"), + ("on_site", "Op locatie"), +) + +REGION_CHOICES = ( + ("Antwerpen", "Antwerpen"), + ("Brussels Hoofdstedelijk Gewest", "Brussel"), + ("Henegouwen", "Henegouwen"), + ("Limburg", "Limburg"), + ("Luik", "Luik"), + ("Luxemburg", "Luxemburg"), + ("Namen", "Namen"), + ("Oost-Vlaanderen", "Oost-Vlaanderen"), + ("Vlaams-Brabant", "Vlaams-Brabant"), + ("Waals-Brabant", "Waals-Brabant"), + ("West-Vlaanderen", "West-Vlaanderen"), +) + class SearchProfileForm(forms.ModelForm): - desired_titles_text = forms.CharField( + desired_titles = forms.MultipleChoiceField( label="Gewenste functietitels", required=False, - widget=forms.Textarea(attrs={"rows": 4, "placeholder": "Eén titel per regel"}), + choices=TITLE_CHOICES, + widget=forms.CheckboxSelectMultiple, + help_text="Selecteer alle rollen die bij je zoekrichting passen.", ) - excluded_titles_text = forms.CharField( - label="Uitgesloten titelwoorden", + excluded_titles = forms.MultipleChoiceField( + label="Uitgesloten functietitels", required=False, - widget=forms.Textarea(attrs={"rows": 3, "placeholder": "Eén term per regel"}), + choices=EXCLUDED_TITLE_CHOICES, + widget=forms.CheckboxSelectMultiple, + help_text="Vacatures met deze titelwoorden worden hard uitgesloten.", ) - desired_skills_text = forms.CharField( + desired_skills = forms.MultipleChoiceField( label="Gewenste skills", required=False, - widget=forms.Textarea(attrs={"rows": 4, "placeholder": "Eén skill per regel"}), + choices=SKILL_CHOICES, + widget=forms.CheckboxSelectMultiple, ) - excluded_skills_text = forms.CharField( - label="Uitgesloten skills", + excluded_skills = forms.MultipleChoiceField( + label="Uitgesloten kenmerken", required=False, - widget=forms.Textarea(attrs={"rows": 3, "placeholder": "Eén skill per regel"}), + choices=EXCLUDED_SKILL_CHOICES, + widget=forms.CheckboxSelectMultiple, + ) + allowed_employment_types = forms.MultipleChoiceField( + label="Toegestane contractvormen", + required=False, + choices=EMPLOYMENT_CHOICES, + widget=forms.CheckboxSelectMultiple, + ) + preferred_workplace = forms.MultipleChoiceField( + label="Voorkeurswerkmodel", + required=False, + choices=WORKPLACE_CHOICES, + widget=forms.CheckboxSelectMultiple, + ) + preferred_regions = forms.MultipleChoiceField( + label="Voorkeursregio's", + required=False, + choices=REGION_CHOICES, + widget=forms.CheckboxSelectMultiple, + ) + excluded_regions = forms.MultipleChoiceField( + label="Uitgesloten regio's", + required=False, + choices=REGION_CHOICES, + widget=forms.CheckboxSelectMultiple, ) class Meta: @@ -33,10 +144,11 @@ class SearchProfileForm(forms.ModelForm): "name", "is_active", "home_postal_code", - "home_municipality", - "home_latitude", - "home_longitude", "max_distance_km", + "desired_titles", + "excluded_titles", + "desired_skills", + "excluded_skills", "allowed_employment_types", "preferred_workplace", "preferred_regions", @@ -46,35 +158,81 @@ class SearchProfileForm(forms.ModelForm): "digest_time", "learning_enabled", ] + labels = { + "name": "Naam", + "is_active": "Actief profiel", + "home_postal_code": "Thuispostcode", + "max_distance_km": "Maximale afstand (km)", + "recommendation_threshold": "Aanbevelingsdrempel", + "top_match_threshold": "Topmatchdrempel", + "digest_time": "Tijdstip dagelijkse samenvatting", + "learning_enabled": "Gecontroleerd leren", + } + help_texts = { + "home_postal_code": ( + "Vier cijfers volstaan. Gemeente en coördinaten worden veilig afgeleid wanneer " + "lokale geodata beschikbaar is." + ), + "learning_enabled": "Past alleen zachte voorkeuren aan; harde regels blijven vast.", + } widgets = { - "allowed_employment_types": forms.TextInput( - attrs={"placeholder": '["full_time", "part_time"]'} + "home_postal_code": forms.TextInput( + attrs={"inputmode": "numeric", "pattern": "[0-9]{4}", "maxlength": 4} ), - "preferred_workplace": forms.TextInput(attrs={"placeholder": '["hybrid", "on_site"]'}), - "preferred_regions": forms.TextInput(attrs={"placeholder": '["Limburg"]'}), - "excluded_regions": forms.TextInput(attrs={"placeholder": '["Brussel"]'}), "digest_time": forms.TimeInput(attrs={"type": "time"}), } def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if self.instance and self.instance.pk: - self.fields["desired_titles_text"].initial = "\n".join(self.instance.desired_titles) - self.fields["excluded_titles_text"].initial = "\n".join(self.instance.excluded_titles) - self.fields["desired_skills_text"].initial = "\n".join(self.instance.desired_skills) - self.fields["excluded_skills_text"].initial = "\n".join(self.instance.excluded_skills) + self.fields["home_postal_code"].widget.attrs["maxlength"] = 4 + if not self.instance or not self.instance.pk: + return + for field_name in ( + "desired_titles", + "excluded_titles", + "desired_skills", + "excluded_skills", + "allowed_employment_types", + "preferred_workplace", + "preferred_regions", + "excluded_regions", + ): + self._include_existing_choices(field_name) - @staticmethod - def _lines(value: str) -> list[str]: - return [line.strip() for line in value.splitlines() if line.strip()] + def _include_existing_choices(self, field_name: str) -> None: + field = self.fields[field_name] + stored = getattr(self.instance, field_name, []) + existing_by_casefold = { + str(value).casefold(): str(value) for value, _label in field.choices + } + extra = [ + (str(value), str(value)) + for value in stored + if str(value).casefold() not in existing_by_casefold + ] + field.choices = [*field.choices, *extra] + field.initial = [ + existing_by_casefold.get(str(value).casefold(), str(value)) for value in stored + ] - def save(self, commit=True): - instance = super().save(commit=False) - instance.desired_titles = self._lines(self.cleaned_data["desired_titles_text"]) - instance.excluded_titles = self._lines(self.cleaned_data["excluded_titles_text"]) - instance.desired_skills = self._lines(self.cleaned_data["desired_skills_text"]) - instance.excluded_skills = self._lines(self.cleaned_data["excluded_skills_text"]) - if commit: - instance.save() - self.save_m2m() - return instance + def clean_home_postal_code(self) -> str: + value = str(self.cleaned_data.get("home_postal_code") or "").strip() + if value and not re.fullmatch(r"\d{4}", value): + raise forms.ValidationError("Gebruik een Belgische postcode van vier cijfers.") + return value + + def clean(self): + cleaned = super().clean() + overlap_checks = ( + ("desired_titles", "excluded_titles", "functietitels"), + ("desired_skills", "excluded_skills", "skills"), + ("preferred_regions", "excluded_regions", "regio's"), + ) + for desired_field, excluded_field, label in overlap_checks: + overlap = set(cleaned.get(desired_field) or []) & set(cleaned.get(excluded_field) or []) + if overlap: + self.add_error( + excluded_field, + f"Dezelfde {label} kunnen niet tegelijk gewenst en uitgesloten zijn.", + ) + return cleaned diff --git a/apps/profiles/services.py b/apps/profiles/services.py index e866a40..bc34891 100644 --- a/apps/profiles/services.py +++ b/apps/profiles/services.py @@ -2,9 +2,33 @@ from __future__ import annotations from django.db import transaction +from apps.jobs.services.geocoding import LocationMatchResult, resolve_cached_location + from .models import ProfileRevision, SearchProfile +@transaction.atomic +def resolve_profile_home_location(profile: SearchProfile) -> LocationMatchResult: + """Resolve a user-entered postal code without exposing coordinate fields in the UI.""" + query = profile.home_postal_code.strip() + result = ( + resolve_cached_location(query) if query else LocationMatchResult(query=query, location=None) + ) + location = result.location + profile.home_municipality = location.municipality or "" if location else "" + profile.home_latitude = location.point.latitude if location and location.point else None + profile.home_longitude = location.point.longitude if location and location.point else None + profile.save( + update_fields=[ + "home_municipality", + "home_latitude", + "home_longitude", + "updated_at", + ] + ) + return result + + @transaction.atomic def save_profile_revision(profile: SearchProfile, *, reason: str) -> ProfileRevision: latest = profile.revisions.order_by("-version").first() diff --git a/apps/profiles/views.py b/apps/profiles/views.py index 5508432..e70a9b8 100644 --- a/apps/profiles/views.py +++ b/apps/profiles/views.py @@ -8,7 +8,7 @@ from django.views.generic import ListView, UpdateView from .forms import SearchProfileForm from .models import SearchProfile -from .services import save_profile_revision +from .services import resolve_profile_home_location, save_profile_revision class ProfileListView(LoginRequiredMixin, ListView): @@ -31,10 +31,24 @@ class ProfileUpdateView(LoginRequiredMixin, UpdateView): def form_valid(self, form): response = super().form_valid(form) + location_result = resolve_profile_home_location(self.object) if self.object.is_active: self.object.activate() save_profile_revision(self.object, reason="user_edit") - messages.success(self.request, "Zoekprofiel opgeslagen.") + if self.object.home_postal_code and location_result.ambiguous: + messages.warning( + self.request, + "Zoekprofiel opgeslagen. Deze postcode omvat meerdere gemeenten; afstand blijft " + "onbekend tot de lokale geodata eenduidig is.", + ) + elif self.object.home_postal_code and location_result.location is None: + messages.warning( + self.request, + "Zoekprofiel opgeslagen. De postcode is bewaard; lokale geodata ontbreekt nog " + "voor een afstandsberekening.", + ) + else: + messages.success(self.request, "Zoekprofiel opgeslagen.") return response diff --git a/apps/sources/adapters/__init__.py b/apps/sources/adapters/__init__.py index ce189e1..d6c7c88 100644 --- a/apps/sources/adapters/__init__.py +++ b/apps/sources/adapters/__init__.py @@ -8,6 +8,9 @@ from .ats import ( from .base import ExtractedJob, ExtractionResult from .generic_html import GenericHtmlAdapter from .jsonld import JsonLdJobPostingAdapter +from .kempen import KempenEmployerAdapter +from .mol_region import MolRegionEmployerAdapter +from .regional import LocalEmployerListingAdapter from .rss import RssAdapter __all__ = [ @@ -16,7 +19,10 @@ __all__ = [ "GenericHtmlAdapter", "GreenhouseAdapter", "JsonLdJobPostingAdapter", + "KempenEmployerAdapter", "LeverAdapter", + "LocalEmployerListingAdapter", + "MolRegionEmployerAdapter", "RecruiteeAdapter", "RssAdapter", "SmartRecruitersAdapter", diff --git a/apps/sources/adapters/ats.py b/apps/sources/adapters/ats.py index e932dfc..3eef25b 100644 --- a/apps/sources/adapters/ats.py +++ b/apps/sources/adapters/ats.py @@ -169,16 +169,20 @@ class _AtsAdapter(ABC): def _extract_location(self, record: dict[str, object]) -> tuple[str, str, str, str]: location_raw = _first_text( record, - "location", + "location.city", + "location.name", + "location.address", + "location.raw", + "categories.location", "city", "cityName", "place", "office", - "location.address", "address", "data.location", "officeLocation", "locationName", + "location", ) if not location_raw: location_raw = _first_text(record, "data.city", "data.location") @@ -223,6 +227,13 @@ class _AtsAdapter(ABC): "remote_type", "job_type", ).lower() + if not workplace_raw: + if record.get("hybrid") is True: + return "hybrid" + if record.get("remote") is True: + return "remote" + if record.get("on_site") is True: + return "on_site" if "remote" in workplace_raw: return "remote" if "hybrid" not in workplace_raw else "hybrid" if "hybrid" in workplace_raw: @@ -256,18 +267,21 @@ class _AtsAdapter(ABC): "url", "jobUrl", "job_url", + "hostedUrl", "applyUrl", "link", "absoluteUrl", + "careers_url", "data.url", ) employer_name = _first_text( record, + "company_name", + "company.name", "company", "employer", "organization", "organizationName", - "company.name", "department", "hiringOrganization", ) @@ -290,6 +304,7 @@ class _AtsAdapter(ABC): "published_at", "created_at", "jobCreated", + "releasedDate", ) valid_through = _first_text( record, @@ -308,6 +323,9 @@ class _AtsAdapter(ABC): "jobType", "type", "data.employmentType", + "employment_type_code", + "categories.commitment", + "typeOfEmployment.label", ) ) workplace_type = self._extract_workplace(record) @@ -428,13 +446,20 @@ class GreenhouseAdapter(_AtsAdapter): class LeverAdapter(_AtsAdapter): parser_key = "ats-lever" - source_hosts = ("jobs.lever.co",) + source_hosts = ( + "jobs.lever.co", + "jobs.eu.lever.co", + "api.lever.co", + "api.eu.lever.co", + ) support_markers = ("lever", "requisition", "posting") listing_paths = (("data",), ("jobs",), ("results",)) detail_paths = (("data",), ("job",), ("position",), ("result",)) closed_statuses = CLOSED_STATUSES | {"archived", "deleted"} def _extract_records(self, payload): + if isinstance(payload, list): + return [item for item in payload if isinstance(item, dict)] for path in self.listing_paths: value = payload for key in path: @@ -466,7 +491,7 @@ class RecruiteeAdapter(_AtsAdapter): parser_key = "ats-recruitee" source_hosts = ("recruitee.com",) support_markers = ("recruitee", "career", "vacancy") - listing_paths = (("jobs",), ("data", "jobs"), ("vacancies",)) + listing_paths = (("jobs",), ("offers",), ("data", "jobs"), ("vacancies",)) detail_paths = (("job",), ("data", "job"), ("vacancy",), ("result",)) closed_statuses = CLOSED_STATUSES | {"hidden", "paused"} @@ -494,12 +519,26 @@ class RecruiteeAdapter(_AtsAdapter): class SmartRecruitersAdapter(_AtsAdapter): parser_key = "ats-smartrecruiters" + parser_version = "1.1.0" source_hosts = ("smartrecruiters.com",) support_markers = ("smartrecruiters", "smart recruiter") - listing_paths = (("jobs",), ("data", "jobs"), ("results",)) + listing_paths = (("jobs",), ("data", "jobs"), ("results",), ("content",)) detail_paths = (("job",), ("data", "job"), ("posting",), ("result",)) closed_statuses = CLOSED_STATUSES | {"unpublished", "expired"} + def _to_job(self, record: dict[str, object], base_url: str) -> ExtractedJob: + prepared = dict(record) + if not _first_text(prepared, "url", "jobUrl", "hostedUrl", "applyUrl", "link"): + company_identifier = _first_text(prepared, "company.identifier") + posting_id = _first_text(prepared, "id", "uuid") + if company_identifier and posting_id: + prepared["url"] = ( + f"https://jobs.smartrecruiters.com/{company_identifier}/{posting_id}" + ) + if _find_nested(prepared, "location.remote") is True: + prepared["remote"] = True + return super()._to_job(prepared, base_url) + def _extract_records(self, payload): for path in self.listing_paths: value = payload diff --git a/apps/sources/adapters/email_alert.py b/apps/sources/adapters/email_alert.py index 6cecc91..eeb09f5 100644 --- a/apps/sources/adapters/email_alert.py +++ b/apps/sources/adapters/email_alert.py @@ -9,17 +9,58 @@ from urllib.parse import urljoin, urlsplit from bs4 import BeautifulSoup +from apps.sources.platforms import PLATFORM_ALERTS from apps.sources.services.canonicalize import canonicalize_url from .base import ExtractedJob, ExtractionResult, FieldEvidence URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I) -SKIP_TEXT = re.compile(r"unsubscribe|afmelden|uitschrijven|privacy|view in browser", re.I) +SKIP_TEXT = re.compile( + r"unsubscribe|afmelden|uitschrijven|privacy|view in browser|bekijk online|" + r"account|aanmelden|inloggen|login|voorkeuren|preferences|voorwaarden|terms|" + r"contact|help|hulp|over ons|about us", + re.I, +) +SKIP_PATH = re.compile( + r"/(?:unsubscribe|uitschrijven|afmelden|privacy|account|login|signin|preferences|" + r"settings|terms|legal|help|contact)(?:/|$)", + re.I, +) + + +def _matches_domain(hostname: str, domains: tuple[str, ...]) -> bool: + return any(hostname == domain or hostname.endswith(f".{domain}") for domain in domains) class EmailAlertAdapter: parser_key = "email-alert" - parser_version = "1.0.0" + parser_version = "1.1.0" + + @staticmethod + def _provider_for_candidates(candidates: list[tuple[str, str]]) -> str: + hostnames = { + (urlsplit(href).hostname or "").lower() + for _, href in candidates + if href.lower().startswith(("http://", "https://")) + } + for provider, alert in PLATFORM_ALERTS.items(): + if any(_matches_domain(hostname, alert.domains) for hostname in hostnames): + return provider + return "other" + + @staticmethod + def _is_expected_platform_link(*, label: str, href: str, provider: str) -> bool: + alert = PLATFORM_ALERTS.get(provider) + if alert is None or len(label.strip()) < 4: + return False + parsed = urlsplit(href) + hostname = (parsed.hostname or "").lower() + return ( + parsed.scheme.lower() == "https" + and _matches_domain(hostname, alert.domains) + and not SKIP_TEXT.search(label) + and not SKIP_PATH.search(parsed.path) + ) @staticmethod def _decode_parts(message: Message) -> tuple[str, str]: @@ -41,7 +82,9 @@ class EmailAlertAdapter: html_parts.append(str(payload)) return "\n".join(plain_parts), "\n".join(html_parts) - def extract_message(self, raw_message: bytes) -> ExtractionResult: + def extract_message( + self, raw_message: bytes, *, expected_provider: str | None = None + ) -> ExtractionResult: message = BytesParser(policy=policy.default).parsebytes(raw_message) plain, html_body = self._decode_parts(message) candidates: list[tuple[str, str]] = [] @@ -61,6 +104,18 @@ class EmailAlertAdapter: continue candidates.append(("", clean_href)) + alert_provider = expected_provider or self._provider_for_candidates(candidates) + if expected_provider: + candidates = [ + (label, href) + for label, href in candidates + if self._is_expected_platform_link( + label=label, + href=href, + provider=expected_provider, + ) + ] + jobs: list[ExtractedJob] = [] seen: set[str] = set() subject = str(message.get("subject") or "Vacature uit e-mail").strip() @@ -84,6 +139,7 @@ class EmailAlertAdapter: "email_subject": subject, "email_sender": str(message.get("from") or ""), "target_domain": hostname, + "alert_provider": alert_provider, }, evidence=[FieldEvidence("url", "email-anchor", 0.75, label[:240])], ) diff --git a/apps/sources/adapters/kempen.py b/apps/sources/adapters/kempen.py new file mode 100644 index 0000000..a16aa28 --- /dev/null +++ b/apps/sources/adapters/kempen.py @@ -0,0 +1,290 @@ +from __future__ import annotations + +import re +from urllib.parse import parse_qs, urljoin, urlsplit + +from bs4 import BeautifulSoup, Tag + +from .base import ExtractedJob, ExtractionResult, FieldEvidence + + +class KempenEmployerAdapter: + """Extract public employer lists around Mol from exact reviewed routes.""" + + parser_key = "regional-kempen-employers" + parser_version = "1.0.0" + supported_routes = { + "ziekenhuisgeel.careersite.be": {"/nl/vacatures"}, + "geel.hro.be": {"/"}, + "jobs.turnhout.be": {"/"}, + "jobs.renotec.be": {"/nl/alle-jobs"}, + "ravago.softgarden.io": {"/en/vacancies"}, + "jobs.sanofi.com": {"/en/belgium"}, + "www.daf.com": {"/nl-nl/werken-bij-daf/vacatures"}, + } + + def _route(self, url: str) -> tuple[str, str] | None: + parts = urlsplit(url) + host = (parts.hostname or "").lower() + path = parts.path.rstrip("/") or "/" + if parts.scheme != "https" or path not in self.supported_routes.get(host, set()): + return None + return host, path + + def _supports_url(self, url: str) -> bool: + return self._route(url) is not None + + @staticmethod + def _same_host_url(base_url: str, href: str) -> str: + job_url = urljoin(base_url, href) + parts = urlsplit(job_url) + if parts.scheme != "https": + return "" + if (parts.hostname or "").lower() != (urlsplit(base_url).hostname or "").lower(): + return "" + return job_url + + @staticmethod + def _job( + *, + url: str, + title: str, + employer: str, + location: str, + postal_code: str, + description: str, + external_id: str, + evidence_source: str, + ) -> ExtractedJob: + return ExtractedJob( + url=url, + title=title, + employer_name=employer, + external_id=external_id, + location_text=location, + postal_code=postal_code, + region="Antwerpen", + country="BE", + description_text=description, + raw={"regional_listing": evidence_source, "region_center": "2400 Mol"}, + evidence=[ + FieldEvidence("title", evidence_source, 0.94, title[:240]), + FieldEvidence("employer_name", "reviewed-source", 0.97, employer), + FieldEvidence("location_text", "kempen-location-marker", 0.9, location), + ], + ) + + def _ziekenhuis_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for card in soup.select(".vacature-tegel"): + link = card.select_one("a.vacature-tegel__link[href]") + title_node = card.select_one(".vacature-tegel__titel") + if not isinstance(link, Tag) or not isinstance(title_node, Tag): + continue + job_url = self._same_host_url(url, str(link.get("href") or "")) + match = re.fullmatch(r"/nl/vacature/(\d+)/[^/]+", urlsplit(job_url).path) + title = title_node.get_text(" ", strip=True) + if not title or not match: + continue + jobs.append( + self._job( + url=job_url, + title=title, + employer="Ziekenhuis Geel", + location="Geel, Antwerpen", + postal_code="2440", + description=card.get_text(" ", strip=True), + external_id=match.group(1), + evidence_source="ziekenhuis-geel-card", + ) + ) + return jobs + + def _stad_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for card in soup.select(".vacatureKader[data-href]"): + href = str(card.get("data-href") or "") + match = re.fullmatch(r"vacature\.php\?id=(\d+)", href) + title_node = card.find("h4") + title = title_node.get_text(" ", strip=True) if title_node else "" + if not title or not match: + continue + jobs.append( + self._job( + url=self._same_host_url(url, href), + title=title, + employer="Lokaal bestuur Geel", + location="Geel, Antwerpen", + postal_code="2440", + description=card.get_text(" ", strip=True), + external_id=match.group(1), + evidence_source="stad-geel-hro-card", + ) + ) + return jobs + + def _stad_turnhout(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for link in soup.find_all("a", href=True): + href = str(link.get("href") or "") + match = re.fullmatch(r"vacature\.php\?id=(\d+)", href) + title_node = link.select_one(".block-update__body__title__inner") + title = title_node.get_text(" ", strip=True) if title_node else "" + if not title or not match: + continue + jobs.append( + self._job( + url=self._same_host_url(url, href), + title=title, + employer="Stad Turnhout", + location="Turnhout, Antwerpen", + postal_code="2300", + description=link.get_text(" ", strip=True), + external_id=match.group(1), + evidence_source="stad-turnhout-hro-card", + ) + ) + return jobs + + def _renotec(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for card in soup.select(".s-tile.s-card"): + card_text = card.get_text(" ", strip=True) + if "Geel" not in card_text: + continue + title_node = card.find("h3") + link = card.find("a", href=True) + title = title_node.get_text(" ", strip=True) if title_node else "" + job_url = self._same_host_url(url, str(link.get("href") or "")) if link else "" + parts = urlsplit(job_url) + query = parse_qs(parts.query) + external_id = (query.get("id") or [""])[0] + if parts.path != "/nl/detail/" or not external_id.isdigit() or not title: + continue + jobs.append( + self._job( + url=job_url, + title=title, + employer="Group Renotec", + location="Geel, Antwerpen (mogelijk meerdere werfregio's)", + postal_code="2440", + description=card_text, + external_id=external_id, + evidence_source="renotec-geel-card", + ) + ) + return jobs + + def _ravago(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for card in soup.select(".matchElement"): + locations = { + node.get_text(" ", strip=True) for node in card.select(".location-view-item") + } + local_places = locations.intersection({"Arendonk", "Olen"}) + if not local_places: + continue + link = card.find("a", href=True) + title = link.get_text(" ", strip=True) if link else "" + job_url = self._same_host_url(url, str(link.get("href") or "")) if link else "" + match = re.fullmatch(r"/job/(\d+)/[^/]+/?", urlsplit(job_url).path) + if not title or not match: + continue + place = "Arendonk" if "Arendonk" in local_places else "Olen" + jobs.append( + self._job( + url=job_url, + title=title, + employer="Ravago", + location=f"{place}, Antwerpen", + postal_code="2370" if place == "Arendonk" else "2250", + description=card.get_text(" ", strip=True), + external_id=match.group(1), + evidence_source="ravago-softgarden-card", + ) + ) + return jobs + + def _sanofi(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for link in soup.select(".job-list a[data-job-id][href]"): + location_node = link.select_one(".job-location") + title_node = link.select_one(".job-title") + location = location_node.get_text(" ", strip=True) if location_node else "" + title = title_node.get_text(" ", strip=True) if title_node else "" + external_id = str(link.get("data-job-id") or "") + job_url = self._same_host_url(url, str(link.get("href") or "")) + if location != "Geel, Belgium" or not title or not external_id.isdigit(): + continue + if not re.fullmatch(r"/en/job/geel/[^/]+/\d+/\d+", urlsplit(job_url).path): + continue + jobs.append( + self._job( + url=job_url, + title=title, + employer="Sanofi", + location="Geel, Antwerpen", + postal_code="2440", + description=link.get_text(" ", strip=True), + external_id=external_id, + evidence_source="sanofi-belgium-geel-card", + ) + ) + return jobs + + def _daf(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for card in soup.select("li.itemlist__item"): + location_node = card.select_one(".js-vac-metalocation") + title_link = card.select_one("a.js-vac-title[href]") + location = location_node.get_text(" ", strip=True) if location_node else "" + title = title_link.get_text(" ", strip=True) if title_link else "" + job_url = ( + self._same_host_url(url, str(title_link.get("href") or "")) if title_link else "" + ) + path = urlsplit(job_url).path + prefix = "/nl-nl/werken-bij-daf/vacatures/" + if location != "Westerlo" or not title or not path.startswith(prefix): + continue + slug = path.removeprefix(prefix).strip("/") + if not slug or "/" in slug: + continue + jobs.append( + self._job( + url=job_url, + title=title, + employer="DAF Trucks", + location="Westerlo, Antwerpen", + postal_code="2260", + description=card.get_text(" ", strip=True), + external_id=slug, + evidence_source="daf-westerlo-card", + ) + ) + return jobs + + def extract(self, content: str, *, url: str) -> ExtractionResult: + route = self._route(url) + if route is None: + return ExtractionResult( + [], self.parser_key, self.parser_version, 0.0, ["Onherkende Kempen-bron"] + ) + host, _ = route + extractors = { + "ziekenhuisgeel.careersite.be": self._ziekenhuis_geel, + "geel.hro.be": self._stad_geel, + "jobs.turnhout.be": self._stad_turnhout, + "jobs.renotec.be": self._renotec, + "ravago.softgarden.io": self._ravago, + "jobs.sanofi.com": self._sanofi, + "www.daf.com": self._daf, + } + jobs = extractors[host](BeautifulSoup(content, "lxml"), url) + unique_jobs = list({job.url: job for job in jobs if job.url}.values()) + return ExtractionResult( + unique_jobs, + self.parser_key, + self.parser_version, + 0.92 if unique_jobs else 0.0, + [] if unique_jobs else ["Geen actuele regionale vacatures gevonden"], + ) diff --git a/apps/sources/adapters/mol_region.py b/apps/sources/adapters/mol_region.py new file mode 100644 index 0000000..9338563 --- /dev/null +++ b/apps/sources/adapters/mol_region.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +import re +from urllib.parse import parse_qs, urljoin, urlsplit + +from bs4 import BeautifulSoup + +from .base import ExtractedJob, ExtractionResult, FieldEvidence + + +class MolRegionEmployerAdapter: + """Extract reviewed employer listings around postcode 2400 without detail fetches.""" + + parser_key = "regional-mol-employers" + parser_version = "1.0.0" + thomas_more_company_guid = "eab9ca13-ee10-4504-8a87-a785d0b037ef" + supported_routes = { + "www.sckcen.be": {"/nl/carriere/vacatures"}, + "cipalschaubroeck.teamtailor.com": {"/jobs"}, + "www.vanroey.be": {"/en/job-overview"}, + "netropolix.recruitee.com": {"/"}, + "jobpage.cvwarehouse.com": {"/"}, + } + + def _route(self, url: str) -> tuple[str, str] | None: + parts = urlsplit(url) + host = (parts.hostname or "").lower() + path = parts.path.rstrip("/") or "/" + if parts.scheme != "https" or path not in self.supported_routes.get(host, set()): + return None + if host == "jobpage.cvwarehouse.com": + query = parse_qs(parts.query) + if query.get("companyGuid") != [self.thomas_more_company_guid] or "job" in query: + return None + return host, path + + def _supports_url(self, url: str) -> bool: + return self._route(url) is not None + + @staticmethod + def _same_host_url(base_url: str, href: str) -> str: + job_url = urljoin(base_url, href) + if urlsplit(job_url).scheme != "https": + return "" + if (urlsplit(job_url).hostname or "").lower() != ( + urlsplit(base_url).hostname or "" + ).lower(): + return "" + return job_url + + @staticmethod + def _job( + *, + url: str, + title: str, + employer: str, + location: str, + postal_code: str, + description: str, + external_id: str, + evidence_source: str, + ) -> ExtractedJob: + return ExtractedJob( + url=url, + title=title, + employer_name=employer, + external_id=external_id, + location_text=location, + postal_code=postal_code, + region="Antwerpen", + country="BE", + description_text=description, + raw={"regional_listing": evidence_source, "region_center": "2400 Mol"}, + evidence=[ + FieldEvidence("title", evidence_source, 0.94, title[:240]), + FieldEvidence("employer_name", "reviewed-source", 0.96, employer), + FieldEvidence("location_text", "mol-region-review", 0.82, location), + ], + ) + + def _sck_cen(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for link in soup.select("article a[href]"): + job_url = self._same_host_url(url, str(link.get("href") or "")) + path = urlsplit(job_url).path if job_url else "" + if not path.startswith("/nl/carriere/vacatures/"): + continue + title = link.get_text(" ", strip=True) + if not title: + continue + card = link.find_parent("article") + description = card.get_text(" ", strip=True) if card else title + jobs.append( + self._job( + url=job_url, + title=title, + employer="SCK CEN", + location="Mol, Antwerpen", + postal_code="2400", + description=description, + external_id=path.rstrip("/").rsplit("/", 1)[-1], + evidence_source="sck-cen-vacancy-card", + ) + ) + return jobs + + def _cipal(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for card in soup.select("li"): + card_text = card.get_text(" ", strip=True) + if "Westerlo" not in card_text and "Geel" not in card_text: + continue + link = card.find("a", href=True) + if not link: + continue + title = link.get_text(" ", strip=True) + job_url = self._same_host_url(url, str(link.get("href") or "")) + path = urlsplit(job_url).path if job_url else "" + match = re.fullmatch(r"/jobs/(\d+)-[^/]+", path.rstrip("/")) + if not title or not match: + continue + location = "Geel, Antwerpen" if "Geel" in card_text else "Westerlo, Antwerpen" + jobs.append( + self._job( + url=job_url, + title=title, + employer="Cipal Schaubroeck", + location=location, + postal_code="2440" if "Geel" in card_text else "2260", + description=card_text, + external_id=match.group(1), + evidence_source="cipal-teamtailor-card", + ) + ) + return jobs + + def _vanroey(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for heading in soup.select("h3.elementor-heading-title"): + link = heading.find("a", href=True) + if not link: + continue + title = link.get_text(" ", strip=True) + job_url = self._same_host_url(url, str(link.get("href") or "")) + path = urlsplit(job_url).path if job_url else "" + if not title or not re.fullmatch(r"/en/job/[^/]+/", path): + continue + slug = path.rstrip("/").rsplit("/", 1)[-1] + if "oost-vlaanderen" in slug: + continue + card = heading.find_parent("section") or heading.parent + description = card.get_text(" ", strip=True) if card else title + jobs.append( + self._job( + url=job_url, + title=title, + employer="VanRoey", + location="Turnhout/Geel (hybride; controleer vacaturedetail)", + postal_code="", + description=description, + external_id=slug, + evidence_source="vanroey-job-card", + ) + ) + return jobs + + def _netropolix(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for link in soup.find_all("a", href=True): + title = link.get_text(" ", strip=True) + if "Geel" not in title: + continue + job_url = self._same_host_url(url, str(link.get("href") or "")) + path = urlsplit(job_url).path if job_url else "" + if not re.fullmatch(r"/o/[^/]+", path.rstrip("/")): + continue + card = link.find_parent("div") + description = card.get_text(" ", strip=True) if card else title + jobs.append( + self._job( + url=job_url, + title=title, + employer="NTX (Netropolix)", + location="Geel, Antwerpen", + postal_code="2440", + description=description, + external_id=path.rstrip("/").rsplit("/", 1)[-1], + evidence_source="netropolix-recruitee-card", + ) + ) + return jobs + + def _thomas_more(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for link in soup.select("a.jobLink[data-item='readmore'][data-jobid][href]"): + title = link.get_text(" ", strip=True) + title_casefold = title.casefold() + if not any(place in title_casefold for place in ("geel", "turnhout", "vorselaar")): + continue + external_id = str(link.get("data-jobid") or "") + if not external_id.isdigit(): + continue + job_url = self._same_host_url(url, str(link.get("href") or "")) + query = parse_qs(urlsplit(job_url).query) if job_url else {} + if query.get("companyGuid") != [self.thomas_more_company_guid] or query.get("job") != [ + external_id + ]: + continue + if "turnhout" in title_casefold: + location, postal_code = "Turnhout, Antwerpen", "2300" + elif "vorselaar" in title_casefold: + location, postal_code = "Vorselaar, Antwerpen", "2290" + else: + location, postal_code = "Geel, Antwerpen", "2440" + jobs.append( + self._job( + url=job_url, + title=title, + employer="Thomas More", + location=location, + postal_code=postal_code, + description=f"Publieke regionale vacature bij Thomas More: {title}.", + external_id=external_id, + evidence_source="thomas-more-cvwarehouse-card", + ) + ) + return jobs + + def extract(self, content: str, *, url: str) -> ExtractionResult: + route = self._route(url) + if route is None: + return ExtractionResult( + [], self.parser_key, self.parser_version, 0.0, ["Onherkende Mol-regiobron"] + ) + host, _ = route + soup = BeautifulSoup(content, "lxml") + extractors = { + "www.sckcen.be": self._sck_cen, + "cipalschaubroeck.teamtailor.com": self._cipal, + "www.vanroey.be": self._vanroey, + "netropolix.recruitee.com": self._netropolix, + "jobpage.cvwarehouse.com": self._thomas_more, + } + jobs = extractors[host](soup, url) + unique_jobs = list({job.url: job for job in jobs}.values()) + warnings = [] if unique_jobs else ["Geen actuele vacatures rond Mol gevonden"] + return ExtractionResult( + unique_jobs, + self.parser_key, + self.parser_version, + 0.9 if unique_jobs else 0.0, + warnings, + ) diff --git a/apps/sources/adapters/regional.py b/apps/sources/adapters/regional.py new file mode 100644 index 0000000..b0e0f02 --- /dev/null +++ b/apps/sources/adapters/regional.py @@ -0,0 +1,312 @@ +from __future__ import annotations + +from urllib.parse import urljoin, urlsplit + +from bs4 import BeautifulSoup + +from .base import ExtractedJob, ExtractionResult, FieldEvidence + + +class CordaCampusAdapter: + """Extract the public job cards intentionally published by Corda Campus.""" + + parser_key = "regional-corda-campus" + parser_version = "1.0.0" + source_hosts = ("cordacampus.com",) + + def _supports_url(self, url: str) -> bool: + parts = urlsplit(url) + host = (parts.hostname or "").lower() + return host.endswith(self.source_hosts) and parts.path.rstrip("/") == "/jobs" + + def extract(self, content: str, *, url: str) -> ExtractionResult: + if not self._supports_url(url): + return ExtractionResult( + [], self.parser_key, self.parser_version, 0.0, ["Onherkenbare regiobron"] + ) + + source_host = (urlsplit(url).hostname or "").lower() + soup = BeautifulSoup(content, "lxml") + jobs: list[ExtractedJob] = [] + seen_urls: set[str] = set() + + for card in soup.select('a.event-item[href*="/job/"]'): + job_url = urljoin(url, str(card.get("href") or "")) + job_host = (urlsplit(job_url).hostname or "").lower() + if job_host != source_host or job_url in seen_urls: + continue + title_node = card.select_one(".job-title") + employer_node = card.select_one(".company-title") + title = title_node.get_text(" ", strip=True) if title_node else "" + employer = employer_node.get_text(" ", strip=True) if employer_node else "" + if not title: + continue + + date_node = card.select_one(".bottom-info") + date_posted = date_node.get_text(" ", strip=True) if date_node else "" + external_id = urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1] + description = f"Vacature van {employer or 'een Corda-werkgever'} via Corda Campus." + jobs.append( + ExtractedJob( + url=job_url, + title=title, + employer_name=employer, + external_id=external_id, + location_text="Hasselt, Limburg", + region="Limburg", + country="BE", + description_text=description, + date_posted=date_posted, + raw={"regional_listing": "corda-campus"}, + evidence=[ + FieldEvidence("title", "corda-job-card", 0.95, title[:240]), + FieldEvidence("employer_name", "corda-job-card", 0.92, employer[:240]), + FieldEvidence( + "location_text", + "regional-source-scope", + 0.72, + "Corda Campus, Hasselt", + ), + ], + ) + ) + seen_urls.add(job_url) + + warnings = [] if jobs else ["Geen actuele Corda-vacatures gevonden"] + return ExtractionResult( + jobs, + self.parser_key, + self.parser_version, + 0.9 if jobs else 0.0, + warnings, + ) + + +class LocalEmployerListingAdapter: + """Extract reviewed public listing pages of employers in the Hasselt-Genk area.""" + + parser_key = "regional-local-employers" + parser_version = "1.0.0" + supported_routes = { + "acagroup.be": {"/en/jobs"}, + "www.xploregroup.be": {"/en/jobs"}, + "www.uhasselt.be": {"/vacatures"}, + "ses.pxl.be": {"/"}, + "ziekenhuis-oost-limburg.cvw.io": {"/"}, + } + + def _supports_url(self, url: str) -> bool: + return self._route(url) is not None + + def _route(self, url: str) -> tuple[str, str] | None: + parts = urlsplit(url) + host = (parts.hostname or "").lower() + path = parts.path.rstrip("/") or "/" + if parts.scheme != "https" or path not in self.supported_routes.get(host, set()): + return None + return host, path + + @staticmethod + def _job( + *, + url: str, + title: str, + employer: str, + location: str, + description: str, + external_id: str, + evidence_source: str, + ) -> ExtractedJob: + return ExtractedJob( + url=url, + title=title, + employer_name=employer, + external_id=external_id, + location_text=location, + region="Limburg", + country="BE", + description_text=description, + raw={"regional_listing": evidence_source}, + evidence=[ + FieldEvidence("title", evidence_source, 0.94, title[:240]), + FieldEvidence("employer_name", "reviewed-source", 0.96, employer), + FieldEvidence("location_text", "regional-source-scope", 0.78, location), + ], + ) + + @staticmethod + def _same_host_url(base_url: str, href: str) -> str: + job_url = urljoin(base_url, href) + if (urlsplit(job_url).hostname or "").lower() != ( + urlsplit(base_url).hostname or "" + ).lower(): + return "" + return job_url + + def _aca(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for link in soup.find_all("a", href=True): + href = str(link.get("href") or "") + if not href.startswith("/en/jobs/"): + continue + title_node = link.find("h3") + title = title_node.get_text(" ", strip=True) if title_node else "" + job_url = self._same_host_url(url, href) + if not title or not job_url: + continue + description_node = link.find("p") + description = ( + description_node.get_text(" ", strip=True) + if description_node + else f"Vacature bij ACA Group: {title}." + ) + jobs.append( + self._job( + url=job_url, + title=title, + employer="ACA Group", + location="Hasselt (hybride; kantoorselectie per vacature)", + description=description, + external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1], + evidence_source="aca-job-card", + ) + ) + return jobs + + def _xplore(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for heading in soup.find_all("h3"): + link = heading.find("a", href=True) + if not link or not str(link.get("href") or "").startswith("/en/jobs/"): + continue + card = heading.parent + location_node = next( + ( + node + for node in card.find_all("p") + if "Hasselt" in node.get_text(" ", strip=True) + ), + None, + ) + if not location_node: + continue + title = link.get_text(" ", strip=True) + job_url = self._same_host_url(url, str(link.get("href") or "")) + if not title or not job_url: + continue + location = location_node.get_text(" ", strip=True) + jobs.append( + self._job( + url=job_url, + title=title, + employer="Xplore Group", + location=location, + description=( + f"Vacature bij Xplore Group met Hasselt als mogelijke werklocatie: {title}." + ), + external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1], + evidence_source="xplore-job-card", + ) + ) + return jobs + + def _uhasselt(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for card in soup.select("section.vacancy-item"): + title_node = card.find("h3") + link = card.find("a", href=True) + title = title_node.get_text(" ", strip=True) if title_node else "" + job_url = self._same_host_url(url, str(link.get("href") or "")) if link else "" + if not title or not job_url or "/vacatures/detail/" not in urlsplit(job_url).path: + continue + external_id = urlsplit(job_url).path.split("/detail/", 1)[-1].split("-", 1)[0] + jobs.append( + self._job( + url=job_url, + title=title, + employer="Universiteit Hasselt", + location="Hasselt/Diepenbeek, Limburg", + description=card.get_text(" ", strip=True), + external_id=external_id, + evidence_source="uhasselt-vacancy-card", + ) + ) + return jobs + + def _pxl(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for card in soup.select(".vacature-card[id^='Vacature_']"): + external_id = str(card.get("id") or "").removeprefix("Vacature_") + title_node = card.select_one(".vacature-card-titel") + title = title_node.get_text(" ", strip=True) if title_node else "" + if not title or not external_id.isdigit(): + continue + cells = [ + node.get_text(" ", strip=True) for node in card.select("td.vacature-card-td-text") + ] + campus = next((value for value in cells if value.startswith("Campus ")), "Hasselt") + job_url = f"{url.rstrip('/')}?vacature_id={external_id}" + jobs.append( + self._job( + url=job_url, + title=title, + employer="Hogeschool PXL", + location=f"{campus}, Limburg", + description=" · ".join(cells), + external_id=external_id, + evidence_source="pxl-vacancy-card", + ) + ) + return jobs + + def _zol(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]: + jobs = [] + for link in soup.select("a[data-item='readmore'][data-jobid][href]"): + external_id = str(link.get("data-jobid") or "") + title_node = link.select_one(".job-title") + title = title_node.get_text(" ", strip=True) if title_node else "" + job_url = self._same_host_url(url, str(link.get("href") or "")) + if not title or not external_id.isdigit() or not job_url: + continue + jobs.append( + self._job( + url=job_url, + title=title, + employer="Ziekenhuis Oost-Limburg", + location="Genk/Lanaken/Maaseik, Limburg", + description=f"Publieke vacature van Ziekenhuis Oost-Limburg: {title}.", + external_id=external_id, + evidence_source="zol-cvwarehouse-card", + ) + ) + return jobs + + def extract(self, content: str, *, url: str) -> ExtractionResult: + route = self._route(url) + if route is None: + return ExtractionResult( + [], + self.parser_key, + self.parser_version, + 0.0, + ["Onherkenbare lokale werkgeversbron"], + ) + host, _ = route + soup = BeautifulSoup(content, "lxml") + extractors = { + "acagroup.be": self._aca, + "www.xploregroup.be": self._xplore, + "www.uhasselt.be": self._uhasselt, + "ses.pxl.be": self._pxl, + "ziekenhuis-oost-limburg.cvw.io": self._zol, + } + jobs = extractors[host](soup, url) + unique_jobs = list({job.url: job for job in jobs}.values()) + warnings = [] if unique_jobs else ["Geen actuele lokale vacatures gevonden"] + return ExtractionResult( + unique_jobs, + self.parser_key, + self.parser_version, + 0.9 if unique_jobs else 0.0, + warnings, + ) diff --git a/apps/sources/adapters/registry.py b/apps/sources/adapters/registry.py index 472bb7a..8428704 100644 --- a/apps/sources/adapters/registry.py +++ b/apps/sources/adapters/registry.py @@ -12,6 +12,9 @@ from .ats import ( from .base import ExtractionResult from .generic_html import GenericHtmlAdapter from .jsonld import JsonLdJobPostingAdapter +from .kempen import KempenEmployerAdapter +from .mol_region import MolRegionEmployerAdapter +from .regional import CordaCampusAdapter, LocalEmployerListingAdapter from .rss import RssAdapter @@ -22,10 +25,18 @@ class AdapterRegistry: self.recruitee = RecruiteeAdapter() self.smartrecruiters = SmartRecruitersAdapter() self.workable = WorkableAdapter() + self.corda_campus = CordaCampusAdapter() + self.local_employers = LocalEmployerListingAdapter() + self.mol_region_employers = MolRegionEmployerAdapter() + self.kempen_employers = KempenEmployerAdapter() self.jsonld = JsonLdJobPostingAdapter() self.generic = GenericHtmlAdapter() self.rss = RssAdapter() self.providers = [ + self.corda_campus, + self.local_employers, + self.mol_region_employers, + self.kempen_employers, self.greenhouse, self.lever, self.recruitee, diff --git a/apps/sources/admin.py b/apps/sources/admin.py index 1ad3ebe..ee32e74 100644 --- a/apps/sources/admin.py +++ b/apps/sources/admin.py @@ -2,6 +2,7 @@ from django.contrib import admin from .models import ( EmailMessageRecord, + MailboxConnection, RawDocument, Source, SourceLease, @@ -12,6 +13,22 @@ from .models import ( ) +@admin.register(MailboxConnection) +class MailboxConnectionAdmin(admin.ModelAdmin): + list_display = ( + "platform", + "provider", + "username", + "enabled", + "poll_interval_minutes", + "last_success_at", + "next_poll_at", + ) + list_filter = ("platform", "provider", "enabled") + search_fields = ("username",) + exclude = ("encrypted_password", "lease_token") + + @admin.register(Source) class SourceAdmin(admin.ModelAdmin): list_display = ( diff --git a/apps/sources/forms.py b/apps/sources/forms.py index 5c56923..5cc00f1 100644 --- a/apps/sources/forms.py +++ b/apps/sources/forms.py @@ -2,6 +2,58 @@ from __future__ import annotations from django import forms +from .models import MailboxConnection + + +class MailboxConnectionForm(forms.ModelForm): + password = forms.CharField( + required=False, + label="App-wachtwoord", + help_text="Laat leeg bij wijzigen om het bestaande app-wachtwoord te behouden.", + widget=forms.PasswordInput( + attrs={"autocomplete": "new-password", "placeholder": "App-wachtwoord"}, + render_value=False, + ), + ) + + class Meta: + model = MailboxConnection + fields = ( + "platform", + "provider", + "custom_host", + "port", + "username", + "mailbox", + "poll_interval_minutes", + "enabled", + ) + labels = { + "platform": "Vacatureplatform", + "provider": "Mailboxprovider", + "custom_host": "Aangepaste IMAP-host", + "port": "IMAP-poort", + "username": "Mailboxaccount", + "mailbox": "Map", + "poll_interval_minutes": "Controlefrequentie", + "enabled": "Automatisch synchroniseren", + } + widgets = { + "username": forms.EmailInput(attrs={"autocomplete": "username"}), + "custom_host": forms.TextInput( + attrs={"autocomplete": "off", "placeholder": "imap.provider.be"} + ), + "mailbox": forms.TextInput(attrs={"autocomplete": "off"}), + } + + def clean(self): + data = super().clean() + if data.get("provider") != MailboxConnection.Provider.CUSTOM: + data["custom_host"] = "" + if not self.instance.pk and not data.get("password"): + self.add_error("password", "Een app-wachtwoord is verplicht voor een nieuwe koppeling.") + return data + class ManualImportForm(forms.Form): source_url = forms.URLField( diff --git a/apps/sources/management/commands/import_job_fixture.py b/apps/sources/management/commands/import_job_fixture.py index 1b4eddd..2fc2ab1 100644 --- a/apps/sources/management/commands/import_job_fixture.py +++ b/apps/sources/management/commands/import_job_fixture.py @@ -31,11 +31,11 @@ class Command(BaseCommand): if not hostname: raise CommandError("--url bevat geen geldig domein") source, _ = Source.objects.get_or_create( - domain=hostname, + base_url=options["url"], source_type=options["source_type"], defaults={ + "domain": hostname, "name": options["source_name"], - "base_url": options["url"], "status": Source.Status.ACTIVE, "policy": Source.Policy.ALLOW, "parser_key": "auto", diff --git a/apps/sources/management/commands/rotate_mailbox_credentials.py b/apps/sources/management/commands/rotate_mailbox_credentials.py new file mode 100644 index 0000000..8b183bd --- /dev/null +++ b/apps/sources/management/commands/rotate_mailbox_credentials.py @@ -0,0 +1,17 @@ +from django.core.management.base import BaseCommand, CommandError + +from apps.sources.services.mailbox_connections import ( + MailboxCredentialError, + rotate_mailbox_credentials, +) + + +class Command(BaseCommand): + help = "Versleutel alle mailbox-app-wachtwoorden opnieuw met de eerste ingestelde sleutel." + + def handle(self, *args, **options): + try: + count = rotate_mailbox_credentials() + except MailboxCredentialError as exc: + raise CommandError(str(exc)) from exc + self.stdout.write(self.style.SUCCESS(f"{count} mailboxcredential(s) geroteerd.")) diff --git a/apps/sources/management/commands/seed_sources.py b/apps/sources/management/commands/seed_sources.py index c7f44d7..41d5872 100644 --- a/apps/sources/management/commands/seed_sources.py +++ b/apps/sources/management/commands/seed_sources.py @@ -5,8 +5,9 @@ from urllib.parse import urlsplit import yaml from django.core.management.base import BaseCommand, CommandError +from django.utils.dateparse import parse_datetime -from apps.sources.models import Source +from apps.sources.models import Source, SourcePolicyReview from apps.sources.services.policy import is_denied_domain @@ -21,7 +22,38 @@ class Command(BaseCommand): if not path.is_file(): raise CommandError(f"Seedbestand ontbreekt: {path}") payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - created = updated = 0 + created = updated = reviews_created = 0 + if payload.get("disable_demo_sources") is True: + demo_sources = Source.objects.filter(domain="jobs.example.org") + for demo_source in demo_sources: + demo_source.status = Source.Status.DISABLED + demo_source.policy = Source.Policy.DENY + demo_source.policy_reason = "Demobron verborgen na installatie van livebronnen" + demo_source.metadata = {**demo_source.metadata, "demo": True} + demo_source.save( + update_fields=[ + "status", + "policy", + "policy_reason", + "metadata", + "updated_at", + ] + ) + retired = 0 + for retired_url in payload.get("retire_source_urls", []): + for retired_source in Source.objects.filter(base_url=str(retired_url).strip()): + retired_source.status = Source.Status.DISABLED + retired_source.next_run_at = None + retired_source.metadata = { + **retired_source.metadata, + "retired": True, + "retired_reason": "Buiten de ingestelde regionale dekking", + "hidden_from_source_list": True, + } + retired_source.save( + update_fields=["status", "next_run_at", "metadata", "updated_at"] + ) + retired += 1 for item in payload.get("sources", []): url = str(item.get("url") or "").strip() domain = (urlsplit(url).hostname or "").lower() @@ -30,11 +62,11 @@ class Command(BaseCommand): continue deny = is_denied_domain(domain) source, was_created = Source.objects.update_or_create( - domain=domain, + base_url=url, source_type=item.get("type", Source.Type.EMPLOYER), defaults={ + "domain": domain, "name": item.get("name") or domain, - "base_url": url, "status": Source.Status.PAUSED if deny else item.get("status", Source.Status.TRIAL), @@ -46,10 +78,39 @@ class Command(BaseCommand): else item.get("policy_reason", ""), "parser_key": item.get("parser", "auto"), "crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)), + "minimum_interval_seconds": int(item.get("minimum_interval_seconds", 30)), + "allow_public_endpoint": bool(item.get("allow_public_endpoint", False)), + "metadata": item.get("metadata") or {}, }, ) + review = item.get("review") or {} + if review: + expires_at = parse_datetime(str(review.get("expires_at") or "")) + if expires_at is None: + raise CommandError( + f"Bron {source.name}: review.expires_at moet een ISO-datetime zijn" + ) + decision = str(review.get("decision") or "") + if decision not in SourcePolicyReview.Decision.values: + raise CommandError(f"Bron {source.name}: ongeldige reviewbeslissing") + _, review_created = SourcePolicyReview.objects.get_or_create( + source=source, + decision=decision, + evidence_link=str(review.get("evidence_link") or ""), + expires_at=expires_at, + defaults={ + "scope": SourcePolicyReview.Scope.SOURCE, + "reason": str(review.get("reason") or "Bronseedreview"), + "notes": str(review.get("notes") or ""), + "metadata": review.get("metadata") or {}, + }, + ) + reviews_created += int(review_created) created += int(was_created) updated += int(not was_created) self.stdout.write( - self.style.SUCCESS(f"Bronnen: {created} aangemaakt, {updated} bijgewerkt") + self.style.SUCCESS( + f"Bronnen: {created} aangemaakt, {updated} bijgewerkt, " + f"{reviews_created} reviews geregistreerd, {retired} buiten regio uitgeschakeld" + ) ) diff --git a/apps/sources/migrations/0004_remove_source_unique_domain_source_type_and_more.py b/apps/sources/migrations/0004_remove_source_unique_domain_source_type_and_more.py new file mode 100644 index 0000000..fdbb366 --- /dev/null +++ b/apps/sources/migrations/0004_remove_source_unique_domain_source_type_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 5.2.16 on 2026-07-21 22:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sources', '0003_source_lease_and_origin_state'), + ] + + operations = [ + migrations.RemoveConstraint( + model_name='source', + name='unique_domain_source_type', + ), + migrations.AddConstraint( + model_name='source', + constraint=models.UniqueConstraint(condition=models.Q(('base_url', ''), _negated=True), fields=('source_type', 'base_url'), name='unique_source_type_base_url_nonempty'), + ), + ] diff --git a/apps/sources/migrations/0005_alter_emailmessagerecord_message_id_and_more.py b/apps/sources/migrations/0005_alter_emailmessagerecord_message_id_and_more.py new file mode 100644 index 0000000..39e8bb4 --- /dev/null +++ b/apps/sources/migrations/0005_alter_emailmessagerecord_message_id_and_more.py @@ -0,0 +1,67 @@ +# Generated by Django 5.2.16 on 2026-07-22 00:12 + +import django.db.models.deletion +import django.utils.timezone +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sources', '0004_remove_source_unique_domain_source_type_and_more'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AlterField( + model_name='emailmessagerecord', + name='message_id', + field=models.CharField(max_length=998), + ), + migrations.CreateModel( + name='MailboxConnection', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('platform', models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed')], max_length=24)), + ('provider', models.CharField(choices=[('gmail', 'Gmail'), ('outlook', 'Outlook / Microsoft 365'), ('custom', 'Andere IMAP-provider')], max_length=24)), + ('custom_host', models.CharField(blank=True, max_length=255)), + ('port', models.PositiveIntegerField(default=993)), + ('username', models.CharField(max_length=320)), + ('encrypted_password', models.TextField()), + ('mailbox', models.CharField(default='INBOX', max_length=255)), + ('enabled', models.BooleanField(default=True)), + ('poll_interval_minutes', models.PositiveIntegerField(choices=[(5, 'Elke 5 minuten'), (15, 'Elke 15 minuten'), (30, 'Elke 30 minuten'), (60, 'Elk uur'), (180, 'Elke 3 uur'), (360, 'Elke 6 uur')], default=30)), + ('next_poll_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), + ('last_polled_at', models.DateTimeField(blank=True, null=True)), + ('last_success_at', models.DateTimeField(blank=True, null=True)), + ('last_error_category', models.CharField(blank=True, max_length=80)), + ('last_error_message', models.CharField(blank=True, max_length=500)), + ('lease_token', models.CharField(blank=True, max_length=64)), + ('lease_expires_at', models.DateTimeField(blank=True, db_index=True, null=True)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mailbox_connections', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['platform'], + }, + ), + migrations.AddField( + model_name='emailmessagerecord', + name='mailbox_connection', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='sources.mailboxconnection'), + ), + migrations.AddConstraint( + model_name='emailmessagerecord', + constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', False)), fields=('mailbox_connection', 'message_id'), name='unique_message_per_mailbox_connection'), + ), + migrations.AddConstraint( + model_name='emailmessagerecord', + constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', True)), fields=('message_id',), name='unique_legacy_email_message_id'), + ), + migrations.AddConstraint( + model_name='mailboxconnection', + constraint=models.UniqueConstraint(fields=('user', 'platform'), name='unique_mailbox_platform_per_user'), + ), + ] diff --git a/apps/sources/migrations/0006_alter_mailboxconnection_platform.py b/apps/sources/migrations/0006_alter_mailboxconnection_platform.py new file mode 100644 index 0000000..923ca7d --- /dev/null +++ b/apps/sources/migrations/0006_alter_mailboxconnection_platform.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.16 on 2026-07-22 00:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sources', '0005_alter_emailmessagerecord_message_id_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='mailboxconnection', + name='platform', + field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('eurobrussels', 'EuroBrussels'), ('brusselsjobs', 'BrusselsJobs')], max_length=24), + ), + ] diff --git a/apps/sources/migrations/0007_alter_mailboxconnection_platform.py b/apps/sources/migrations/0007_alter_mailboxconnection_platform.py new file mode 100644 index 0000000..149a2cc --- /dev/null +++ b/apps/sources/migrations/0007_alter_mailboxconnection_platform.py @@ -0,0 +1,26 @@ +# Generated by Django 5.2.16 on 2026-07-22 01:25 + +from django.db import migrations, models + + +def disable_retired_brussels_mailboxes(apps, schema_editor): + mailbox_connection = apps.get_model("sources", "MailboxConnection") + mailbox_connection.objects.filter( + platform__in=("eurobrussels", "brusselsjobs") + ).update(enabled=False) + + +class Migration(migrations.Migration): + + dependencies = [ + ('sources', '0006_alter_mailboxconnection_platform'), + ] + + operations = [ + migrations.RunPython(disable_retired_brussels_mailboxes, migrations.RunPython.noop), + migrations.AlterField( + model_name='mailboxconnection', + name='platform', + field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('careerjet', 'Careerjet'), ('randstad', 'Randstad'), ('roberthalf', 'Robert Half')], max_length=24), + ), + ] diff --git a/apps/sources/models.py b/apps/sources/models.py index 6bf03ba..dd69fe3 100644 --- a/apps/sources/models.py +++ b/apps/sources/models.py @@ -62,7 +62,9 @@ class Source(TimeStampedModel): ordering = ["name"] constraints = [ models.UniqueConstraint( - fields=["domain", "source_type"], name="unique_domain_source_type" + fields=["source_type", "base_url"], + condition=~models.Q(base_url=""), + name="unique_source_type_base_url_nonempty", ) ] @@ -315,8 +317,101 @@ class RawDocument(TimeStampedModel): return self.final_url or self.url or f"Document {self.pk}" +class MailboxConnection(TimeStampedModel): + class Platform(models.TextChoices): + VDAB = "vdab", "VDAB" + INDEED = "indeed", "Indeed" + LINKEDIN = "linkedin", "LinkedIn" + ICTJOB = "ictjob", "ictjob.be" + JOBAT = "jobat", "Jobat" + STEPSTONE = "stepstone", "StepStone" + CAREERJET = "careerjet", "Careerjet" + RANDSTAD = "randstad", "Randstad" + ROBERTHALF = "roberthalf", "Robert Half" + + class Provider(models.TextChoices): + GMAIL = "gmail", "Gmail" + OUTLOOK = "outlook", "Outlook / Microsoft 365" + CUSTOM = "custom", "Andere IMAP-provider" + + class PollInterval(models.IntegerChoices): + FIVE_MINUTES = 5, "Elke 5 minuten" + FIFTEEN_MINUTES = 15, "Elke 15 minuten" + THIRTY_MINUTES = 30, "Elke 30 minuten" + HOURLY = 60, "Elk uur" + THREE_HOURS = 180, "Elke 3 uur" + SIX_HOURS = 360, "Elke 6 uur" + + PROVIDER_HOSTS = { + Provider.GMAIL: "imap.gmail.com", + Provider.OUTLOOK: "outlook.office365.com", + } + + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + related_name="mailbox_connections", + ) + platform = models.CharField(max_length=24, choices=Platform.choices) + provider = models.CharField(max_length=24, choices=Provider.choices) + custom_host = models.CharField(max_length=255, blank=True) + port = models.PositiveIntegerField(default=993) + username = models.CharField(max_length=320) + encrypted_password = models.TextField() + mailbox = models.CharField(max_length=255, default="INBOX") + enabled = models.BooleanField(default=True) + poll_interval_minutes = models.PositiveIntegerField( + choices=PollInterval.choices, + default=PollInterval.THIRTY_MINUTES, + ) + next_poll_at = models.DateTimeField(default=timezone.now, db_index=True) + last_polled_at = models.DateTimeField(null=True, blank=True) + last_success_at = models.DateTimeField(null=True, blank=True) + last_error_category = models.CharField(max_length=80, blank=True) + last_error_message = models.CharField(max_length=500, blank=True) + lease_token = models.CharField(max_length=64, blank=True) + lease_expires_at = models.DateTimeField(null=True, blank=True, db_index=True) + + class Meta: + ordering = ["platform"] + constraints = [ + models.UniqueConstraint( + fields=["user", "platform"], name="unique_mailbox_platform_per_user" + ) + ] + + def __str__(self) -> str: + return f"{self.get_platform_display()} · {self.username}" + + @property + def imap_host(self) -> str: + return self.PROVIDER_HOSTS.get(self.provider, self.custom_host).strip().lower() + + def clean(self) -> None: + host = self.imap_host + if ( + not host + or "://" in host + or "/" in host + or "@" in host + or any(c.isspace() for c in host) + ): + raise ValidationError({"custom_host": "Vul een geldige IMAP-hostnaam in."}) + if not 1 <= self.port <= 65535: + raise ValidationError({"port": "De IMAP-poort moet tussen 1 en 65535 liggen."}) + if "\n" in self.mailbox or "\r" in self.mailbox: + raise ValidationError({"mailbox": "De mailboxnaam bevat ongeldige tekens."}) + + class EmailMessageRecord(TimeStampedModel): - message_id = models.CharField(max_length=998, unique=True) + mailbox_connection = models.ForeignKey( + MailboxConnection, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="messages", + ) + message_id = models.CharField(max_length=998) mailbox = models.CharField(max_length=255, default="INBOX") sender = models.CharField(max_length=500, blank=True) subject = models.CharField(max_length=998, blank=True) @@ -330,6 +425,18 @@ class EmailMessageRecord(TimeStampedModel): class Meta: ordering = ["-received_at", "-created_at"] + constraints = [ + models.UniqueConstraint( + fields=["mailbox_connection", "message_id"], + condition=models.Q(mailbox_connection__isnull=False), + name="unique_message_per_mailbox_connection", + ), + models.UniqueConstraint( + fields=["message_id"], + condition=models.Q(mailbox_connection__isnull=True), + name="unique_legacy_email_message_id", + ), + ] def __str__(self) -> str: return self.subject or self.message_id diff --git a/apps/sources/platforms.py b/apps/sources/platforms.py new file mode 100644 index 0000000..c432a93 --- /dev/null +++ b/apps/sources/platforms.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class PlatformAlert: + key: str + label: str + domains: tuple[str, ...] + alert_url: str + help_text: str + + +PLATFORM_ALERTS = { + alert.key: alert + for alert in ( + PlatformAlert( + "vdab", + "VDAB", + ("vdab.be",), + "https://www.vdab.be/jobs/job-alerts", + "Brede Vlaamse vacaturealerts.", + ), + PlatformAlert( + "indeed", + "Indeed", + ("indeed.com", "indeed.be"), + "https://support.indeed.com/hc/nl/articles/204488890-" + "Vacature-alerts-activeren-stopzetten-en-beheren", + "Algemene en regionale vacaturealerts.", + ), + PlatformAlert( + "linkedin", + "LinkedIn", + ("linkedin.com",), + "https://www.linkedin.com/help/linkedin/answer/a511279", + "Dagelijkse of wekelijkse alerts, maximaal twintig per LinkedIn-account.", + ), + PlatformAlert( + "ictjob", + "ictjob.be", + ("ictjob.be",), + "https://www.ictjob.be/nl/it-job-alert", + "Gespecialiseerde Belgische IT- en telecomvacatures.", + ), + PlatformAlert( + "jobat", + "Jobat", + ("jobat.be",), + "https://www.jobat.be/nl/jobalert", + "Belgische jobs per functie, regio en contracttype.", + ), + PlatformAlert( + "stepstone", + "StepStone", + ("stepstone.be",), + "https://www.stepstone.be/vacatures", + "Dagelijkse vergelijkbare vacatures vanuit een zoekopdracht.", + ), + PlatformAlert( + "careerjet", + "Careerjet", + ("careerjet.be",), + "https://www.careerjet.be/", + "Brede Belgische zoekresultaten met regio- en postcodegebonden e-mailalerts.", + ), + PlatformAlert( + "randstad", + "Randstad", + ("randstad.be",), + "https://www.randstad.be/jobalert-aanmaken/", + "Belgische jobs met instelbare specialisatie, locatie, contract en frequentie.", + ), + PlatformAlert( + "roberthalf", + "Robert Half", + ("roberthalf.com",), + "https://www.roberthalf.com/be/en/find-jobs/job-alerts", + "Belgische IT- en digitaljobs, waaronder systeem-, netwerk- en supportfuncties.", + ), + ) +} + + +def platform_label(key: str) -> str: + alert = PLATFORM_ALERTS.get(key) + return alert.label if alert else key diff --git a/apps/sources/services/discovery.py b/apps/sources/services/discovery.py index 22952e3..872af67 100644 --- a/apps/sources/services/discovery.py +++ b/apps/sources/services/discovery.py @@ -140,23 +140,28 @@ def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int for candidate in ordered: provenance = _provenance_entry(candidate) now = _utc_now() - source, was_created = Source.objects.get_or_create( - domain=candidate.domain, - source_type=candidate.source_type, - defaults={ - "name": _candidate_name(candidate), - "base_url": candidate.url, - "status": Source.Status.CANDIDATE, - "policy": Source.Policy.REVIEW, - "policy_reason": "Automatisch ontdekt", - "parser_key": "auto", - "strict_mode": True, - "metadata": { - "discovery": [provenance], - "discovered_at": now, - }, - }, + source = ( + Source.objects.filter( + domain=candidate.domain, + source_type=candidate.source_type, + ) + .order_by("id") + .first() ) + was_created = source is None + if source is None: + source = Source.objects.create( + domain=candidate.domain, + source_type=candidate.source_type, + name=_candidate_name(candidate), + base_url=candidate.url, + status=Source.Status.CANDIDATE, + policy=Source.Policy.REVIEW, + policy_reason="Automatisch ontdekt", + parser_key="auto", + strict_mode=True, + metadata={"discovery": [provenance], "discovered_at": now}, + ) if was_created: created += 1 continue diff --git a/apps/sources/services/email_import.py b/apps/sources/services/email_import.py index ebaa1e8..5772269 100644 --- a/apps/sources/services/email_import.py +++ b/apps/sources/services/email_import.py @@ -13,7 +13,8 @@ from django.utils import timezone from apps.jobs.services.normalization import normalize_extracted_job from apps.jobs.services.pipeline import persist_draft from apps.sources.adapters.email_alert import EmailAlertAdapter -from apps.sources.models import EmailMessageRecord, RawDocument, Source +from apps.sources.models import EmailMessageRecord, MailboxConnection, RawDocument, Source +from apps.sources.platforms import platform_label def message_identity(raw_message: bytes) -> str: @@ -24,22 +25,32 @@ def message_identity(raw_message: bytes) -> str: @transaction.atomic -def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageRecord: +def ingest_email( + raw_message: bytes, + *, + mailbox: str = "INBOX", + mailbox_connection: MailboxConnection | None = None, + platform: str | None = None, +) -> EmailMessageRecord: parsed = BytesParser(policy=policy.default).parsebytes(raw_message) message_id = message_identity(raw_message) - existing = EmailMessageRecord.objects.filter(message_id=message_id).first() + existing = EmailMessageRecord.objects.filter( + message_id=message_id, mailbox_connection=mailbox_connection + ).first() if existing: return existing + source_domain = f"{platform}.mailbox.local" if platform else "mailbox.local" + source_name = f"{platform_label(platform)} vacaturemailbox" if platform else "Vacaturemailbox" source, _ = Source.objects.get_or_create( - domain="mailbox.local", + domain=source_domain, source_type=Source.Type.EMAIL, defaults={ - "name": "Vacaturemailbox", + "name": source_name, "status": Source.Status.ACTIVE, "policy": Source.Policy.ALLOW, "parser_key": "email-alert", - "crawl_interval_minutes": 10, + "crawl_interval_minutes": 1440, }, ) content_hash = hashlib.sha256(raw_message).hexdigest() @@ -52,7 +63,11 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR body_text=raw_message.decode("utf-8", errors="replace"), byte_length=len(raw_message), retain_until=retain_until, - metadata={"mailbox": mailbox}, + metadata={ + "mailbox": mailbox, + "mailbox_connection_id": mailbox_connection.pk if mailbox_connection else None, + "alert_provider": platform or "unknown", + }, ) received_at = None if parsed.get("date"): @@ -64,12 +79,28 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR received_at = None adapter = EmailAlertAdapter() - result = adapter.extract_message(raw_message) + result = adapter.extract_message(raw_message, expected_provider=platform) + detected_provider = next( + ( + str(extracted.raw.get("alert_provider")) + for extracted in result.jobs + if extracted.raw.get("alert_provider") + ), + "unknown", + ) + alert_provider = platform or detected_provider document.parser_key = result.parser_key document.parser_version = result.parser_version document.extraction_confidence = result.confidence + document.metadata = {**document.metadata, "alert_provider": alert_provider} document.save( - update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"] + update_fields=[ + "parser_key", + "parser_version", + "extraction_confidence", + "metadata", + "updated_at", + ] ) links: list[str] = [] errors: list[str] = [] @@ -87,6 +118,7 @@ def ingest_email(raw_message: bytes, *, mailbox: str = "INBOX") -> EmailMessageR except Exception as exc: errors.append(f"{exc.__class__.__name__}: {exc}") return EmailMessageRecord.objects.create( + mailbox_connection=mailbox_connection, message_id=message_id, mailbox=mailbox, sender=str(parsed.get("from") or "")[:500], diff --git a/apps/sources/services/fetcher.py b/apps/sources/services/fetcher.py index 0587a47..9daea56 100644 --- a/apps/sources/services/fetcher.py +++ b/apps/sources/services/fetcher.py @@ -14,6 +14,7 @@ from django.conf import settings from apps.sources.models import Source from .policy import assess_url +from .tls import trusted_tls_context from .url_security import ValidatedUrl, validate_public_url ALLOWED_CONTENT_TYPES = ( @@ -123,6 +124,7 @@ def fetch_url( timeout=httpx.Timeout(settings.FETCHER_TIMEOUT_SECONDS), follow_redirects=False, headers=headers, + verify=trusted_tls_context(), ) current_url = url previous_validation: ValidatedUrl | None = None diff --git a/apps/sources/services/imap_import.py b/apps/sources/services/imap_import.py new file mode 100644 index 0000000..08ea214 --- /dev/null +++ b/apps/sources/services/imap_import.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import imaplib +from collections.abc import Callable +from contextlib import suppress +from typing import Any + +from django.conf import settings + +from apps.sources.models import EmailMessageRecord, MailboxConnection +from apps.sources.services.email_import import ingest_email, message_identity +from apps.sources.services.mailbox_connections import decrypt_mailbox_password +from apps.sources.services.tls import trusted_tls_context +from apps.sources.services.url_security import validate_public_url + +ImapClientFactory = Callable[..., Any] +HostValidator = Callable[[str], object] + + +def poll_imap_mailbox( + connection: MailboxConnection, + *, + client_factory: ImapClientFactory | None = None, + host_validator: HostValidator = validate_public_url, +) -> dict[str, int | str]: + """Import one bounded mailbox batch without letting one bad message abort the batch.""" + if not connection.enabled: + return {"status": "disabled", "imported": 0} + + if client_factory is None: + client_factory = imaplib.IMAP4_SSL + host_validator(f"https://{connection.imap_host}") + password = decrypt_mailbox_password(connection) + client_options = {"timeout": settings.IMAP_CONNECT_TIMEOUT_SECONDS} + if client_factory is imaplib.IMAP4_SSL: + client_options["ssl_context"] = trusted_tls_context() + client = client_factory(connection.imap_host, connection.port, **client_options) + counters = { + "imported": 0, + "duplicates": 0, + "skipped_oversized": 0, + "failed": 0, + } + try: + client.login(connection.username, password) + status, _ = client.select(connection.mailbox, readonly=True) + if status != "OK": + raise RuntimeError("IMAP-mailbox kon niet worden geopend") + status, data = client.uid("search", None, "ALL") + if status != "OK": + raise RuntimeError("IMAP-zoekopdracht mislukt") + uids = (data[0] or b"").split()[-settings.IMAP_MAX_MESSAGES_PER_POLL :] + for uid in uids: + try: + status, payload = client.uid("fetch", uid, "(RFC822)") + if status != "OK" or not payload: + counters["failed"] += 1 + continue + raw = next((part[1] for part in payload if isinstance(part, tuple)), None) + if not isinstance(raw, bytes): + counters["failed"] += 1 + continue + if len(raw) > settings.IMAP_MAX_MESSAGE_BYTES: + counters["skipped_oversized"] += 1 + continue + identity = message_identity(raw) + if EmailMessageRecord.objects.filter( + mailbox_connection=connection, message_id=identity + ).exists(): + counters["duplicates"] += 1 + continue + ingest_email( + raw, + mailbox=connection.mailbox, + mailbox_connection=connection, + platform=connection.platform, + ) + counters["imported"] += 1 + except Exception: + counters["failed"] += 1 + return {"status": "ok", **counters} + finally: + with suppress(Exception): + client.logout() diff --git a/apps/sources/services/mailbox_connections.py b/apps/sources/services/mailbox_connections.py new file mode 100644 index 0000000..49a22a0 --- /dev/null +++ b/apps/sources/services/mailbox_connections.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from datetime import timedelta +from uuid import uuid4 + +from cryptography.fernet import Fernet, InvalidToken, MultiFernet +from django.conf import settings +from django.db import transaction +from django.utils import timezone + +from apps.sources.models import MailboxConnection + + +class MailboxCredentialError(RuntimeError): + """Safe credential configuration error without secret material.""" + + +def _cipher() -> MultiFernet: + keys = settings.MAILBOX_CREDENTIAL_KEYS + if not keys: + raise MailboxCredentialError( + "Credentialopslag is niet geconfigureerd. Stel MAILBOX_CREDENTIAL_KEYS in." + ) + try: + return MultiFernet([Fernet(key.encode("ascii")) for key in keys]) + except (ValueError, TypeError) as exc: + raise MailboxCredentialError( + "MAILBOX_CREDENTIAL_KEYS bevat een ongeldige sleutel." + ) from exc + + +def encrypt_mailbox_password(password: str) -> str: + if not password: + raise MailboxCredentialError("Een app-wachtwoord is verplicht.") + return _cipher().encrypt(password.encode("utf-8")).decode("ascii") + + +def decrypt_mailbox_password(connection: MailboxConnection) -> str: + try: + return _cipher().decrypt(connection.encrypted_password.encode("ascii")).decode("utf-8") + except (InvalidToken, ValueError, UnicodeError) as exc: + raise MailboxCredentialError( + "Het opgeslagen app-wachtwoord kan niet worden ontsleuteld. Sla het opnieuw op." + ) from exc + + +@transaction.atomic +def rotate_mailbox_credentials() -> int: + """Re-encrypt every mailbox password with the first configured key.""" + rotated = 0 + for connection in MailboxConnection.objects.select_for_update().all(): + password = decrypt_mailbox_password(connection) + connection.encrypted_password = encrypt_mailbox_password(password) + connection.save(update_fields=["encrypted_password", "updated_at"]) + rotated += 1 + return rotated + + +@transaction.atomic +def save_mailbox_connection( + *, + user, + cleaned_data: dict[str, object], + instance: MailboxConnection | None = None, +) -> MailboxConnection: + connection = instance or MailboxConnection(user=user) + if connection.pk and connection.user_id != user.pk: + raise PermissionError("Mailboxkoppeling behoort niet tot deze gebruiker.") + + password = str(cleaned_data.pop("password", "") or "") + for field in ( + "platform", + "provider", + "custom_host", + "port", + "username", + "mailbox", + "enabled", + "poll_interval_minutes", + ): + setattr(connection, field, cleaned_data[field]) + if password: + connection.encrypted_password = encrypt_mailbox_password(password) + elif not connection.encrypted_password: + raise MailboxCredentialError("Een app-wachtwoord is verplicht.") + connection.next_poll_at = timezone.now() + connection.last_error_category = "" + connection.last_error_message = "" + connection.full_clean() + connection.save() + return connection + + +@transaction.atomic +def claim_mailbox_connection( + *, connection_id: int, worker_token: str | None = None, force: bool = False +) -> tuple[MailboxConnection, str] | None: + now = timezone.now() + connection = MailboxConnection.objects.select_for_update().get(pk=connection_id) + if not connection.enabled: + return None + if connection.lease_expires_at and connection.lease_expires_at > now: + return None + if not force and connection.next_poll_at > now: + return None + token = worker_token or uuid4().hex + connection.lease_token = token + connection.lease_expires_at = now + timedelta(minutes=5) + connection.save(update_fields=["lease_token", "lease_expires_at", "updated_at"]) + return connection, token + + +@transaction.atomic +def finish_mailbox_poll( + *, + connection_id: int, + worker_token: str, + success: bool, + error_category: str = "", + error_message: str = "", +) -> None: + connection = MailboxConnection.objects.select_for_update().get(pk=connection_id) + if connection.lease_token != worker_token: + return + now = timezone.now() + connection.last_polled_at = now + connection.next_poll_at = now + timedelta(minutes=connection.poll_interval_minutes) + connection.lease_token = "" + connection.lease_expires_at = None + if success: + connection.last_success_at = now + connection.last_error_category = "" + connection.last_error_message = "" + else: + connection.last_error_category = error_category[:80] + connection.last_error_message = error_message[:500] + connection.save( + update_fields=[ + "last_polled_at", + "next_poll_at", + "lease_token", + "lease_expires_at", + "last_success_at", + "last_error_category", + "last_error_message", + "updated_at", + ] + ) diff --git a/apps/sources/services/policy.py b/apps/sources/services/policy.py index da738e8..e2cf4f2 100644 --- a/apps/sources/services/policy.py +++ b/apps/sources/services/policy.py @@ -19,6 +19,10 @@ DEFAULT_DENYLIST = { "stepstone.be", "jobat.be", "vdab.be", + "ictjob.be", + "careerjet.be", + "randstad.be", + "roberthalf.com", } diff --git a/apps/sources/services/robots.py b/apps/sources/services/robots.py index 284f0d8..c8fbbb9 100644 --- a/apps/sources/services/robots.py +++ b/apps/sources/services/robots.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime, timedelta -from urllib.parse import urlsplit, urlunsplit +from urllib.parse import urljoin, urlsplit, urlunsplit import httpx from django.conf import settings @@ -11,6 +11,7 @@ from django.utils import timezone from apps.sources.models import Source, SourceRobotsCache +from .tls import trusted_tls_context from .url_security import UnsafeUrlError, validate_public_url ALLOW = "allow" @@ -123,20 +124,37 @@ def _max_bytes() -> int: def _fetch_robots(origin: str, *, client: httpx.Client | None = None) -> tuple[str, int, str, str]: robots_url = _robots_url(origin) - validate_public_url( - robots_url, - allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False), - ) own_client = client is None http_client = client or httpx.Client( timeout=httpx.Timeout(getattr(settings, "ROBOTS_FETCH_TIMEOUT_SECONDS", 5)), follow_redirects=False, + verify=trusted_tls_context(), ) try: - response = http_client.get( - robots_url, - headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")}, - ) + current_url = robots_url + for _ in range(getattr(settings, "FETCHER_MAX_REDIRECTS", 5) + 1): + validate_public_url( + current_url, + allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False), + ) + response = http_client.get( + current_url, + headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")}, + ) + if response.status_code not in {301, 302, 303, 307, 308}: + break + location = response.headers.get("location") + if not location: + raise httpx.HTTPStatusError( + "Robotsredirect zonder Location-header", + request=response.request, + response=response, + ) + current_url = urljoin(current_url, location) + else: + raise httpx.TooManyRedirects( + "Te veel redirects bij robotscontrole", request=response.request + ) finally: if own_client: http_client.close() diff --git a/apps/sources/services/tls.py b/apps/sources/services/tls.py new file mode 100644 index 0000000..b405b9e --- /dev/null +++ b/apps/sources/services/tls.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import ssl +from functools import lru_cache +from pathlib import Path + +import certifi + + +@lru_cache(maxsize=1) +def trusted_tls_context() -> ssl.SSLContext: + """Build a strict TLS context from the pinned application CA bundle.""" + bundle = Path(certifi.where()).resolve(strict=True) + if not bundle.is_file(): + raise RuntimeError("De vertrouwde CA-bundel ontbreekt.") + return ssl.create_default_context(cafile=str(bundle)) diff --git a/apps/sources/tasks.py b/apps/sources/tasks.py index aaa1b13..b03916a 100644 --- a/apps/sources/tasks.py +++ b/apps/sources/tasks.py @@ -1,7 +1,5 @@ from __future__ import annotations -import imaplib -from contextlib import suppress from datetime import timedelta from uuid import uuid4 @@ -11,8 +9,7 @@ from django.db.models import Q from django.utils import timezone from apps.jobs.services.pipeline import process_raw_document -from apps.sources.models import EmailMessageRecord, RawDocument, Source, SourceRun -from apps.sources.services.email_import import ingest_email, message_identity +from apps.sources.models import MailboxConnection, RawDocument, Source, SourceRun from apps.sources.services.fetcher import ( FetchError, FetchTimeoutError, @@ -25,6 +22,12 @@ from apps.sources.services.health import ( evaluate_source_health, start_health_canary, ) +from apps.sources.services.imap_import import poll_imap_mailbox +from apps.sources.services.mailbox_connections import ( + MailboxCredentialError, + claim_mailbox_connection, + finish_mailbox_poll, +) from apps.sources.services.policy import assess_url from apps.sources.services.scheduling import ( acquire_source_lease, @@ -192,40 +195,63 @@ def schedule_due_sources() -> dict[str, int]: return {"scheduled": len(ids)} -@shared_task(name="apps.sources.tasks.poll_imap_inbox") -def poll_imap_inbox() -> dict[str, int | str]: - if not settings.IMAP_ENABLED: - return {"status": "disabled", "imported": 0} - if not settings.IMAP_HOST or not settings.IMAP_USER or not settings.IMAP_PASSWORD: - return {"status": "not_configured", "imported": 0} - client_cls = imaplib.IMAP4_SSL if settings.IMAP_USE_SSL else imaplib.IMAP4 - client = client_cls(settings.IMAP_HOST, settings.IMAP_PORT) - imported = 0 +@shared_task(name="apps.sources.tasks.schedule_mailbox_polls") +def schedule_mailbox_polls() -> dict[str, int]: + now = timezone.now() + due_ids = list( + MailboxConnection.objects.filter(enabled=True, next_poll_at__lte=now) + .filter(Q(lease_expires_at__isnull=True) | Q(lease_expires_at__lte=now)) + .values_list("id", flat=True)[:100] + ) + for connection_id in due_ids: + poll_mailbox.delay(connection_id) + return {"scheduled": len(due_ids)} + + +@shared_task(bind=True, name="apps.sources.tasks.poll_mailbox") +def poll_mailbox(self, connection_id: int, force: bool = False) -> dict[str, int | str]: + worker_token = str(self.request.id or uuid4()) + claimed = claim_mailbox_connection( + connection_id=connection_id, worker_token=worker_token, force=force + ) + if claimed is None: + return {"status": "skipped", "imported": 0} + connection, token = claimed try: - client.login(settings.IMAP_USER, settings.IMAP_PASSWORD) - status, _ = client.select(settings.IMAP_MAILBOX, readonly=not settings.IMAP_MARK_SEEN) - if status != "OK": - raise RuntimeError("IMAP-mailbox kon niet worden geopend") - status, data = client.uid("search", None, "ALL") - if status != "OK": - raise RuntimeError("IMAP-zoekopdracht mislukt") - uids = (data[0] or b"").split()[-200:] - for uid in uids: - status, payload = client.uid("fetch", uid, "(RFC822)") - if status != "OK" or not payload: - continue - raw = next((part[1] for part in payload if isinstance(part, tuple)), None) - if not raw: - continue - message_id = message_identity(raw) - was_known = EmailMessageRecord.objects.filter(message_id=message_id).exists() - ingest_email(raw, mailbox=settings.IMAP_MAILBOX) - if not was_known: - imported += 1 - return {"status": "ok", "imported": imported} - finally: - with suppress(Exception): - client.logout() + result = poll_imap_mailbox(connection) + finish_mailbox_poll( + connection_id=connection.pk, + worker_token=token, + success=result["status"] == "ok", + ) + return result + except MailboxCredentialError as exc: + finish_mailbox_poll( + connection_id=connection.pk, + worker_token=token, + success=False, + error_category="credentials", + error_message=str(exc), + ) + return {"status": "failed", "error_category": "credentials", "imported": 0} + except Exception as exc: + error_category = "imap" + if exc.__class__.__name__.lower().startswith("unsafe"): + error_category = "host_policy" + finish_mailbox_poll( + connection_id=connection.pk, + worker_token=token, + success=False, + error_category=error_category, + error_message="IMAP-verbinding of authenticatie mislukt.", + ) + return {"status": "failed", "error_category": error_category, "imported": 0} + + +@shared_task(name="apps.sources.tasks.poll_imap_inbox") +def poll_imap_inbox() -> dict[str, int]: + """Compatibility alias for old deployments; it now schedules due per-platform mailboxes.""" + return schedule_mailbox_polls() @shared_task(name="apps.sources.tasks.cleanup_raw_documents") diff --git a/apps/sources/urls.py b/apps/sources/urls.py index a9275d1..1802b72 100644 --- a/apps/sources/urls.py +++ b/apps/sources/urls.py @@ -1,10 +1,23 @@ from django.urls import path -from .views import SourceListView, bulk_candidate_action, manual_import_view, retry_source +from .views import ( + SourceListView, + bulk_candidate_action, + edit_mailbox, + manual_import_view, + retry_source, + save_mailbox, + sync_platform_alerts, + toggle_mailbox, +) urlpatterns = [ path("", SourceListView.as_view(), name="list"), path("manual-import/", manual_import_view, name="manual_import"), + path("mailboxes/add/", save_mailbox, name="mailbox_add"), + path("mailboxes//edit/", edit_mailbox, name="mailbox_edit"), + path("mailboxes//sync/", sync_platform_alerts, name="sync_platform_alerts"), + path("mailboxes//toggle/", toggle_mailbox, name="mailbox_toggle"), path("bulk/", bulk_candidate_action, name="bulk"), path("/retry/", retry_source, name="retry"), ] diff --git a/apps/sources/views.py b/apps/sources/views.py index a34e011..35ba7a6 100644 --- a/apps/sources/views.py +++ b/apps/sources/views.py @@ -4,39 +4,88 @@ from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin +from django.db.models import Count, Q from django.http import HttpRequest from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse +from django.utils import timezone from django.views.decorators.cache import never_cache from django.views.decorators.http import require_POST from django.views.generic import ListView from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure +from apps.profiles.models import SearchProfile -from .forms import ManualImportForm -from .models import Source, SourcePolicyReview +from .forms import MailboxConnectionForm, ManualImportForm +from .models import MailboxConnection, Source, SourcePolicyReview +from .platforms import PLATFORM_ALERTS +from .services.mailbox_connections import MailboxCredentialError, save_mailbox_connection from .services.manual_import import ManualImportError, ManualImportSummary, import_manual_source from .services.policy import create_policy_review -from .tasks import fetch_source +from .tasks import fetch_source, poll_mailbox def _source_list_queryset(): - return Source.objects.prefetch_related("policy_reviews").all().order_by("name") + return ( + Source.objects.prefetch_related("policy_reviews") + .exclude(domain="jobs.example.org", status=Source.Status.DISABLED) + .filter( + Q(metadata__hidden_from_source_list__isnull=True) + | Q(metadata__hidden_from_source_list=False) + ) + .filter(Q(metadata__watchlist__isnull=True) | Q(metadata__watchlist=False)) + .annotate(job_count=Count("job_aliases", distinct=True)) + .order_by("name") + ) -def _manual_import_context(request: HttpRequest, *, form: ManualImportForm, manual_result=None): +def _watchlist_queryset(): + return ( + Source.objects.filter(metadata__watchlist=True) + .filter( + Q(metadata__hidden_from_source_list__isnull=True) + | Q(metadata__hidden_from_source_list=False) + ) + .order_by("name") + ) + + +def _manual_import_context( + request: HttpRequest, + *, + form: ManualImportForm, + manual_result=None, + mailbox_form: MailboxConnectionForm | None = None, + editing_mailbox: MailboxConnection | None = None, +): base_queryset = _source_list_queryset() + watchlist_queryset = _watchlist_queryset() + mailbox_connections = list( + request.user.mailbox_connections.filter(platform__in=PLATFORM_ALERTS) + ) + active_search_profile = SearchProfile.objects.filter(user=request.user, is_active=True).first() return { "sources": base_queryset, "active_source_count": Source.objects.filter(status=Source.Status.ACTIVE).count(), "source_total_count": base_queryset.count(), "review_pending_count": Source.objects.filter(policy=Source.Policy.REVIEW).count(), + "watchlist_sources": watchlist_queryset, + "watchlist_count": watchlist_queryset.count(), "manual_import_form": form, "manual_import_bookmarklet_endpoint": request.build_absolute_uri( reverse("sources:manual_import") ), "manual_import_paste_max_bytes": settings.MANUAL_IMPORT_PASTE_MAX_BYTES, "manual_import_result": manual_result, + "platform_alert_ingress_ready": any( + connection.enabled for connection in mailbox_connections + ), + "mailbox_connections": mailbox_connections, + "platform_alerts": PLATFORM_ALERTS.values(), + "mailbox_form": mailbox_form or MailboxConnectionForm(), + "editing_mailbox": editing_mailbox, + "mailbox_credential_key_ready": bool(settings.MAILBOX_CREDENTIAL_KEYS), + "active_search_profile": active_search_profile, } @@ -108,7 +157,7 @@ def _action_to_review( def bulk_candidate_action(request): action = request.POST.get("action") source_ids = [int(raw_id) for raw_id in request.POST.getlist("source_ids") if raw_id.isdigit()] - sources = Source.objects.filter(id__in=source_ids).exclude(policy=Source.Policy.DENY) + sources = _source_list_queryset().filter(id__in=source_ids).exclude(policy=Source.Policy.DENY) if not sources.exists(): messages.info(request, "Geen bruikbare bronregels geselecteerd.") return redirect("sources:list") @@ -148,6 +197,84 @@ def retry_source(request, pk: int): return redirect("sources:list") +@login_required +@require_POST +def save_mailbox(request, pk: int | None = None): + instance = None + if pk is not None: + instance = get_object_or_404(MailboxConnection, pk=pk, user=request.user) + form = MailboxConnectionForm(request.POST, instance=instance) + if form.is_valid(): + try: + connection = save_mailbox_connection( + user=request.user, + cleaned_data=dict(form.cleaned_data), + instance=instance, + ) + except MailboxCredentialError as exc: + form.add_error("password", str(exc)) + else: + poll_mailbox.delay(connection.pk, force=True) + messages.success( + request, + f"{connection.get_platform_display()}-mailbox opgeslagen; " + "verbindingstest ingepland.", + ) + return redirect("sources:list") + messages.error(request, "Controleer de mailboxinstellingen.") + return render( + request, + "sources/list.html", + _manual_import_context( + request, + form=ManualImportForm(), + mailbox_form=form, + editing_mailbox=instance, + ), + ) + + +@login_required +def edit_mailbox(request, pk: int): + connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user) + if request.method == "POST": + return save_mailbox(request, pk=pk) + return render( + request, + "sources/list.html", + _manual_import_context( + request, + form=ManualImportForm(), + mailbox_form=MailboxConnectionForm(instance=connection), + editing_mailbox=connection, + ), + ) + + +@login_required +@require_POST +def sync_platform_alerts(request, pk: int): + connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user) + if not connection.enabled: + messages.error(request, "Activeer deze mailbox voordat je synchroniseert.") + else: + poll_mailbox.delay(connection.pk, force=True) + messages.success(request, f"{connection.get_platform_display()}-mailimport ingepland.") + return redirect("sources:list") + + +@login_required +@require_POST +def toggle_mailbox(request, pk: int): + connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user) + connection.enabled = not connection.enabled + connection.next_poll_at = timezone.now() + connection.save(update_fields=["enabled", "next_poll_at", "updated_at"]) + state = "geactiveerd" if connection.enabled else "gepauzeerd" + messages.success(request, f"{connection.get_platform_display()}-mailbox {state}.") + return redirect("sources:list") + + @login_required @never_cache def manual_import_view(request): diff --git a/config-data/geodata/README.local.md b/config-data/geodata/README.local.md index 4217eb9..b62ee2a 100644 --- a/config-data/geodata/README.local.md +++ b/config-data/geodata/README.local.md @@ -1,16 +1,56 @@ # Lokale geodata -Plaats hier de door de beheerder aangeleverde geodatabestanden. -Deze bestanden worden niet automatisch meegeleverd in de repository. +De gekozen bron is de Belgische GeoNames Postal Code export. Het bulkbestand blijft lokaal en +wordt niet in Git, de project-ZIP of logs opgenomen. -Aanbevolen bestandsformaat: +## Vastgelegde bron -- `geodata-belgie.csv` (of vergelijkbaar) -- kolommen: `postal_code,municipality,region,latitude,longitude` -- geexporteerd als UTF-8 met komma als scheidingsteken +- Herkomst: `https://download.geonames.org/export/zip/BE.zip` +- Upstreamversie/Last-Modified: `2026-04-03 03:39 CET` +- Download- en reviewdatum: `2026-07-22` +- Licentie: Creative Commons Attribution 4.0 (`https://creativecommons.org/licenses/by/4.0/`) +- Upstream-archief SHA-256: `4c412c8482c5002273e880289adc83ec7d5c37866a5cf5bff893d8b38fb7eefa` +- Afgeleide CSV SHA-256: `7ffddc97b07b4bac8e9aade68bbfbc2730dea5291505489adc97c3992ad16198` +- Afgeleide CSV: 2.781 unieke `postcode + plaatsnaam`-rijen; geen persoonsgegevens +- Benodigde attributie bij distributie of publieke presentatie: `GeoNames (www.geonames.org)` -Importeer met: +GeoNames beschrijft de coördinaten als geschatte WGS84-punten. VacatureRadar gebruikt ze als +postcode-/plaatscentrum, nooit als exact woon- of werkadres. + +## Lokale bestanden + +De gecontroleerde runtime gebruikt: + +- `local/geodata/BE-2026-04-03.zip` +- `local/geodata/geonames-be-2026-04-03.csv` +- CSV-kolommen: `postal_code,municipality,region,latitude,longitude` + +## Gecontroleerde steekproef + +| Postcode | Plaats | Latitude | Longitude | +|---|---|---:|---:| +| 2400 | Mol | 51.1919 | 5.1166 | +| 2440 | Geel | 51.1656 | 4.9892 | +| 2260 | Westerlo | 51.0905 | 4.9154 | +| 2300 | Turnhout | 51.3225 | 4.9447 | +| 2250 | Olen | 51.1439 | 4.8598 | +| 2370 | Arendonk | 51.3227 | 5.0829 | +| 3920 | Lommel | 51.2307 | 5.3135 | + +Valideer en importeer met: ```bash -python manage.py import_geodata config-data/geodata/geodata-belgie.csv --replace +python manage.py import_geodata local/geodata/geonames-be-2026-04-03.csv \ + --source-name geonames-postal-be \ + --dataset-version 2026-04-03 \ + --license-name "Creative Commons Attribution 4.0" \ + --license-url "https://creativecommons.org/licenses/by/4.0/" \ + --validate-only + +python manage.py import_geodata local/geodata/geonames-be-2026-04-03.csv \ + --source-name geonames-postal-be \ + --dataset-version 2026-04-03 \ + --license-name "Creative Commons Attribution 4.0" \ + --license-url "https://creativecommons.org/licenses/by/4.0/" \ + --replace ``` diff --git a/config-data/seed_sources.yaml b/config-data/seed_sources.yaml index b477470..09544b5 100644 --- a/config-data/seed_sources.yaml +++ b/config-data/seed_sources.yaml @@ -1,11 +1,818 @@ -# Alleen expliciet beoordeelde bronnen mogen policy: allow krijgen. -# De demobron is niet bedoeld voor live crawling; fixtures gebruiken hetzelfde domein. +# Publieke vacaturelijsten van werkgevers binnen ongeveer 40 km van postcode 2400 Mol. +# Afstanden gebruiken de lokaal geïmporteerde GeoNames-postcodedataset; vacaturelocaties blijven bronafhankelijk. +disable_demo_sources: true +retire_source_urls: + - https://api.lever.co/v0/postings/deliverect?mode=json + - https://ypto.recruitee.com/api/offers/ + - https://adesso1.recruitee.com/api/offers/ + - https://monizze.recruitee.com/api/offers/ + - https://api.lever.co/v0/postings/Qover?mode=json&location=Brussels + - https://api.lever.co/v0/postings/sfeir?mode=json&location=Bruxelles + - https://api.smartrecruiters.com/v1/companies/ARHS/postings?limit=100&country=be sources: - - name: Voorbeeldwerkgever (fixture) - url: https://jobs.example.org/vacatures/ - type: employer - status: paused - policy: review - policy_reason: Demobron; activeer geen voorbeeld.org-domein voor productie. - parser: auto + - name: VITO + url: https://vito.recruitee.com/api/offers/ + type: ats + status: active + policy: allow + policy_reason: Publieke Recruitee-vacaturefeed van VITO; twee geslaagde livecycli op 2026-07-21. + parser: ats-recruitee + allow_public_endpoint: true crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: VITO + direct_employer: true + employer_homepage: https://vito.be/nl/jobs + public_job_page: https://jobs.vito.be/l/nl + region_scope: Kempen-Limburg + municipality_scope: Mol + country_scope: BE + review: + decision: allow + reason: Publieke werkgeversfeed, toegestane robotsroute en twee geslaagde livecycli. + evidence_link: https://jobs.vito.be/l/nl + expires_at: '2026-10-19T23:59:59+02:00' + notes: Alleen GET op de publieke offers-feed; nooit sollicitatie-endpoints. + metadata: + reviewed_at: '2026-07-21' + robots_url: https://jobs.vito.be/robots.txt + api_documentation: https://docs.recruitee.com/reference/offers + - name: Corda Campus werkgevers + url: https://www.cordacampus.com/jobs/ + type: sitemap + status: active + policy: allow + policy_reason: Publieke regionale vacaturelijst van Corda Campus; twee geslaagde livecycli op 2026-07-22. + parser: regional-corda-campus + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + direct_employer: false + source_scope: regional-employer-hub + region_scope: Limburg + municipality_scope: Hasselt + public_job_page: https://www.cordacampus.com/jobs/ + country_scope: BE + review: + decision: allow + reason: Publieke regionale werkgeverslijst met rechtstreeks gepubliceerde vacaturedetailpagina's. + evidence_link: https://www.cordacampus.com/jobs/ + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen GET op de publieke joblijst; off-domain links worden geweigerd. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://www.cordacampus.com/robots.txt + - name: RPM Europe Logistics Genk + url: https://rpmeuropelogisticsnv.recruitee.com/api/offers/ + type: ats + status: active + policy: allow + policy_reason: Publieke Recruitee-feed van een werkgever in Genk; twee geslaagde livecycli op 2026-07-22. + parser: ats-recruitee + allow_public_endpoint: true + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: RPM Europe Logistics NV + direct_employer: true + employer_homepage: https://rpmeuropelogisticsnv.recruitee.com/ + public_job_page: https://rpmeuropelogisticsnv.recruitee.com/ + region_scope: Limburg + municipality_scope: Genk + country_scope: BE + review: + decision: allow + reason: Publieke werkgeverfeed via de gedocumenteerde Recruitee Offers API. + evidence_link: https://rpmeuropelogisticsnv.recruitee.com/ + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen publieke GET-feed; geen apply-requests of kandidaatdata. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://rpmeuropelogisticsnv.recruitee.com/robots.txt + - name: ACA Group + url: https://acagroup.be/en/jobs + type: employer + status: active + policy: allow + policy_reason: Publieke werkgeverslijst van het Hasseltse hoofdkantoor; twee geslaagde livecycli op 2026-07-22. + parser: regional-local-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: ACA Group + direct_employer: true + employer_homepage: https://acagroup.be/en/ + public_job_page: https://acagroup.be/en/jobs + region_scope: Limburg + municipality_scope: Hasselt + country_scope: BE + review: + decision: allow + reason: Publieke vacaturekaarten op de eigen werkgeverwebsite met same-host detailpagina's. + evidence_link: https://acagroup.be/en/jobs + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen GET; werkgever en lokale kantoorscope zijn vastgelegd, off-domain links worden geweigerd. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://acagroup.be/robots.txt + - name: Xplore Group + url: https://www.xploregroup.be/en/jobs + type: employer + status: active + policy: allow + policy_reason: Publieke werkgeverslijst met expliciete Hasselt-locaties; twee geslaagde livecycli op 2026-07-22. + parser: regional-local-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: Xplore Group + direct_employer: true + public_job_page: https://www.xploregroup.be/en/jobs + region_scope: Limburg + municipality_scope: Hasselt + country_scope: BE + review: + decision: allow + reason: Publieke vacaturekaarten; de adapter houdt alleen kaarten met Hasselt als locatie over. + evidence_link: https://www.xploregroup.be/en/jobs + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen GET en same-host details; kaarten zonder Hasselt worden deterministisch geweigerd. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://www.xploregroup.be/robots.txt + - name: Universiteit Hasselt + url: https://www.uhasselt.be/vacatures + type: employer + status: active + policy: allow + policy_reason: Publieke lokale vacaturelijst van UHasselt; twee geslaagde livecycli op 2026-07-22. + parser: regional-local-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: Universiteit Hasselt + direct_employer: true + public_job_page: https://www.uhasselt.be/vacatures + region_scope: Limburg + municipality_scope: Hasselt-Diepenbeek + country_scope: BE + public_sector: true + review: + decision: allow + reason: Publieke eigen vacaturelijst met lokale same-host detailpagina's. + evidence_link: https://www.uhasselt.be/vacatures + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen GET; ontbrekende robots.txt geldt niet als toestemming maar er zijn geen zichtbare padblokkades. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://www.uhasselt.be/robots.txt + - name: Hogeschool PXL + url: https://ses.pxl.be/ + type: employer + status: active + policy: allow + policy_reason: Publieke lokale vacaturelijst van PXL; twee geslaagde livecycli op 2026-07-22. + parser: regional-local-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: Hogeschool PXL + direct_employer: true + public_job_page: https://ses.pxl.be/ + region_scope: Limburg + municipality_scope: Hasselt-Diepenbeek + country_scope: BE + public_sector: true + review: + decision: allow + reason: Publieke eigen vacaturekaarten met campusinformatie zonder kandidaatlogin. + evidence_link: https://ses.pxl.be/ + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen GET op de lijst; sollicitatie- en kandidaatendpoints worden niet benaderd. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://ses.pxl.be/robots.txt + - name: Ziekenhuis Oost-Limburg + url: https://ziekenhuis-oost-limburg.cvw.io/?lang=nl-BE + type: employer + status: active + policy: allow + policy_reason: Publieke lokale vacaturelijst van ZOL; twee geslaagde livecycli op 2026-07-22. + parser: regional-local-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: Ziekenhuis Oost-Limburg + direct_employer: true + employer_homepage: https://www.zol.be/ + public_job_page: https://ziekenhuis-oost-limburg.cvw.io/?lang=nl-BE + region_scope: Limburg + municipality_scope: Genk-Lanaken-Maaseik + country_scope: BE + public_sector: true + review: + decision: allow + reason: Publieke CVWarehouse-vacaturelijst van ZOL met same-host GET-detailpagina's. + evidence_link: https://ziekenhuis-oost-limburg.cvw.io/?lang=nl-BE + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen publieke GET-lijst; geen login, sollicitatie-endpoint of kandidaatdata. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://ziekenhuis-oost-limburg.cvw.io/robots.txt + - name: SCK CEN + url: https://www.sckcen.be/nl/carriere/vacatures + type: employer + status: active + policy: allow + policy_reason: Publieke eigen vacaturelijst van SCK CEN in postcode 2400 Mol; twee livecycli geslaagd. + parser: regional-mol-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: SCK CEN + direct_employer: true + employer_homepage: https://www.sckcen.be/ + public_job_page: https://www.sckcen.be/nl/carriere/vacatures + region_center: 2400 Mol + municipality_scope: Mol + country_scope: BE + review: + decision: allow + reason: Publieke same-host vacaturekaarten van de werkgever; robots blokkeert het vacaturepad niet. + evidence_link: https://www.sckcen.be/nl/carriere/vacatures + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen GET op de lijst; geen SuccessFactors-sollicitatie- of kandidaatendpoint. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://www.sckcen.be/robots.txt + - name: Cipal Schaubroeck + url: https://cipalschaubroeck.teamtailor.com/jobs + type: employer + status: active + policy: allow + policy_reason: Publieke werkgeverslijst met expliciete Westerlo- en Geel-locaties; twee livecycli geslaagd. + parser: regional-mol-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: Cipal Schaubroeck + direct_employer: true + employer_homepage: https://www.cipalschaubroeck.be/ + public_job_page: https://cipalschaubroeck.teamtailor.com/jobs + region_center: 2400 Mol + municipality_scope: Westerlo-Geel + country_scope: BE + review: + decision: allow + reason: Publieke Teamtailor-lijst; parser bewaart alleen kaarten die Westerlo of Geel expliciet vermelden. + evidence_link: https://cipalschaubroeck.teamtailor.com/jobs + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen publieke GET-lijst; interne, login-, message- en sollicitatieroutes blijven uitgesloten. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://cipalschaubroeck.teamtailor.com/robots.txt + - name: VanRoey + url: https://www.vanroey.be/en/job-overview/ + type: employer + status: active + policy: allow + policy_reason: Publieke IT-vacaturelijst van een werkgever met vestigingen in Turnhout en Geel; twee livecycli geslaagd. + parser: regional-mol-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: VanRoey + direct_employer: true + employer_homepage: https://www.vanroey.be/ + public_job_page: https://www.vanroey.be/en/job-overview/ + region_center: 2400 Mol + municipality_scope: Turnhout-Geel + country_scope: BE + review: + decision: allow + reason: Publieke eigen vacaturelijst met actuele netwerk-, cloud-, Microsoft- en securityrollen. + evidence_link: https://www.vanroey.be/en/job-overview/ + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen de toegestane overzichtspagina wordt gefetcht; robots sluit detailpaden uit en die worden niet opgehaald. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://www.vanroey.be/robots.txt + - name: NTX (Netropolix) + url: https://netropolix.recruitee.com/ + type: employer + status: active + policy: allow + policy_reason: Publieke Recruitee-carrièrepagina van een Kempense IT-werkgever; alleen Geel-kaarten worden verwerkt. + parser: regional-mol-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: NTX (Netropolix) + direct_employer: true + employer_homepage: https://www.netropolix.be/ + public_job_page: https://netropolix.recruitee.com/ + region_center: 2400 Mol + municipality_scope: Geel + country_scope: BE + review: + decision: allow + reason: Publieke werkgeverpagina; parser weigert kaarten zonder expliciete Geel-locatie. + evidence_link: https://netropolix.recruitee.com/ + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen publieke GET-lijst en same-host vacaturelinks; kandidaat- en applyroutes worden niet benaderd. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://netropolix.recruitee.com/robots.txt + - name: Thomas More + url: https://jobpage.cvwarehouse.com/?companyGuid=eab9ca13-ee10-4504-8a87-a785d0b037ef&lang=nl-BE + type: employer + status: active + policy: allow + policy_reason: Publieke Thomas More-vacaturelijst; alleen titels met Geel, Turnhout of Vorselaar worden verwerkt. + parser: regional-mol-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: Thomas More + direct_employer: true + employer_homepage: https://thomasmore.be/nl/jobs + public_job_page: https://jobpage.cvwarehouse.com/?companyGuid=eab9ca13-ee10-4504-8a87-a785d0b037ef&lang=nl-BE + region_center: 2400 Mol + municipality_scope: Geel-Turnhout-Vorselaar + country_scope: BE + public_sector: true + review: + decision: allow + reason: Publieke CVWarehouse-lijst met vaste werkgever-GUID; regio staat expliciet in de bewaarde kaarttitel. + evidence_link: https://thomasmore.be/nl/jobs + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen GET op de lijst; apply-, kandidaat- en andere werkgever-GUIDs worden geweigerd. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://jobpage.cvwarehouse.com/robots.txt + - name: Gemeente Mol + url: https://jobs.gemeentemol.be/rss/vacatures.rss + type: rss + status: active + policy: allow + policy_reason: Officiële publieke vacature-RSS van Gemeente Mol; actuele CA-validatie en twee livecycli geslaagd. + parser: rss-atom + crawl_interval_minutes: 360 + minimum_interval_seconds: 60 + metadata: + employer_name: Gemeente Mol + direct_employer: true + employer_homepage: https://www.gemeentemol.be/ + public_job_page: https://jobs.gemeentemol.be/vacatures + region_center: 2400 Mol + municipality_scope: Mol + country_scope: BE + public_sector: true + review: + decision: allow + reason: Door de werkgever aangeboden RSS-feed; alleen publieke GET en same-host vacaturelinks. + evidence_link: https://jobs.gemeentemol.be/rss/vacatures.rss + expires_at: '2026-10-20T23:59:59+02:00' + notes: TLS valideert met de vastgepinde CA-bundel; robots blokkeert de RSS-route niet. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://jobs.gemeentemol.be/robots.txt + - name: Ziekenhuis Geel + url: https://ziekenhuisgeel.careersite.be/nl/vacatures + type: employer + status: active + policy: allow + policy_reason: Publieke eigen vacaturelijst in Geel; actuele CA-validatie en twee livecycli geslaagd. + parser: regional-kempen-employers + crawl_interval_minutes: 360 + minimum_interval_seconds: 60 + metadata: + employer_name: Ziekenhuis Geel + direct_employer: true + employer_homepage: https://www.ziekenhuisgeel.be/ + public_job_page: https://ziekenhuisgeel.careersite.be/nl/vacatures + region_center: 2400 Mol + municipality_scope: Geel + country_scope: BE + public_sector: true + review: + decision: allow + reason: Publieke werkgeverslijst; adapter accepteert alleen exacte same-host vacaturepaden. + evidence_link: https://ziekenhuisgeel.careersite.be/nl/vacatures + expires_at: '2026-10-20T23:59:59+02:00' + notes: Geen TLS-bypass; actuele certifi-keten valideert. Apply- en controlcenterpaden blijven uitgesloten. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://ziekenhuisgeel.careersite.be/robots.txt + - name: Lokaal bestuur Geel + url: https://geel.hro.be/ + type: employer + status: active + policy: allow + policy_reason: Publieke HRO-vacaturelijst van het lokaal bestuur Geel met exacte vacature-ID's. + parser: regional-kempen-employers + crawl_interval_minutes: 360 + minimum_interval_seconds: 60 + metadata: + employer_name: Lokaal bestuur Geel + direct_employer: true + employer_homepage: https://www.geel.be/ + public_job_page: https://geel.hro.be/ + region_center: 2400 Mol + municipality_scope: Geel + country_scope: BE + public_sector: true + review: + decision: allow + reason: Publieke lijst op de officiële sollicitatiesite; alleen de rootlijst wordt opgehaald. + evidence_link: https://geel.hro.be/ + expires_at: '2026-10-20T23:59:59+02:00' + notes: Private routes en spontane sollicitatie worden niet benaderd. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://geel.hro.be/robots.txt + - name: Stad Turnhout + url: https://jobs.turnhout.be/ + type: employer + status: active + policy: allow + policy_reason: Publieke eigen vacaturelijst van Stad Turnhout met exacte vacature-ID's. + parser: regional-kempen-employers + crawl_interval_minutes: 360 + minimum_interval_seconds: 60 + metadata: + employer_name: Stad Turnhout + direct_employer: true + employer_homepage: https://www.turnhout.be/ + public_job_page: https://jobs.turnhout.be/ + region_center: 2400 Mol + municipality_scope: Turnhout + country_scope: BE + public_sector: true + review: + decision: allow + reason: Publieke lijst van het lokaal bestuur; parser bewaart alleen lokale vacaturekaarten. + evidence_link: https://jobs.turnhout.be/ + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen GET op de rootlijst; private, jobstudent- en sollicitatieroutes worden niet gevolgd. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://jobs.turnhout.be/robots.txt + - name: Group Renotec + url: https://jobs.renotec.be/nl/alle-jobs/ + type: employer + status: active + policy: allow + policy_reason: Publieke werkgeverslijst; alleen kaarten die Geel expliciet als werkregio noemen worden verwerkt. + parser: regional-kempen-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: Group Renotec + direct_employer: true + employer_homepage: https://renotec.be/ + public_job_page: https://jobs.renotec.be/nl/alle-jobs/ + region_center: 2400 Mol + municipality_scope: Geel + country_scope: BE + review: + decision: allow + reason: Publieke vacaturekaarten op de eigen jobsite met zichtbare regionale markers. + evidence_link: https://jobs.renotec.be/nl/alle-jobs/ + expires_at: '2026-10-20T23:59:59+02:00' + notes: Meerdere werfregio's blijven expliciet gelabeld; sollicitatieformulieren worden niet benaderd. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://jobs.renotec.be/robots.txt + - name: Ravago + url: https://ravago.softgarden.io/en/vacancies + type: employer + status: active + policy: allow + policy_reason: Publieke Softgarden-lijst; alleen kaarten met Arendonk of Olen als expliciete locatie. + parser: regional-kempen-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: Ravago + direct_employer: true + employer_homepage: https://ravago.com/en-BE/lifeatravago + public_job_page: https://ravago.softgarden.io/en/vacancies + region_center: 2400 Mol + municipality_scope: Arendonk-Olen + country_scope: BE + review: + decision: allow + reason: Publieke werkgeverslijst met stabiele vacature-ID's en expliciete plaatsvelden. + evidence_link: https://ravago.softgarden.io/en/vacancies + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen de lijst; Softgarden API-, apply-, HR- en loginroutes zijn uitgesloten. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://ravago.softgarden.io/robots.txt + - name: Sanofi Geel + url: https://jobs.sanofi.com/en/belgium + type: employer + status: active + policy: allow + policy_reason: Publieke Belgische werkgeverspagina; parser bewaart alleen kaarten met Geel als exacte locatie. + parser: regional-kempen-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: Sanofi + direct_employer: true + employer_homepage: https://www.sanofi.be/ + public_job_page: https://jobs.sanofi.com/en/belgium + region_center: 2400 Mol + municipality_scope: Geel + country_scope: BE + review: + decision: allow + reason: Publieke werkgeverspagina met lokale jobkaarten en same-host details. + evidence_link: https://jobs.sanofi.com/en/belgium + expires_at: '2026-10-20T23:59:59+02:00' + notes: Search-, talentcommunity- en applyroutes worden niet gecrawld. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://jobs.sanofi.com/robots.txt + - name: DAF Trucks Westerlo + url: https://www.daf.com/nl-nl/werken-bij-daf/vacatures + type: employer + status: active + policy: allow + policy_reason: Publieke DAF-vacaturelijst; alleen kaarten met Westerlo als exacte locatie. + parser: regional-kempen-employers + crawl_interval_minutes: 720 + minimum_interval_seconds: 60 + metadata: + employer_name: DAF Trucks + direct_employer: true + employer_homepage: https://www.daf.be/nl-be/over-daf/werken-bij-daf-trucks + public_job_page: https://www.daf.com/nl-nl/werken-bij-daf/vacatures + region_center: 2400 Mol + municipality_scope: Westerlo + country_scope: BE + review: + decision: allow + reason: Publieke globale werkgeverslijst met deterministische Westerlo-locatiefilter. + evidence_link: https://www.daf.be/nl-be/over-daf/werken-bij-daf-trucks/vacatures + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen lijst-GET; Randstad-, jobalert- en sollicitatieroutes worden niet gevolgd. + metadata: + reviewed_at: '2026-07-22' + robots_url: https://www.daf.com/robots.txt + - name: Johnson & Johnson Beerse + url: https://www.careers.jnj.com/en/locations/emea/belgium/ + type: employer + status: candidate + policy: review + policy_reason: Interessante regionale werkgever; geautomatiseerde toegang gaf een challenge en blijft gepauzeerd. + parser: auto + metadata: + employer_name: Johnson & Johnson + employer_homepage: https://www.jnj.com/ + public_job_page: https://www.careers.jnj.com/en/locations/emea/belgium/ + municipality_scope: Beerse + watchlist: true + watchlist_reason: Grote farmaceutische en digitale werkgever met veel rollen in Beerse; handmatig volgen zolang de publieke pagina automatisering blokkeert. + current_observation: Actuele Beerse-vacatures bevestigd; geen automatische fetch wegens HTTP 403/challenge. + review: &watchlist_review + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://www.careers.jnj.com/en/locations/emea/belgium/ + expires_at: '2026-10-20T23:59:59+02:00' + notes: Geen challenge- of botomzeiling. + metadata: {reviewed_at: '2026-07-22'} + - name: Nike European Logistics Campus + url: https://careers.nike.com/jobs + type: employer + status: candidate + policy: review + policy_reason: Interessante werkgever in Laakdal/Ham; globale dynamische lijst nog niet fail-closed te filteren. + parser: auto + metadata: + employer_name: Nike + public_job_page: https://careers.nike.com/jobs + municipality_scope: Laakdal-Ham + watchlist: true + watchlist_reason: Groot logistiek en technologisch campuswerkgever met operations-, data-, onderhouds- en leidinggevende rollen. + current_observation: Actuele Laakdal-vacatures bevestigd; locatiefilter wordt client-side toegepast. + review: + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://careers.nike.com/jobs + expires_at: '2026-10-20T23:59:59+02:00' + notes: Geen clientfilter of verborgen endpoint omzeilen. + metadata: {reviewed_at: '2026-07-22'} + - name: AZ Turnhout + url: https://jobpage.cvwarehouse.com/?companyGuid=588d29e6-7c84-4f07-a5bb-42d21e04445a&lang=nl-BE + type: employer + status: candidate + policy: review + policy_reason: Regionaal ziekenhuis met veel jobs; sectiegebonden CVWarehouse-lijst vraagt nog adapteruitbreiding. + parser: auto + metadata: + employer_name: AZ Turnhout + public_job_page: https://www.azturnhout.be/werken-bij-az-turnhout + municipality_scope: Turnhout + watchlist: true + watchlist_reason: Grote regionale zorgwerkgever met ook administratie, techniek en IT-ondersteuning. + current_observation: Actuele vacatures bevestigd; secties moeten afzonderlijk fail-closed worden gelezen. + review: + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://www.azturnhout.be/werken-bij-az-turnhout + expires_at: '2026-10-20T23:59:59+02:00' + notes: Geen sollicitatie- of kandidaatendpoint benaderen. + metadata: {reviewed_at: '2026-07-22'} + - name: Cartamundi Turnhout + url: https://cartamundi.com/en/careers/?country=belgium + type: employer + status: candidate + policy: review + policy_reason: Regionale hoofdzetel met publieke carrièrepagina; ATS-grens nog te valideren. + parser: auto + metadata: + employer_name: Cartamundi + public_job_page: https://cartamundi.com/en/careers/?country=belgium + municipality_scope: Turnhout + watchlist: true + watchlist_reason: Internationale productie- en hoofdzetelomgeving met engineering, IT, logistiek en ondersteunende functies. + current_observation: Carrièrepagina en CVWarehouse-ingang bevestigd; lokale vacaturetelling niet stabiel server-side. + review: + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://cartamundi.com/en/careers/?country=belgium + expires_at: '2026-10-20T23:59:59+02:00' + notes: ATS-werkgevergrens eerst valideren. + metadata: {reviewed_at: '2026-07-22'} + - name: Nyrstar Balen/Pelt + url: https://jobs.nyrstar.com/go/BalenPelt-NL-BE/8741302/?locale=nl_BE + type: employer + status: candidate + policy: review + policy_reason: Grote industriële werkgever nabij Mol; momenteel geen lokale kaart in de publieke categoriepagina. + parser: auto + metadata: + employer_name: Nyrstar + public_job_page: https://jobs.nyrstar.com/go/BalenPelt-NL-BE/8741302/?locale=nl_BE + municipality_scope: Balen-Pelt + watchlist: true + watchlist_reason: Interessant voor techniek, automatisatie, operations en infrastructuur. + current_observation: Geen actuele Balen/Pelt-vacature aangetroffen; talent community beschikbaar. + review: + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://jobs.nyrstar.com/go/BalenPelt-NL-BE/8741302/?locale=nl_BE + expires_at: '2026-10-20T23:59:59+02:00' + notes: Lege lokale lijst is geldig; geen globale vacatures importeren. + metadata: {reviewed_at: '2026-07-22'} + - name: Aurubis Olen + url: https://www.aurubis.com/olen/carriere/jobs + type: employer + status: candidate + policy: review + policy_reason: Grote Kempense procesindustrie; publieke pagina toont nu vooral spontane sollicitatie. + parser: auto + metadata: + employer_name: Aurubis Olen + public_job_page: https://www.aurubis.com/olen/carriere/jobs + municipality_scope: Olen + watchlist: true + watchlist_reason: Meer dan zeshonderd medewerkers en functies in engineering, OT, productie, logistiek en IT/process services. + current_observation: Geen passende reguliere Olen-vacature aangetroffen; spontane sollicitatie beschikbaar. + review: + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://www.aurubis.com/olen/carriere/jobs + expires_at: '2026-10-20T23:59:59+02:00' + notes: Spontane sollicitatie wordt niet automatisch benaderd. + metadata: {reviewed_at: '2026-07-22'} + - name: Estée Lauder Oevel + url: https://careers.elcompanies.com/careers + type: employer + status: candidate + policy: review + policy_reason: Regionale productie- en distributiesite; lokale Oevel-filter nog niet betrouwbaar server-side. + parser: auto + metadata: + employer_name: Estée Lauder Companies + public_job_page: https://careers.elcompanies.com/careers + municipality_scope: Oevel-Westerlo + watchlist: true + watchlist_reason: Potentieel interessant voor supply chain, engineering, automatisatie en bedrijfs-IT. + current_observation: Geen actuele Oevel-vacature bevestigd; globale carrièrepagina blijft op de waaklijst. + review: + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://careers.elcompanies.com/careers + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen activeren met eenduidige Oevel-locatiefilter. + metadata: {reviewed_at: '2026-07-22'} + - name: Soudal Turnhout + url: https://jobs.soudalgroup.com/front/en + type: employer + status: candidate + policy: review + policy_reason: Regionale hoofdzetel met eigen HR-site; lokale lijstvorm vraagt nog een dedicated parser. + parser: auto + metadata: + employer_name: Soudal + public_job_page: https://jobs.soudalgroup.com/front/en + municipality_scope: Turnhout + watchlist: true + watchlist_reason: Internationale industriële werkgever met productie, techniek, finance en ondersteunende rollen. + current_observation: Actuele Turnhout-vacatures bevestigd; ATS-route nog niet geactiveerd. + review: + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://jobs.soudalgroup.com/front/en + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen publieke lijst; geen applyroute. + metadata: {reviewed_at: '2026-07-22'} + - name: H.Essers Tessenderlo + url: https://jobs.essers.com/nl-be/ + type: employer + status: candidate + policy: review + policy_reason: Regionale logistieke werkgever; gemengde vestigingslijst vereist lokale filtering. + parser: auto + metadata: + employer_name: H.Essers + public_job_page: https://jobs.essers.com/nl-be/ + municipality_scope: Tessenderlo + watchlist: true + watchlist_reason: Logistiek, sales, procesverbetering en bedrijfs-IT binnen de ruime regio. + current_observation: Actuele Tessenderlo-vacature bevestigd; nog geen fail-closed vestigingsparser. + review: + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://jobs.essers.com/nl-be/ + expires_at: '2026-10-20T23:59:59+02:00' + notes: Alleen activeren met eenduidige Tessenderlo-filter. + metadata: {reviewed_at: '2026-07-22'} + - name: Cegeka Hasselt + url: https://www.cegeka.com/nl-be/jobs/vacatures + type: employer + status: candidate + policy: review + policy_reason: Belangrijke regionale IT-werkgever; kaartlocaties zijn niet op elke serverresponse eenduidig. + parser: auto + metadata: + employer_name: Cegeka + public_job_page: https://www.cegeka.com/nl-be/jobs/vacatures + municipality_scope: Hasselt-Corda Campus + watchlist: true + watchlist_reason: Zeer relevante IT-werkgever met cloud, infrastructuur, security, data en softwarefuncties. + current_observation: Actuele Hasselt/flexibele IT-vacatures en officiële jobalert bevestigd. + review: + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://www.cegeka.com/nl-be/jobs/vacatures + expires_at: '2026-10-20T23:59:59+02:00' + notes: Officiële jobalert blijft voorlopig de veilige route. + metadata: {reviewed_at: '2026-07-22'} + - name: IOK en IOK Afvalbeheer + url: https://www.iok.be/werken-bij-iok-en-iok-afvalbeheer + type: employer + status: candidate + policy: review + policy_reason: Regionale publieke werkgever; actuele vacaturedoelen worden via een gedeelde ingang aangeboden. + parser: auto + metadata: + employer_name: IOK en IOK Afvalbeheer + public_job_page: https://www.iok.be/werken-bij-iok-en-iok-afvalbeheer + municipality_scope: Geel-Kempen + watchlist: true + watchlist_reason: Meer dan vijfhonderd medewerkers in milieu, afval, ruimtelijke ontwikkeling, administratie en techniek. + current_observation: Werkgeverspagina bevestigt regelmatige werving; automatische bronroute nog te isoleren. + review: + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://www.iok.be/werken-bij-iok-en-iok-afvalbeheer + expires_at: '2026-10-20T23:59:59+02:00' + notes: Gedeelde vacaturedoelen eerst isoleren. + metadata: {reviewed_at: '2026-07-22'} + - name: Jessa Ziekenhuis + url: https://werkenbijjessa.be/content/Vacatures/?locale=nl_NL + type: employer + status: candidate + policy: review + policy_reason: Grote werkgever binnen de 40-km-zone; gedeelde ATS-lijst vraagt nog lokale en werkgeversafscherming. + parser: auto + metadata: + employer_name: Jessa Ziekenhuis + public_job_page: https://werkenbijjessa.be/content/Vacatures/?locale=nl_NL + municipality_scope: Hasselt-Herk-de-Stad + watchlist: true + watchlist_reason: Grote zorgwerkgever met expliciete ICT-, administratie-, techniek- en staffuncties. + current_observation: Actuele vacatures bevestigd; kandidaatprofiel en applyroutes blijven buiten scope. + review: + decision: pause + reason: Waaklijst; eerst stabiele publieke lijst en parserbewijs vereist. + evidence_link: https://werkenbijjessa.be/content/Vacatures/?locale=nl_NL + expires_at: '2026-10-20T23:59:59+02:00' + notes: Geen profiel-, login- of applyroute benaderen. + metadata: {reviewed_at: '2026-07-22'} diff --git a/config-data/source_policy.yaml b/config-data/source_policy.yaml index 12b8c1e..6f4f216 100644 --- a/config-data/source_policy.yaml +++ b/config-data/source_policy.yaml @@ -14,6 +14,9 @@ deny_domains: - stepstone.be - jobat.be - vdab.be + - ictjob.be + - eurobrussels.com + - brusselsjobs.com review_on: - login_required - captcha_or_challenge diff --git a/config/settings.py b/config/settings.py index 528d424..b8f4cd8 100644 --- a/config/settings.py +++ b/config/settings.py @@ -1,5 +1,7 @@ from __future__ import annotations +import base64 +import hashlib import os from pathlib import Path from urllib.parse import urlparse @@ -17,6 +19,16 @@ def env_list(name: str, default: str = "") -> list[str]: return [item.strip() for item in os.getenv(name, default).split(",") if item.strip()] +def env_int_range(name: str, default: int, *, minimum: int, maximum: int) -> int: + try: + value = int(os.getenv(name, str(default))) + except ValueError as exc: + raise ImproperlyConfigured(f"{name} moet een geheel getal zijn.") from exc + if not minimum <= value <= maximum: + raise ImproperlyConfigured(f"{name} moet tussen {minimum} en {maximum} liggen.") + return value + + def _validate_production_security( *, debug: bool, @@ -218,10 +230,28 @@ CELERY_WORKER_PREFETCH_MULTIPLIER = 1 CELERY_TASK_TIME_LIMIT = 300 CELERY_TASK_SOFT_TIME_LIMIT = 270 CELERY_TIMEZONE = TIME_ZONE + +IMAP_MAX_MESSAGES_PER_POLL = env_int_range( + "IMAP_MAX_MESSAGES_PER_POLL", 200, minimum=1, maximum=1000 +) +IMAP_MAX_MESSAGE_BYTES = env_int_range( + "IMAP_MAX_MESSAGE_BYTES", 1000000, minimum=1024, maximum=10000000 +) +IMAP_CONNECT_TIMEOUT_SECONDS = env_int_range( + "IMAP_CONNECT_TIMEOUT_SECONDS", 15, minimum=3, maximum=60 +) +MAILBOX_CREDENTIAL_KEYS = env_list("MAILBOX_CREDENTIAL_KEYS") +if DEBUG and not MAILBOX_CREDENTIAL_KEYS: + MAILBOX_CREDENTIAL_KEYS = [ + base64.urlsafe_b64encode( + hashlib.sha256(f"vacatureradar-mailbox:{SECRET_KEY}".encode()).digest() + ).decode("ascii") + ] + CELERY_BEAT_SCHEDULE = { - "poll-imap": { - "task": "apps.sources.tasks.poll_imap_inbox", - "schedule": 600.0, + "schedule-mailbox-polls": { + "task": "apps.sources.tasks.schedule_mailbox_polls", + "schedule": 300.0, }, "schedule-due-sources": { "task": "apps.sources.tasks.schedule_due_sources", @@ -276,15 +306,6 @@ DATA_UPLOAD_MAX_MEMORY_SIZE = int(os.getenv("DATA_UPLOAD_MAX_MEMORY_SIZE", str(8 FILE_UPLOAD_MAX_MEMORY_SIZE = 0 DATA_UPLOAD_MAX_NUMBER_FILES = 0 -IMAP_ENABLED = env_bool("IMAP_ENABLED", False) -IMAP_HOST = os.getenv("IMAP_HOST", "") -IMAP_PORT = int(os.getenv("IMAP_PORT", "993")) -IMAP_USER = os.getenv("IMAP_USER", "") -IMAP_PASSWORD = os.getenv("IMAP_PASSWORD", "") -IMAP_MAILBOX = os.getenv("IMAP_MAILBOX", "INBOX") -IMAP_USE_SSL = env_bool("IMAP_USE_SSL", True) -IMAP_MARK_SEEN = env_bool("IMAP_MARK_SEEN", False) - OLLAMA_ENABLED = env_bool("OLLAMA_ENABLED", False) OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434") OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "") diff --git a/docs/ai/BACKLOG.yaml b/docs/ai/BACKLOG.yaml index 84a134c..9c55277 100644 --- a/docs/ai/BACKLOG.yaml +++ b/docs/ai/BACKLOG.yaml @@ -1,6 +1,6 @@ schema_version: 1 project: VacatureRadar -updated_at: '2026-07-21' +updated_at: '2026-07-22' execution_policy: order: file-order, then first ready task whose dependencies are done no_product_questions: true @@ -961,7 +961,8 @@ tasks: - Dashboard, vacatures, detail, pipeline, profiel, bronnen, systeem en login gebruiken echte backenddata en acties. - Externe auto-apply, nepmetrics, onbewezen werkgeverinzichten en onbetrouwbare activiteitenfeeds zijn niet overgenomen. - Zes doelviewports hebben één h1, geen horizontale overflow en werkende mobiele navigatie. - - Stitch-bronnen, integratieplan, screenshots en verificatierapport zijn traceerbaar maar niet onderdeel van de productiecontainer. + - Stitch-bronnen, integratieplan, screenshots en verificatierapport zijn traceerbaar maar niet onderdeel van de + productiecontainer. verification: - uv run pytest tests/e2e tests/integration/test_views.py tests/integration/test_tasks_and_more_views.py - ./scripts/codex_verify.sh @@ -977,9 +978,341 @@ tasks: completed_at: '2026-07-21' note: Stitch-visuele taal geïntegreerd in CSP-veilige Django-templates; dark/light, responsive shell, pipeline, broninteracties en browservalidatie toegevoegd zonder functionele scope-uitbreiding of fictieve data. +- id: VR-119 + title: Stitch-schermcompositie dominant doorvoeren + status: done + priority: P0 + requirement_ids: + - PR-050 + - PR-051 + - NFR-008 + depends_on: + - VR-118 + summary: Herwerk de cockpit na productfeedback zodat de aangeleverde Stitch-frontend niet alleen de visuele tokens + maar ook de schermcompositie, informatiedichtheid en interactiehiërarchie bepaalt, met behoud van echte Django-data, + toegankelijke server-rendered flows en alle bestaande veiligheidsgrenzen. + acceptance_criteria: + - De gedeelde shell, utilitybar en navigatie volgen aantoonbaar de Stitch-compositie op desktop en mobiel. + - Dashboard, vacatureoverzicht en detail nemen de primaire Stitch-layouts vrijwel volledig over met uitsluitend + aantoonbare databasevelden en bestaande veilige acties. + - Pipeline, profiel, bronnen, systeem en login gebruiken dezelfde dichte cockpit-hiërarchie en componentvormen. + - Lege, fout- en een-itemtoestanden behouden de bedoelde compositie zonder grote onverklaarde lege vlakken. + - Zes doelviewports hebben één h1, geen pagina-brede horizontale overflow en werkende toetsenbordnavigatie. + - Geen auto-apply, externe assets, fictieve metrics of onbewezen intelligence wordt toegevoegd. + verification: + - uv run pytest tests/e2e tests/integration/test_views.py tests/integration/test_tasks_and_more_views.py + - ./scripts/codex_verify.sh + - Browsercontrole op 1440x900, 1280x800, 1024x768, 768x1024, 390x844 en 360x800 + primary_paths: + - apps/core/context_processors.py + - apps/core/views.py + - apps/jobs/views.py + - templates/ + - static/css/app.css + - static/js/app.js + - docs/design/STITCH_INTEGRATION_PLAN.md + - docs/design/validation/ + result: + completed_at: '2026-07-21' + note: Stitch-schermcompositie dominant doorgevoerd in shell, dashboard, vacaturefeed/detail, pipeline, profieleditor, + bronnen en systeem; geverifieerd met 146 tests, 82,02% coverage en browsermatrix op zes viewports, dark/light + en mobiele navigatie. +- id: VR-120 + title: Begeleid zoekprofiel en functionele livebronnen opleveren + status: done + priority: P0 + requirement_ids: + - PR-001 + - PR-002 + - PR-005 + - PR-011 + - PR-017 + - PR-018 + - PR-022 + depends_on: + - VR-119 + summary: Verwerk de functionele productfeedback door thuislocatie tot postcode te vereenvoudigen, technische JSON-invoer + door selectielijsten en checkboxes te vervangen en minimaal twee actuele publieke werkgeversfeeds end-to-end + te valideren. + acceptance_criteria: + - Het profiel vraagt alleen een viercijferige thuispostcode en leidt gemeente/coördinaten af wanneer lokale geodata + eenduidig beschikbaar is. + - Functietitels, skills, contractvormen, werkmodellen en regio's zijn begeleide keuzevelden die bestaande waarden + veilig behouden. + - Minimaal twee echte werkgeverfeeds hebben een actuele policyreview, parserfixtures en twee succesvolle livecycli. + - Demobronnen zijn niet zichtbaar in het normale bronoverzicht en actuele vacatures stromen door naar ranking. + - Profiel-, adapter-, pipeline-, view- en browsertests plus de volledige verificatiegate slagen. + verification: + - uv run pytest tests/integration/test_profile_preferences.py tests/integration/test_seed_sources.py tests/integration/test_pipeline.py + tests/unit/test_ats_adapters.py + - Volg docs/operations/SOURCE_ONBOARDING.md en controleer twee livecycli per geactiveerde bron + - ./scripts/codex_verify.sh + primary_paths: + - apps/profiles/ + - templates/profiles/ + - apps/sources/ + - config-data/seed_sources.yaml + - docs/operations/SOURCE_ONBOARDING.md + result: + completed_at: '2026-07-21' + note: Postcode-only profiel, begeleide checkboxkeuzes, gereviewde VITO- en Deliverect-feeds met twee livecycli, + employeridentiteit, broninventaris en robotsredirecthardening; 155 tests geslaagd, 2 optioneel overgeslagen, + 82,94% coverage en volledige codex_verify-gate groen. +- id: VR-121 + title: Regionale werkgevers- en platformalertdekking uitbreiden + status: done + priority: P0 + requirement_ids: + - PR-011 + - PR-012 + - PR-017 + - PR-018 + - PR-032 + depends_on: + - VR-120 + summary: Breid de livebroncatalogus uit met publieke werkgeversfeeds in Limburg en aangrenzende Belgische werkregio's, + maak gedeelde ATS-hosts first-class en documenteer de veilige alertingress voor VDAB en Indeed. + acceptance_criteria: + - Minimaal acht bijkomende publieke bronnen hebben een actuele policyreview, parserdekking en twee succesvolle + livecycli voordat ze actief worden. + - De catalogus bevat aantoonbaar regionale dekking voor Hasselt/Genk en aanvullende IT-werkgevers in Brussel, + Gent of Leuven. + - Meerdere werkgevers op één publieke ATS-host kunnen naast elkaar bestaan zonder bronrecords te overschrijven. + - Corda Campus-vacaturekaarten worden deterministisch uitgelezen en off-domain links worden verworpen. + - Indeed en VDAB worden uitsluitend via officiële vacaturealerts/mailimport of handmatige import gebruikt en blijven + op de directe-fetchdenylist. + verification: + - uv run pytest tests/unit/test_regional_adapters.py tests/unit/test_ats_adapters.py tests/unit/test_source_models.py + tests/integration/test_seed_sources.py + - Voer twee livecycli uit voor iedere nieuwe bron en controleer source health, parserwarnings en inventaristelling + - ./scripts/codex_verify.sh + primary_paths: + - apps/sources/ + - config-data/seed_sources.yaml + - fixtures/pages/corda-campus-jobs.html + - fixtures/ats/smartrecruiters-public-postings.json + - docs/operations/LIVE_SOURCE_CATALOG.md + result: + completed_at: '2026-07-21' + note: Acht extra gereviewde livebronnen, gedeelde ATS-hostidentiteit, Corda-adapter en veilige VDAB/Indeed-alertingress + toegevoegd; twee livecycli per bron, 159 tests, 82,91% coverage en in-app-browsercontrole geslaagd. +- id: VR-122 + title: Dagelijkse officiële VDAB-ingang operationaliseren + status: done + priority: P0 + requirement_ids: + - PR-012 + - PR-016 + - PR-018 + - PR-064 + depends_on: + - VR-121 + summary: Verwerk officiële VDAB-jobalerts eenmaal per dag via de bestaande dedicated mailbox, met herkenbare provenance, + begrensde verwerking en een duidelijke route naar de contractueel toegankelijke VDAB Vacature API. + acceptance_criteria: + - De IMAP-ingang draait standaard eenmaal per dag op een configureerbaar tijdstip en blijft idempotent bij replay. + - VDAB-alertmails worden als VDAB-provenance herkend en vacatures stromen zonder automatische linkfetch door de + bestaande normalisatie-, dedupe- en rankingpipeline. + - De mailboxpoll is begrensd op aantal en berichtgrootte; oversized of onleesbare berichten stoppen de batch niet. + - De bronpagina toont de dagelijkse planning, configuratiestatus, laatste VDAB-import en het onderscheid tussen + jobalertingang en de officiële API waarvoor VDAB-goedkeuring vereist is. + - Directe VDAB-scraping, platformlogin en niet-goedgekeurde API-toegang blijven geblokkeerd en dit is getest en + operationeel gedocumenteerd. + verification: + - uv run pytest tests/unit/test_email_adapter.py tests/integration/test_email_import_notifications.py tests/integration/test_tasks_and_more_views.py + - ./scripts/codex_verify.sh + primary_paths: + - apps/sources/adapters/email_alert.py + - apps/sources/services/email_import.py + - apps/sources/services/imap_import.py + - apps/sources/tasks.py + - config/settings.py + - templates/sources/list.html + - docs/architecture/adrs/0005-email-alerts-as-platform-ingress.md + result: + completed_at: '2026-07-21' + note: Dagelijkse configureerbare VDAB-jobalertimport met expliciete provenance, begrensde foutgeisoleerde IMAP-poll, + beveiligde handmatige inplanning en heldere officiele API-route; 164 tests, 2 optioneel overgeslagen, 83,55% + coverage en desktop/390px Codex-browsercontrole groen. +- id: VR-123 + title: Platformmailboxen veilig in de UI koppelen en frequent synchroniseren + status: done + priority: P0 + requirement_ids: + - PR-012 + - PR-016 + - PR-018 + - PR-064 + depends_on: + - VR-122 + summary: Vervang de globale dagelijkse IMAP-configuratie door gebruikersgescopeerde mailboxkoppelingen per vacatureplatform, + met versleutelde app-wachtwoorden en een eigen frequent synchronisatie-interval. + acceptance_criteria: + - Een ingelogde gebruiker kan voor VDAB en Indeed afzonderlijk een Gmail-, Outlook- of aangepaste IMAP-mailbox + koppelen, wijzigen, pauzeren en direct laten synchroniseren. + - App-wachtwoorden worden met een afzonderlijke roteerbare sleutel versleuteld opgeslagen, nooit teruggetoond + en nooit in logs of taakresultaten opgenomen. + - Iedere mailbox heeft een interval van 5 tot 360 minuten; de scheduler plant alleen verschuldigde actieve mailboxen + en is concurrency- en replayveilig. + - Mailimport bewaart mailbox- en platformprovenance, is per mailbox idempotent en laat een fout in één mailbox + de andere mailboxen niet blokkeren. + - Alle mutaties zijn login-, user-scope- en CSRF-beveiligd; negatieve tests dekken cross-user toegang, ontbrekende + encryptiesleutel, ongeldige host/poort en secretlekken. + - Bronpagina, runbook, threat model, configuratie en migratiedocumentatie zijn bijgewerkt; directe platformfetch + en platformlogin blijven uitgesloten. + verification: + - uv run pytest tests/unit/test_mailbox_credentials.py tests/integration/test_mailbox_connections.py tests/integration/test_email_import_notifications.py + - uv run python manage.py makemigrations --check --dry-run + - ./scripts/codex_verify.sh + primary_paths: + - apps/sources/models.py + - apps/sources/forms.py + - apps/sources/services/mailbox_connections.py + - apps/sources/services/imap_import.py + - apps/sources/tasks.py + - apps/sources/views.py + - templates/sources/list.html + - config/settings.py + result: + completed_at: '2026-07-22' + note: 'Per-platform VDAB/Indeed-mailboxen met versleutelde app-wachtwoorden, intervallen van 5-360 minuten, + leases, user-scoped UI, securitytests en documentatie; volledige gate: 178 passed, 2 skipped, 83.56%.' +- id: VR-124 + title: LinkedIn en Belgische jobboards via veilige jobalertmail toevoegen + status: done + priority: P0 + requirement_ids: + - PR-012 + - PR-016 + - PR-021 + - NFR-001 + - NFR-003 + depends_on: + - VR-123 + summary: Breid de platformmailboxingang uit met LinkedIn, ictjob.be, Jobat, StepStone, EuroBrussels en BrusselsJobs + en beperk iedere parserrun tot vacaturelinks van het expliciet gekozen jobboard. + acceptance_criteria: + - De mailbox-UI biedt naast VDAB en Indeed ook LinkedIn, ictjob.be, Jobat, StepStone, EuroBrussels en BrusselsJobs + met een actuele officiele jobalertlink en dezelfde afzonderlijke mailbox-, interval-, pauze- en syncflow. + - Gesaneerde fixtures en adaptertests bewijzen per nieuw platform titel-, link- en provenance-extractie zonder + live netwerkcall. + - Een platformgebonden mail importeert alleen HTTPS-vacaturelinks op het verwachte jobboarddomein; externe, tracking-, + account-, privacy- en uitschrijflinks worden genegeerd. + - Mailboxrecords, bronprovenance, idempotentie, user scoping, encryptie, leases en foutisolatie blijven voor alle + platformkeuzes behouden. + - Bronbeleid, catalogus, threat model, onboarding, traceability en UI-copy vermelden dat directe platformlogin + en jobboardscraping uitgesloten blijven. + verification: + - uv run pytest tests/unit/test_email_adapter.py tests/integration/test_email_import_notifications.py tests/integration/test_mailbox_connections.py + - uv run python manage.py makemigrations --check --dry-run + - ./scripts/codex_verify.sh + primary_paths: + - apps/sources/models.py + - apps/sources/adapters/email_alert.py + - apps/sources/services/email_import.py + - templates/sources/list.html + - fixtures/emails/ + - docs/operations/LIVE_SOURCE_CATALOG.md + result: + completed_at: '2026-07-22' + note: LinkedIn, ictjob.be, Jobat, StepStone, EuroBrussels en BrusselsJobs toegevoegd aan platformmailboxen; + domeingebonden HTTPS-parser, zes fixtures, denylist, UI en docs; 192 passed, 2 skipped, 83.61%, desktop/390px + browser groen. +- id: VR-125 + title: Brusselse dekking vervangen door regionale werkgevers en Belgische jobplatformen + status: done + priority: P0 + requirement_ids: + - PR-011 + - PR-012 + - PR-017 + - PR-018 + - PR-021 + - NFR-001 + - NFR-003 + depends_on: + - VR-124 + summary: Verwijder de expliciete Brusselse jobboards en werkgevers uit de actieve radar, voeg brede Belgische + jobalertplatformen toe en activeer veilige publieke vacaturelijsten van werkgevers rond Hasselt-Genk. + acceptance_criteria: + - EuroBrussels en BrusselsJobs zijn niet meer configureerbaar of actief; bestaande koppelingen worden veilig uitgeschakeld + en verborgen zonder historische provenance te verwijderen. + - Careerjet, Randstad en Robert Half zijn beschikbaar via hun officiele e-mailalerts en blijven op de directe-fetchdenylist. + - Werkgeversbronnen buiten de regionale scope worden gedeactiveerd en verborgen; bestaande vacaturehistoriek blijft + bewaard. + - Minstens vier lokale werkgeverspagina's uit de kern Hasselt-Genk worden met host- en padgebonden adapters, gesaneerde + fixtures, actuele bronreview en twee succesvolle livecycli toegevoegd. + - Tot een thuispostcode is ingevuld, vermeldt de catalogus expliciet dat Hasselt-Genk als omkeerbaar centrum voor + de 40-km-selectie is gebruikt; ranking blijft de exacte profielafstand toepassen zodra die bekend is. + - Bronbeleid, threat model, onboarding, traceability en gebruikerscopy blijven duidelijk over e-mailingress, robots, + SSRF, off-domain afwijzing en het verbod op platformlogin en scraping. + verification: + - .venv/Scripts/python.exe -m pytest tests/unit/test_regional_adapters.py tests/unit/test_email_adapter.py tests/integration/test_seed_sources.py + tests/integration/test_email_import_notifications.py + - .venv/Scripts/python.exe manage.py makemigrations --check --dry-run + - ./scripts/codex_verify.sh + primary_paths: + - apps/sources/adapters/regional.py + - apps/sources/platforms.py + - apps/sources/management/commands/seed_sources.py + - config-data/seed_sources.yaml + - fixtures/pages/ + - fixtures/emails/ + - docs/operations/LIVE_SOURCE_CATALOG.md + result: + completed_at: '2026-07-22' + note: Brusselse mailboxen en zeven buitenregionale bronnen veilig geretired; Careerjet, Randstad en Robert Half + toegevoegd; ACA, Xplore, UHasselt, PXL en ZOL na host/padreview en twee livecycli actief; 200 passed, 2 skipped, + 83.90%, browser desktop/390px groen. +- id: VR-126 + title: Werkgeversradar herankeren op postcode 2400 Mol + status: done + priority: P0 + requirement_ids: + - PR-002 + - PR-011 + - PR-017 + - PR-018 + - PR-021 + - NFR-001 + - NFR-003 + depends_on: + - VR-125 + summary: Vervang de tijdelijke Hasselt-Genk-aanname door postcode 2400 Mol en voeg veilige publieke vacaturelijsten + toe van relevante werkgevers binnen ongeveer 40 km, met nadruk op infrastructuur-, systeem- en netwerkfuncties. + acceptance_criteria: + - De actieve profielruntime bewaart postcode 2400 en een maximale afstand van 40 km; ontbrekende gelicenseerde + coördinatendata blijft expliciet zichtbaar als extern restpunt en wordt niet stil ingevuld. + - De broncatalogus en UI noemen Mol als centrum en maken onderscheid tussen geografisch gereviewde bronnen en + exacte vacatureafstand die pas na geocoding kan worden berekend. + - Minstens vijf werkgevers uit Mol, Geel, Westerlo of Turnhout met regelmatig gepubliceerde eigen vacatures worden + host- en padgebonden toegevoegd, inclusief relevante IT-/infrastructuurwerkgevers. + - Iedere nieuwe bron heeft een gesaneerde fixture, positieve adaptertest, off-domain/route-negatie en minstens + twee geslaagde livecycli zonder parserwarnings voor activering. + - Bestaande regionale bronnen blijven alleen actief wanneer hun primaire vestiging of vacaturelocatie indicatief + binnen 40 km van Mol valt; historische vacatures en provenance worden bij retirement behouden. + - Bronbeleid, threat model, livecatalogus, onboarding, traceability en gebruikerscopy documenteren robots-, SSRF-, + sanitization- en locatiegrenzen; login, sollicitatie-endpoints en platformscraping blijven uitgesloten. + verification: + - .venv/Scripts/python.exe -m pytest tests/unit/test_regional_adapters.py tests/integration/test_seed_sources.py + tests/integration/test_profile_preferences.py tests/security/test_policy.py + - .venv/Scripts/python.exe manage.py makemigrations --check --dry-run + - ./scripts/codex_verify.sh + primary_paths: + - apps/sources/adapters/regional.py + - apps/sources/management/commands/seed_sources.py + - config-data/seed_sources.yaml + - fixtures/pages/ + - templates/sources/list.html + - docs/operations/LIVE_SOURCE_CATALOG.md + result: + completed_at: '2026-07-22' + note: Profielruntime op postcode 2400 en 40 km gezet; SCK CEN, Cipal Schaubroeck, VanRoey, NTX en Thomas More + met fail-closed Mol-regioadapter, fixtures, docs en twee livecycli geactiveerd; 206 passed, 2 skipped, 84.12%, + browser desktop/360px groen. - id: VR-201 title: Concrete live-bronnen inventariseren en juridisch/operationeel reviewen - status: blocked-external + status: done priority: P0 requirement_ids: - PR-011 @@ -1001,8 +1334,11 @@ tasks: - config-data/seed_sources.local.yaml - Source admin - docs/operations/SOURCE_ONBOARDING.md - external_blocker: Vereist persoonlijke werkgeverskeuzes en een actuele menselijke bronvoorwaardenreview. Commit - lokale bronlijst niet wanneer die persoonlijke informatie bevat. + result: + completed_at: '2026-07-22' + note: Gemeente Mol en zeven Kempen-werkgevers na actuele menselijke review, fail-closed fixtures/tests en twee + livecycli geactiveerd; twaalf interessante regionale werkgevers als niet-scanbare waaklijst toegevoegd; TLS + gebruikt strikt de gepinde certifi-bundel zonder bypass. - id: VR-202 title: Live IMAP- en SMTP-credentials configureren status: blocked-external @@ -1015,18 +1351,23 @@ tasks: depends_on: - VR-111 - VR-115 - summary: Configureer een dedicated vacaturemailbox, app-password en digestrecipient; test read-only import en - mailverzending zonder credentials in Git/logs. + - VR-123 + - VR-124 + summary: Configureer de gewenste afzonderlijke platformmailboxen via de beveiligde UI, de Fernet-secretkey en + digestrecipient; test read-only import en mailverzending zonder credentials in Git/logs. acceptance_criteria: - - IMAP gebruikt TLS, dedicated mailbox en mark-seen standaard uit tijdens acceptatie. - - Dezelfde live testmail importeert idempotent. + - Iedere platformmailbox gebruikt TLS, een dedicated account/app-wachtwoord en read-only mailboxselectie. + - Dezelfde live testmail importeert per mailbox idempotent en alle geconfigureerde intervallen worden door de + scheduler gerespecteerd. - SMTP verstuurt één testdigest en outboxstatus klopt. - - Credentials zijn geroteerbaar en alleen in productie-secretstore/.env aanwezig. + - De encryptiesleutel staat alleen in de productie-secretstore/.env; app-wachtwoorden zijn als ciphertext in de + database aanwezig en sleutelrotatie is live getest. verification: - Volg Unraid mailboxchecklist - Inspecteer logs op afwezigheid van secret/payload primary_paths: - /mnt/user/appdata/vacatureradar/config/.env + - Bronnen -> Platformmailboxen external_blocker: Vereist mailbox-/SMTP-account en app-password van de gebruiker. - id: VR-203 title: Productiedomein, TLS en reverse proxy activeren @@ -1078,7 +1419,7 @@ tasks: external_blocker: Vereist registrykeuze, account en credentials. - id: VR-205 title: Gelicenseerde Belgische geodatabron laden - status: blocked-external + status: done priority: P1 requirement_ids: - PR-002 @@ -1098,7 +1439,11 @@ tasks: primary_paths: - lokale geodatafile - config-data/geodata/README.local.md - external_blocker: Vereist een actuele datasetkeuze/licentiecheck; bulkdata hoort niet automatisch in de project-ZIP. + result: + completed_at: '2026-07-22' + note: GeoNames België postcodedataset 2026-04-03 onder CC BY 4.0 gereviewd, checksums vastgelegd, 2781 unieke + rijen gevalideerd en 5562 lookuprecords geïmporteerd; 2400 Mol opgelost naar 51.1919, 5.1166 en actieve profielscores + herberekend. - id: VR-301 title: Multi-tenant SaaS, teams en billing status: deferred diff --git a/docs/ai/PROJECT_STATE.md b/docs/ai/PROJECT_STATE.md index a713522..b8db429 100644 --- a/docs/ai/PROJECT_STATE.md +++ b/docs/ai/PROJECT_STATE.md @@ -1,7 +1,7 @@ # Projectstatus -- Laatst bijgewerkt: 2026-07-21 -- Repositoryversie: 0.2.1 Stitch Intelligence Cockpit +- Laatst bijgewerkt: 2026-07-22 +- Repositoryversie: 0.2.10 Kempen-werkgeversradar en GeoNames - Uitvoeringsmodus: autonome backlog - Eerste uitvoerbare taak: controleer met `python scripts/backlog.py next` @@ -41,6 +41,93 @@ VR-118-herverificatie op 2026-07-21: - taakledger geldig met 36 taken: 28 done, 5 blocked-external, 3 deferred en 0 ready; - geïntegreerde browservalidatie op zes viewports en beide thema's geslaagd. +VR-119-herverificatie op 2026-07-21: + +- 146 geslaagde tests en 2 optioneel overgeslagen pytest-Playwrightvarianten; uitgebreide HTML/a11y-fallback geslaagd; +- 82,02% branch-aware codedekking; +- Ruff, Django system check, migratiecontrole, taakledger en repositoryvalidatie geslaagd; +- geïntegreerde browsercontrole op alle kernroutes, zes doelviewports, dark/light en mobiele drawer geslaagd zonder pagina-overflow of consolefouten; +- taakledger geldig met 37 taken: 29 done, 5 blocked-external, 3 deferred en 0 ready. + +VR-120-herverificatie op 2026-07-21: + +- 155 geslaagde tests en 2 optioneel overgeslagen pytest-Playwrightvarianten; HTML/a11y-fallback en in-app-browsercontrole geslaagd; +- 82,94% branch-aware codedekking; +- Ruff, Django system check, migratiecontrole, taakledger en repositoryvalidatie geslaagd via de volledige `scripts/codex_verify.sh`-gate; +- VITO leverde 37 vacatures in twee geslaagde cycli; Deliverect leverde 26 vacatures en verwerkte de tweede cyclus correct als HTTP 304; +- taakledger geldig met 38 taken: 30 done, 5 blocked-external, 3 deferred en 0 ready. + +VR-121-herverificatie op 2026-07-22: + +- 159 geslaagde tests en 2 optioneel overgeslagen pytest-Playwrightvarianten; HTML/a11y-fallback geslaagd; +- 82,91% branch-aware codedekking; +- Ruff, Django system check, migratiecontrole en repositoryvalidatie geslaagd via `scripts/codex_verify.sh`; +- acht nieuwe bronnen passeerden elk twee livecycli zonder parserwarnings; de lokale inventaris telt 10 actieve bronnen en 145 actieve vacatures; +- in-app-browsercontrole bevestigde 10 bronkaarten, de VDAB/Indeed-alertingress, geen horizontale overflow en geen consolefouten; +- taakledger bevat 39 taken; na afronding van VR-121 blijven alleen 5 `blocked-external` en 3 `deferred` taken over. + +VR-122-herverificatie op 2026-07-22: + +- 164 geslaagde tests en 2 optioneel overgeslagen pytest-Playwrightvarianten; HTML/a11y-fallback geslaagd; +- 83,55% branch-aware codedekking; +- Ruff, Django system check, migratiecontrole, taakledger en repositoryvalidatie geslaagd via `scripts/codex_verify.sh`; +- de dagelijkse mailboxingang is configureerbaar, idempotent en begrensd; VDAB-mails krijgen expliciete provenance zonder platformlinks automatisch te volgen; +- Codex-browsercontrole bevestigde de VDAB-/API-uitleg, desktop en 390 px zonder horizontale overflow en zonder consolefouten; +- taakledger bevat 40 taken; na afronding van VR-122 blijven alleen 5 `blocked-external` en 3 `deferred` taken over. + +VR-123-herverificatie op 2026-07-22: + +- 178 geslaagde tests en 2 optioneel overgeslagen pytest-Playwrightvarianten; HTML/a11y-fallback geslaagd; +- 83,56% branch-aware codedekking; +- Ruff, Django system check, migratiecontrole, taakledger en repositoryvalidatie geslaagd via `scripts/codex_verify.sh`; +- VDAB en Indeed hebben afzonderlijke gebruikersgescopeerde mailboxkoppelingen met eigen provider, account, app-wachtwoord en interval van 5 tot 360 minuten; +- app-wachtwoorden zijn versleuteld met roteerbare MultiFernet-sleutels; schedulerleases, per-mailbox-idempotentie, SSRF-validatie en foutisolatie zijn regressiegedekt; +- Codex-browsercontrole bevestigde de platformmailboxformulieren, desktop en 390 px zonder horizontale overflow en zonder consolefouten; +- taakledger bevat 41 taken; na afronding van VR-123 blijven alleen 5 `blocked-external` en 3 `deferred` taken over. + +VR-124-herverificatie op 2026-07-22: + +- 192 geslaagde tests en 2 optioneel overgeslagen pytest-Playwrightvarianten; HTML/a11y-fallback geslaagd; +- 83,61% branch-aware codedekking; +- Ruff, Django system check, migratiecontrole, taakledger en repositoryvalidatie geslaagd via `scripts/codex_verify.sh`; +- platformmailboxen ondersteunen naast VDAB en Indeed ook LinkedIn, ictjob.be, Jobat, StepStone, EuroBrussels en BrusselsJobs met afzonderlijke accounts, intervallen en status; +- zes gesaneerde jobboardfixtures bewijzen platformprovenance en de afwijzing van externe, HTTP-, tracking-, account-, help-, privacy- en uitschrijflinks; +- alle acht jobboardgroepen staan op de directe-fetchdenylist; platformlogin en scraping blijven uitgesloten; +- Codex-browsercontrole bevestigde acht keuzelijstopties en alertkaarten, desktop en 390 px zonder horizontale overflow en zonder consolefouten; +- taakledger bevat 42 taken; na afronding van VR-124 blijven alleen 5 `blocked-external` en 3 `deferred` taken over. + +VR-125-herverificatie op 2026-07-22: + +- 200 geslaagde tests en 2 optioneel overgeslagen pytest-Playwrightvarianten; HTML/a11y-fallback geslaagd; +- 83,90% branch-aware codedekking; +- Ruff, Django system check, migratiecontrole, taakledger en repositoryvalidatie geslaagd via `scripts/codex_verify.sh`; +- EuroBrussels en BrusselsJobs zijn uit de configureerbare mailboxcatalogus verwijderd; Careerjet, Randstad en Robert Half gebruiken dezelfde versleutelde platformmailboxflow; +- Deliverect, YPTO, adesso, Monizze, Qover, SFEIR en ARHS zijn buiten de regionale scope uitgeschakeld en verborgen zonder vacaturehistoriek of provenance te wissen; +- ACA Group, Xplore Group, UHasselt, PXL en ZOL passeerden ieder twee livecycli zonder parserwarnings; de acht zichtbare actieve bronnen bevatten 162 vacature-aliassen; +- Codex-browsercontrole bevestigde exact acht regionale bronkaarten, negen platformkeuzes, geen Brusselse kaarten, desktop en 390 px zonder horizontale overflow en zonder consolefouten; +- taakledger bevat 43 taken; na afronding van VR-125 blijven alleen 5 `blocked-external` en 3 `deferred` taken over. + +VR-126-herverificatie op 2026-07-22: + +- 206 geslaagde tests en 2 optioneel overgeslagen pytest-Playwrightvarianten; HTML/a11y-fallback geslaagd; +- 84,12% branch-aware codedekking; +- Ruff, Django system check, migratiecontrole, taakledger en repositoryvalidatie geslaagd via `scripts/codex_verify.sh`; +- het actieve profiel bewaart postcode 2400 en 40 km; ontbrekende gelicenseerde coördinatendata blijft zichtbaar en wordt niet geschat; +- SCK CEN, Cipal Schaubroeck, VanRoey, NTX en Thomas More passeerden twee livecycli zonder parserwarnings; de dertien actieve bronnen bevatten 210 vacature-aliassen; +- Codex-browsercontrole bevestigde 13 bronkaarten, de Mol/40-km-status, desktop en 360 px zonder horizontale overflow en zonder consolefouten; +- taakledger bevat 44 taken; na afronding van VR-126 blijven alleen 5 `blocked-external` en 3 `deferred` taken over. + +VR-201/VR-205-herverificatie op 2026-07-22: + +- 215 geslaagde tests en 2 optioneel overgeslagen pytest-Playwrightvarianten; HTML/a11y-fallback geslaagd; +- 84,42% branch-aware codedekking; +- Ruff, Django system check, migratiecontrole, taakledger en repositoryvalidatie geslaagd via `scripts/codex_verify.sh`; +- GeoNames België 2026-04-03 is onder CC BY 4.0 met checksums gedocumenteerd, 2.781 unieke invoerrijen zijn gevalideerd en 2400 Mol is opgelost naar het benaderende postcodecentrum 51.1919, 5.1166; +- Gemeente Mol en zeven bijkomende Kempen-werkgevers passeerden elk twee livecycli zonder parserwarnings; samen leverden zij 53 kaarten per cyclus; +- twaalf regionale werkgevers staan afzonderlijk als niet-scanbare waaklijst in de UI, ook wanneer zij momenteel geen vacature hebben; +- Codex-browsercontrole bevestigde 21 actieve bronnen, 12 waaklijstkaarten, de GeoNames-status, één `h1`, geen horizontale overflow op desktop of 390 px en geen consolefouten; +- taakledger bevat 44 taken: 38 `done`, 3 `blocked-external`, 3 `deferred` en 0 `ready`. + ## Deploymentstatus - `main` is via de bestaande Gitea-SSH-sleutel naar `NuklearRabbit/VacatureRadar` gepusht. @@ -51,6 +138,71 @@ VR-118-herverificatie op 2026-07-21: ## Laatste uitgevoerde backlogtaak +- 2026-07-22: `VR-201` en `VR-205` voltooid. + - De officiële GeoNames-postcodedataset lost postcode 2400 op naar Mol en levert indicatieve postcodecentrumafstanden; licentie, versie, reviewdatum en twee SHA-256-checksums zijn vastgelegd. + - Gemeente Mol, Ziekenhuis Geel, lokaal bestuur Geel, Stad Turnhout, Group Renotec, Ravago, Sanofi Geel en DAF Trucks Westerlo zijn na fixtures, negatieve routechecks en twee livecycli actief. + - Johnson & Johnson Beerse, Nike Laakdal, AZ Turnhout, Cartamundi, Nyrstar, Aurubis, Estée Lauder, Soudal, H.Essers, Cegeka, IOK en Jessa staan als interessante niet-scanbare werkgeverswaaklijst in de applicatie. + - HTTP, robots en IMAP gebruiken een strikte gepinde CA-bundel; de eerdere lokale Windows-CA-fout bij Gemeente Mol en Ziekenhuis Geel was geen geldige reden voor een bypass. + - Volgende taak: geen uitvoerbare `ready` taak; alleen `VR-202`, `VR-203` en `VR-204` blijven `blocked-external` voor productiecredentials/-infrastructuur. + +- 2026-07-22: `VR-126` voltooid. + - Het actieve lokale profiel bewaart postcode 2400 en een straal van 40 km; gemeente en coördinaten blijven leeg tot de gelicenseerde geodata uit `VR-205` beschikbaar is. + - De catalogus is herankerd op Mol en uitgebreid met SCK CEN, Cipal Schaubroeck, VanRoey, NTX/Netropolix en Thomas More. + - Exacte host-, pad-, plaats- en CVWarehouse-GUID-filters weigeren off-domain, andere vestigingen en kandidaat-/sollicitatieroutes. + - Gemeente Mol en Ziekenhuis Geel zijn niet geactiveerd wegens falende TLS-certificaatvalidatie; er is geen bypass toegepast. + - Volgende taak: geen uitvoerbare `ready` taak; `VR-205` blijft `blocked-external` voor de gelicenseerde geocodedataset en `VR-202` voor productie-mailboxcredentials. + +- 2026-07-22: `VR-125` voltooid. + - De regionale catalogus gebruikt voorlopig Hasselt-Genk als omkeerbaar centrum voor de indicatieve 40-km-selectie omdat het actieve profiel nog geen postcode bevat; de profielafstand wordt exact zodra die postcode is ingevuld. + - Careerjet, Randstad en Robert Half vervangen de expliciete Brusselse nicheboards in de platformmailboxflow; alle brede jobboards blijven op de directe-fetchdenylist. + - ACA Group, Xplore Group, Universiteit Hasselt, Hogeschool PXL en Ziekenhuis Oost-Limburg worden via host- en padgebonden publieke lijstparsers ingelezen; alle vijf slaagden tweemaal live zonder warnings. + - Zeven werkgeversbronnen buiten de regionale scope zijn gedeactiveerd en verborgen, maar hun historische aliassen, documenten en provenance blijven intact. + - Volgende taak: geen uitvoerbare `ready` taak; `VR-201` en `VR-202` blijven `blocked-external` voor persoonlijke postcode-/bronreview en productiecredentials. + +- 2026-07-22: `VR-124` voltooid. + - LinkedIn, ictjob.be, Jobat, StepStone, EuroBrussels en BrusselsJobs zijn toegevoegd aan dezelfde versleutelde platformmailboxflow als VDAB en Indeed. + - De bronpagina toont per jobboard een actuele officiële alertingang; ieder platform behoudt een eigen mailbox, interval, status, pauze- en syncactie. + - De mailparser accepteert per koppeling uitsluitend gelabelde HTTPS-links op het gekozen jobboarddomein en verwerpt externe en administratieve links. + - Alle brede jobboards blijven directe-fetchdenylistbronnen; VacatureRadar logt alleen in op het mailboxaccount en nooit op een jobboard. + - Volgende taak: `VR-202` blijft `blocked-external` voor productieconfiguratie en live validatie met de gewenste afzonderlijke alertmailboxen. + +- 2026-07-22: `VR-123` voltooid. + - Een ingelogde gebruiker kan VDAB en Indeed elk aan een afzonderlijke Gmail-, Outlook- of aangepaste IMAP-mailbox koppelen, wijzigen, pauzeren en direct synchroniseren. + - Iedere koppeling heeft een eigen interval van 5 minuten tot 6 uur; de scheduler verwerkt alleen verschuldigde mailboxen met leases en foutisolatie. + - App-wachtwoorden worden nooit teruggetoond en zijn versleuteld met afzonderlijke roteerbare sleutels; mailboxmutaties zijn login-, gebruikersscope- en CSRF-beveiligd. + - Mailimport bewaart platform- en mailboxprovenance en is idempotent per mailbox en Message-ID; directe platformfetch en platformlogin blijven uitgesloten. + - Volgende taak: `VR-202` blijft `blocked-external` voor productieconfiguratie en live validatie met de afzonderlijke VDAB- en Indeed-mailboxaccounts. + +- 2026-07-22: `VR-122` voltooid. + - Officiële VDAB-jobalerts worden standaard dagelijks om 06:15 Europe/Brussels via de dedicated IMAP-mailbox verwerkt; planning en batchlimieten zijn configureerbaar. + - De mailadapter bewaart VDAB-provenance, replay blijft idempotent en een oversized of fout bericht stopt de rest van de batch niet. + - De bronpagina toont mailboxstatus, laatste VDAB-import, een beveiligde handmatige inplanknop en de contractueel afgeschermde officiële VDAB Vacature API als afzonderlijke route. + - Directe VDAB-scraping, platformlogin en niet-goedgekeurde API-toegang blijven geblokkeerd. + - Volgende taak: `VR-202` blijft `blocked-external` voor het invullen en live valideren van mailbox-/SMTP-credentials. + +- 2026-07-22: `VR-121` voltooid. + - Corda Campus, RPM Europe Logistics, YPTO, adesso Belgium, Monizze, Qover, SFEIR Belgium en ARHS Belgium zijn na actuele review en twee succesvolle cycli geactiveerd. + - De catalogus omvat nu regionale Hasselt/Genk-dekking en aanvullende Belgische IT-werkgevers in Brussel en Leuven. + - Bronnen op gedeelde ATS-hosts worden uniek per exacte feed-URL bewaard; per-origin leases en rate limits blijven gezamenlijk gelden. + - De Corda-adapter bewaart werkgever en titel van publieke jobkaarten en weigert off-domain links. + - VDAB en Indeed blijven op de directe-fetchdenylist en worden in de UI via officiële e-mailalerts naar de bestaande beveiligde mailboxingress geleid. + - Volgende taak: `VR-201` blijft `blocked-external` voor verdere persoonlijke bronkeuzes en periodieke menselijke voorwaardenreview. + +- 2026-07-21: `VR-120` voltooid. + - Thuislocatie is teruggebracht tot een viercijferige postcode; gemeente en coördinaten worden alleen uit lokale geodata afgeleid. + - Alle profieltaxonomie gebruikt begeleide checkboxgroepen met dynamisch behoud van bestaande waarden. + - VITO en Deliverect zijn na actuele policy-/robotsreview en twee livecycli geactiveerd; samen leverden ze 63 actuele vacaturealiases. + - Recruitee- en Leverresponsevormen, employeridentiteit, direct-employerstatus, 304-inventaristelling en robotsredirectvalidatie zijn functioneel hersteld. + - De fixturebron is uit het normale bronoverzicht verborgen; browserhandoff staat op de vernieuwde profieleditor. + - Volgende taak: `VR-201` blijft `blocked-external` voor verdere persoonlijke werkgevers- en overheidsdekking. + +- 2026-07-21: `VR-119` voltooid. + - Stitch bepaalt nu de dominante schermcompositie: utilityzoeker, brede matchfeed, vaste radar-/actierails, dichte resultaatkaarten, intelligence-strip, kanban, profielpreview, compacte bronimport en echte systeemtelemetrie. + - Vacaturelijstscores worden gebruikersgescopeerd uit de laatste `ScoreRun` geannoteerd; er is geen fictieve frontenddata toegevoegd. + - Nederlandstalige profielvelden, lokale favicon en aanvullende HTML/a11y-regressieasserties zijn toegevoegd. + - Browserbewijs en exact verificatierapport staan onder `docs/design/validation/` met prefix `vr119-`. + - Volgende taak: `VR-201` (`blocked-external`). + - 2026-07-21: `VR-118` voltooid. - Alle kernroutes delen nu één responsive Intelligence Cockpit-shell met lokale SVG-iconen en CSP-veilige assets. - Stitch-dark en een volwaardige light variant zijn gekoppeld aan echte Django-data; externe quick apply, nepmetrics en onbewezen intelligence zijn bewust geweerd. @@ -156,13 +308,13 @@ Een toekomstige agent moet altijd de actuele waarheid opnieuw bepalen met: ## Nog te bouwen -Momenteel staan er geen `ready` taken meer over; de volgende te behandelen item is `VR-201` als `blocked-external` (bronreview). +Momenteel staan er geen `ready` taken meer over; het volgende te behandelen item is `VR-202` als `blocked-external` voor live mailbox-/SMTP-credentials en echte jobalertmails van de gekozen platformen. ## Externe blokkades Deze blokkeren onafhankelijke code niet: -- concrete werkgevers-/bronnenkeuze en actuele menselijke bronreview; +- verdere persoonlijke werkgevers-/bronnenkeuze en actuele menselijke review buiten de actieve VITO-/Deliverect-startercatalogus; - live IMAP/SMTP-account en app-password; - Unraid-domein, DNS, TLS en reverse proxy; - containerregistry en credentials; @@ -171,7 +323,7 @@ Deze blokkeren onafhankelijke code niet: ## Bekende verificatiebeperking van de aangeleverde basis -De Pythonapplicatie, Dockerimage en PostgreSQL/Redis/Unraid-stack zijn geverifieerd. De optionele pytest-Playwrightvarianten vallen lokaal terug op de HTML/a11y-probe; VR-118 is aanvullend handmatig met de geïntegreerde browser op zes viewports uitgevoerd. Een publieke HTTPS-smoke blijft afhankelijk van domein-, DNS- en reverse-proxyconfiguratie. +De Pythonapplicatie, Dockerimage en PostgreSQL/Redis/Unraid-stack zijn geverifieerd. De optionele pytest-Playwrightvarianten vallen lokaal terug op de HTML/a11y-probe; VR-119 is aanvullend met de geïntegreerde browser op alle kernroutes uitgevoerd, VR-120 op het begeleide profiel en de live vacaturefeed en VR-124 op de acht platformmailboxen en alertkaarten. Een publieke HTTPS-smoke blijft afhankelijk van domein-, DNS- en reverse-proxyconfiguratie. ## Hervatten diff --git a/docs/architecture/SECURITY_ARCHITECTURE.md b/docs/architecture/SECURITY_ARCHITECTURE.md index e9b92cc..3f6c3b4 100644 --- a/docs/architecture/SECURITY_ARCHITECTURE.md +++ b/docs/architecture/SECURITY_ARCHITECTURE.md @@ -23,6 +23,8 @@ Verplichte stappen: 5. response status/contenttype/grootte controleren; 6. alleen allowlisted headers bewaren. +Webfetches, robotsrequests en IMAP-TLS gebruiken expliciet de bij de release vastgelegde `certifi`-CA-bundel. Hostnaamcontrole en `CERT_REQUIRED` blijven verplicht. Een verouderde Windows-rootstore is daarom geen reden voor `verify=False`; werk de gepinde bundel en lockfile gecontroleerd bij en voer de TLS-negatietests opnieuw uit. + Geen adapter of view mag dit pad omzeilen. ## 3. SSRF @@ -51,11 +53,13 @@ Resterend risico: klassieke DNS rebinding tussen resolutie en socketconnectie. P ## 5. E-mail -- dedicated mailbox; +- een dedicated mailbox per platform; - attachments worden genegeerd; - message identity voorkomt dubbele verwerking; +- platformgebonden import accepteert alleen gelabelde HTTPS-links op het expliciet gekozen jobboarddomein; +- externe tracking-, account-, help-, privacy- en uitschrijflinks worden niet als vacature opgeslagen; - links worden niet automatisch gevolgd tenzij later door bronpolicy goedgekeurd; -- mailboxcredentials alleen environment/secret; +- mailbox-app-wachtwoorden versleuteld in de database met een afzonderlijke, roteerbare environmentkey; - raw mail korte retentie; - log geen volledige onderwerp/body wanneer daarin PII kan staan. diff --git a/docs/architecture/SOURCE_ADAPTERS.md b/docs/architecture/SOURCE_ADAPTERS.md index 25a330e..5f21096 100644 --- a/docs/architecture/SOURCE_ADAPTERS.md +++ b/docs/architecture/SOURCE_ADAPTERS.md @@ -47,8 +47,10 @@ Voor e-mail: 1. MIME decode zonder attachments; 2. linkextractie met unsubscribe/privacyfilter; -3. platformalias bewaren; -4. oorspronkelijke werkgever proberen te resolveren in een aparte begrensde taak. +3. bij een platformmailbox uitsluitend gelabelde HTTPS-links op het gekozen jobboarddomein aanvaarden; +4. externe tracking-, account-, help-, privacy- en uitschrijflinks verwerpen; +5. platformalias en provenance bewaren; +6. oorspronkelijke werkgever proberen te resolveren in een aparte begrensde taak. ## 3. Minimumvelden @@ -153,8 +155,20 @@ Automatisch herstel mag veilige selectorvarianten proberen, maar nieuwe logica m | SmartRecruiters | `*.smartrecruiters.com` | gespecialiseerd listing+detail parser | geen interne API-auth nodig; stop bij CAPTCHA/anti-bot | 30 requests/min per domein | | Workable | `apply.workable.com` | gespecialiseerd listing+detail parser | alleen publieke vacaturepagina/JSON; geen private endpoints | 30 requests/min per domein | +Gedeelde ATS-hosts zijn geen bronidentiteit: meerdere werkgevers mogen hetzelfde providerdomein gebruiken zolang iedere bron een unieke publieke `base_url` heeft. De scheduler blijft per origin coördineren, zodat deze extra bronrecords de providerlimiet niet omzeilen. + +Voor Corda Campus is een begrensde regionale listingadapter beschikbaar. Die leest uitsluitend de expliciete `/jobs/`-kaarten, bewaart de vermelde werkgever en weigert iedere joblink buiten `cordacampus.com`. Vacaturedetail en sollicitatie-URL's worden niet automatisch gevolgd. + Voor elke provider-adapter: - `supports()` kijkt eerst op host/marker om andere pagina's niet te matchen; - listingpayloads produceren kandidaten zonder fetch van detail; - detailpayloads vullen een enkele vacaturestructuur; - elke veldafklaring krijgt veld-evidence met method-labels en confidence. + +## Regionale Mol-adapter (VR-126) + +`MolRegionEmployerAdapter` verwerkt uitsluitend vijf vooraf gereviewde HTTPS-lijstroutes rond postcode 2400: SCK CEN, Cipal Schaubroeck, VanRoey, NTX/Netropolix en Thomas More. De adapter doet geen netwerkrequests en accepteert alleen same-host vacaturelinks. Cipal, NTX en Thomas More vereisen een expliciete lokale marker; bij Thomas More wordt bovendien de vaste publieke werkgever-GUID gecontroleerd. VanRoey-detailpaden zijn door robots uitgesloten en worden daarom nooit door VacatureRadar gefetcht; alleen de toegestane overzichtspagina dient als bron. + +## Kempen-werkgeversadapter (VR-201) + +`KempenEmployerAdapter` voegt zeven gereviewde werkgeversroutes toe voor Ziekenhuis Geel, lokaal bestuur Geel, Stad Turnhout, Group Renotec, Ravago, Sanofi en DAF Trucks Westerlo. Iedere route heeft een vaste HTTPS-host en een toegestaan lijstpad; vacaturelinks moeten same-host zijn. Voor gedeelde of landelijke lijsten blijft alleen een kaart met de expliciete gereviewde vestiging over. Gemeente Mol wordt via de officiële same-host RSS-feed verwerkt. Interessante werkgevers zonder stabiele fail-closed parser blijven als niet-scanbare waaklijstbron zichtbaar. diff --git a/docs/architecture/adrs/0005-email-alerts-as-platform-ingress.md b/docs/architecture/adrs/0005-email-alerts-as-platform-ingress.md index fe4ca9b..34f5689 100644 --- a/docs/architecture/adrs/0005-email-alerts-as-platform-ingress.md +++ b/docs/architecture/adrs/0005-email-alerts-as-platform-ingress.md @@ -2,6 +2,7 @@ - Status: accepted - Datum: 2026-07-20 +- Aangevuld door: ADR-0009 ## Context @@ -9,11 +10,16 @@ Vacatureplatformen bieden nuttige zoekalgoritmes en alerts, terwijl directe scra ## Besluit -Gebruik een dedicated IMAP-mailbox en idempotente MIME/linkextractie. Bewaar platformlinks als bronalias. Volg een link alleen wanneer bronpolicy dat later toestaat; probeer bij voorkeur een oorspronkelijke werkgeverspagina te vinden. +Gebruik per vacatureplatform een dedicated IMAP-mailbox en idempotente MIME/linkextractie. Iedere mailbox heeft een eigen interval en kan vanuit de bronpagina afzonderlijk worden gepauzeerd of ingepland. Bewaar platformlinks als bronalias. Volg een link alleen wanneer bronpolicy dat later toestaat; probeer bij voorkeur een oorspronkelijke werkgeverspagina te vinden. + +De gereviewde mailboxcatalogus omvat VDAB, Indeed, LinkedIn, ictjob.be, Jobat, StepStone, Careerjet, Randstad en Robert Half. De gebruiker activeert de officiële jobalert zelf. VacatureRadar logt alleen in op het opgegeven mailboxaccount, beperkt extractie tot gelabelde HTTPS-links op het gekozen platformdomein en gebruikt geen jobboardaccount, zoekbot of directe crawl. Expliciet Brusselse nicheboards zijn in VR-125 uit de configureerbare catalogus verwijderd omdat ze buiten de regionale productscope vallen. + +Voor VDAB bestaan twee officiële routes. Door de gebruiker ingestelde [jobalerts](https://www.vdab.be/jobs/job-alerts) mogen onmiddellijk via deze mailboxingang worden verwerkt. De [VDAB Vacature API](https://extranet.vdab.be/api-center-excellence-coe/vacatures-ophalen-met-de-vacatures-api) mag pas als fetchbron worden geactiveerd nadat VDAB het partnership heeft goedgekeurd en de samenwerkingsovereenkomst en credentials beschikbaar zijn. Deze ADR staat geen scraping of gebruik van een verborgen/onbevoegd endpoint toe. ## Gevolgen - gebruiker stelt alerts eenmalig in; -- parser moet variërende e-mailsjablonen verdragen; -- dedicated mailbox en retentie nodig; +- parser moet variërende e-mailsjablonen verdragen en een onbekende wijziging fail-closed als geen herkenbare vacaturelink behandelen; +- batches en berichtgrootte zijn begrensd en één fout bericht stopt de rest niet; +- dedicated mailbox per platform en retentie nodig; - geen platformcredentials in VacatureRadar. diff --git a/docs/architecture/adrs/0009-encrypted-platform-mailboxes.md b/docs/architecture/adrs/0009-encrypted-platform-mailboxes.md new file mode 100644 index 0000000..931e351 --- /dev/null +++ b/docs/architecture/adrs/0009-encrypted-platform-mailboxes.md @@ -0,0 +1,32 @@ +# ADR-0009 — Versleutelde platformmailboxen in de applicatie + +- Status: accepted +- Datum: 2026-07-22 + +## Context + +Alerts van VDAB, Indeed, LinkedIn en andere gereviewde jobboards komen in verschillende mailboxaccounts binnen en moeten vaker dan eenmaal per dag verwerkt kunnen worden. Alleen globale environmentcredentials bieden geen bruikbare beheerflow in het platform. Mailbox-app-wachtwoorden zijn secrets en mogen niet leesbaar in de database, logs, taakresultaten of HTML terechtkomen. + +## Besluit + +- Bewaar één gebruikersgescopeerde `MailboxConnection` per vacatureplatform. +- Ondersteun Gmail, Outlook / Microsoft 365 en een gecontroleerde aangepaste publieke IMAP-host, uitsluitend via IMAP over TLS. +- Versleutel app-wachtwoorden met Fernet/MultiFernet en een afzonderlijke `MAILBOX_CREDENTIAL_KEYS`-sleutellijst. De eerste sleutel versleutelt; volgende sleutels zijn alleen voor decryptie tijdens rotatie. +- Laat de scheduler elke vijf minuten verschuldigde mailboxen selecteren. Iedere mailbox kiest zelf 5, 15, 30, 60, 180 of 360 minuten. +- Gebruik een databaselease per mailbox, idempotentie per mailbox + Message-ID en foutisolatie per mailbox/bericht. +- Alle UI-mutaties zijn login-, user-scope- en CSRF-beveiligd. Het wachtwoordveld wordt nooit opnieuw gevuld. +- Valideer de IMAP-host als publieke host voor de verbinding. Platformlinks uit mail worden niet automatisch gevolgd. + +## Sleutelrotatie + +1. Voeg de nieuwe sleutel vóór de oude toe aan `MAILBOX_CREDENTIAL_KEYS`. +2. Herstart web, worker en scheduler. +3. Draai `python manage.py rotate_mailbox_credentials`. +4. Verwijder de oude sleutel pas nadat de opdracht slaagt en een mailboxsync is getest. + +## Gevolgen + +- `cryptography` is een productieafhankelijkheid en staat exact gepind in de lockfile. +- Een databasedump bevat ciphertext en blijft gevoelig; de encryptiesleutel wordt apart in de secretstore/back-up bewaard. +- Zonder expliciete sleutel is mailboxbeheer in productie fail-closed. Alleen DEBUG gebruikt een lokale sleutel afgeleid van de development `SECRET_KEY`. +- Dit is een mailboxlogin, geen login bij een jobboard en geen toestemming voor directe platformfetch. diff --git a/docs/design/STITCH_INTEGRATION_PLAN.md b/docs/design/STITCH_INTEGRATION_PLAN.md index 5531ee4..c024933 100644 --- a/docs/design/STITCH_INTEGRATION_PLAN.md +++ b/docs/design/STITCH_INTEGRATION_PLAN.md @@ -6,6 +6,19 @@ Deze integratie vertaalt het Stitch-project **VacatureRadar Intelligence Cockpit De volledige ontvangen referentie staat in `docs/design/stitch-reference/`. Deze map en de validatiescreenshots zijn uitgesloten van de productiecontainer via `.dockerignore`; er worden geen externe Tailwind-, font-, icoon-, afbeelding- of JavaScript-CDN's aan de applicatie toegevoegd. +## VR-119: compositie als primaire bron + +Na productfeedback is de integratie aangescherpt: Stitch bepaalt nu niet alleen kleur, vorm en typografie, maar ook de dominante schermcompositie en informatiedichtheid. Alleen aantoonbaar onveilige of fictieve onderdelen worden vervangen door een functioneel equivalent met echte repositorydata. + +- De utilitybar neemt de brede zoekbalk en directe profielactie over. +- Het dashboard gebruikt een enkelkoloms matchfeed met vaste radar-/actierail; één resultaat blijft daardoor een volwaardig primair signaal. +- De vacaturelijst toont een dichte intelligencefeed met de laatste gebruikersgescopeerde score via databaseannotaties. +- Het detail combineert scorehero, drie echte kernmetingen, analyse en een vaste menselijke actierail. +- Pipeline, profieleditor, bronkaarten en systeemtelemetrie volgen de Stitch-kolommen en panelhiërarchie. +- Handmatige import blijft volledig beschikbaar, maar staat compact achter een disclosure zodat brongezondheid visueel voorrang krijgt. + +De bedoelde “circa 90% Stitch” slaat hiermee op overgenomen visuele structuur en interactiehiërarchie, niet op het kopiëren van fictieve content, externe assets, auto-apply of routes waarvoor het domeinmodel geen betrouwbare data bezit. + ## Visuele richting - Donkere cockpit als standaard: diep marineblauw, tonale panelen, fijne slate-randen, signaalcyaan voor primaire interactie, emerald voor aantoonbaar gezonde status en koraal/amber voor fouten of aandacht. diff --git a/docs/design/validation/VR-119_REPORT.md b/docs/design/validation/VR-119_REPORT.md new file mode 100644 index 0000000..72b1d3d --- /dev/null +++ b/docs/design/validation/VR-119_REPORT.md @@ -0,0 +1,21 @@ +# VR-119 browservalidatie + +Datum: 2026-07-21 + +## Scope + +De tweede Stitch-pass is gecontroleerd tegen dashboard, vacaturelijst, vacaturedetail, sollicitatiepipeline, profieloverzicht/editor, bronnen en systeemstatus met de lokale `visualtest`-fixture. Alle weergegeven cijfers en acties kwamen uit de lokale Django-database; er is geen fictieve frontenddata toegevoegd. + +## Resultaten + +- Dashboardmatrix geslaagd op 1440×900, 1280×800, 1024×768, 768×1024, 390×844 en 360×800. +- Iedere gecontroleerde kernroute had exact één `h1`, zichtbare inhoud en geen pagina-brede horizontale overflow. +- De pipeline gebruikt bewust een intern horizontaal scrollgebied voor zes statuskolommen; de pagina zelf overflowt niet. +- Mobiele navigatie opent op 390×844, verplaatst focus naar de drawer en houdt de pagina binnen de viewport. +- Dark en light renderen met dezelfde informatiehiërarchie; de themawissel gaf geen consolefout. +- Externe vacaturelinks behouden `noopener`; er werden geen browserconsolefouten gevonden. +- Lokale CSS, JavaScript, iconsprite en favicon laden zonder externe productie-assets. + +## Bewijs + +Nieuwe screenshots gebruiken het prefix `vr119-` in deze map. De oudere VR-118-screenshots blijven bewaard als visuele voor/na-historie. diff --git a/docs/design/validation/vr119-applications-dark-1440x900.png b/docs/design/validation/vr119-applications-dark-1440x900.png new file mode 100644 index 0000000..6f08225 Binary files /dev/null and b/docs/design/validation/vr119-applications-dark-1440x900.png differ diff --git a/docs/design/validation/vr119-dashboard-dark-1440x900.png b/docs/design/validation/vr119-dashboard-dark-1440x900.png new file mode 100644 index 0000000..05c2c67 Binary files /dev/null and b/docs/design/validation/vr119-dashboard-dark-1440x900.png differ diff --git a/docs/design/validation/vr119-dashboard-light-1280x800.png b/docs/design/validation/vr119-dashboard-light-1280x800.png new file mode 100644 index 0000000..5e195b3 Binary files /dev/null and b/docs/design/validation/vr119-dashboard-light-1280x800.png differ diff --git a/docs/design/validation/vr119-dashboard-mobile-390x844.png b/docs/design/validation/vr119-dashboard-mobile-390x844.png new file mode 100644 index 0000000..f3319a3 Binary files /dev/null and b/docs/design/validation/vr119-dashboard-mobile-390x844.png differ diff --git a/docs/design/validation/vr119-job-detail-dark-1440x900.png b/docs/design/validation/vr119-job-detail-dark-1440x900.png new file mode 100644 index 0000000..c089e52 Binary files /dev/null and b/docs/design/validation/vr119-job-detail-dark-1440x900.png differ diff --git a/docs/design/validation/vr119-jobs-dark-1440x900.png b/docs/design/validation/vr119-jobs-dark-1440x900.png new file mode 100644 index 0000000..a0b9fd3 Binary files /dev/null and b/docs/design/validation/vr119-jobs-dark-1440x900.png differ diff --git a/docs/design/validation/vr119-mobile-navigation-390x844.png b/docs/design/validation/vr119-mobile-navigation-390x844.png new file mode 100644 index 0000000..ff6d4e2 Binary files /dev/null and b/docs/design/validation/vr119-mobile-navigation-390x844.png differ diff --git a/docs/design/validation/vr119-profile-editor-dark-1440x900.png b/docs/design/validation/vr119-profile-editor-dark-1440x900.png new file mode 100644 index 0000000..0ad7a22 Binary files /dev/null and b/docs/design/validation/vr119-profile-editor-dark-1440x900.png differ diff --git a/docs/design/validation/vr119-sources-dark-1440x900.png b/docs/design/validation/vr119-sources-dark-1440x900.png new file mode 100644 index 0000000..ba1fe82 Binary files /dev/null and b/docs/design/validation/vr119-sources-dark-1440x900.png differ diff --git a/docs/design/validation/vr119-system-dark-1440x900.png b/docs/design/validation/vr119-system-dark-1440x900.png new file mode 100644 index 0000000..753a566 Binary files /dev/null and b/docs/design/validation/vr119-system-dark-1440x900.png differ diff --git a/docs/operations/BACKUP_RESTORE.md b/docs/operations/BACKUP_RESTORE.md index c8b977e..1be67a2 100644 --- a/docs/operations/BACKUP_RESTORE.md +++ b/docs/operations/BACKUP_RESTORE.md @@ -76,7 +76,7 @@ Een hersteltest is pas geslaagd wanneer: - een score en digest kunnen worden aangemaakt; - geen live mailbox of bron onbedoeld wordt gepolld in de testomgeving. -Gebruik in een hersteltest `IMAP_ENABLED=0`, `CELERY_TASK_ALWAYS_EAGER=1`, consolemail en geen live source polling. +Pauzeer in een hersteltest alle `MailboxConnection`-records, gebruik `CELERY_TASK_ALWAYS_EAGER=1`, consolemail en geen live source polling. Bewaar `MAILBOX_CREDENTIAL_KEYS` afzonderlijk van de databaseback-up; zonder minstens één bijbehorende sleutel zijn opgeslagen app-wachtwoorden niet herstelbaar. ## Verwijdering en privacy diff --git a/docs/operations/GEOCODING_DATA_IMPORT.md b/docs/operations/GEOCODING_DATA_IMPORT.md index e87d395..3946f0b 100644 --- a/docs/operations/GEOCODING_DATA_IMPORT.md +++ b/docs/operations/GEOCODING_DATA_IMPORT.md @@ -1,6 +1,8 @@ # Geodata import voor Belgische locaties Deze module gebruikt lokaal beheerde postcode/gemeentedata om locatiecoördinaten te kunnen afleiden. +De actuele lokale keuze is GeoNames Postal Codes BE, versie 2026-04-03 onder CC BY 4.0. Bron, +checksums, reviewdatum en steekproef staan in `config-data/geodata/README.local.md`. ## CLI @@ -34,3 +36,9 @@ python manage.py import_geodata \ ## Fallback Zonder geïmporteerde geodata blijft afstandsafleiding op "onbekend". + +## Nauwkeurigheid en privacy + +Een postcodepunt is een geschat centrum en geen routeafstand of exact adres. VacatureRadar bewaart +voor het thuisprofiel alleen de ingevoerde postcode en het bijbehorende lookup-punt. De dataset bevat +geen namen, adressen, contactgegevens of andere persoonsgegevens. diff --git a/docs/operations/LIVE_SOURCE_CATALOG.md b/docs/operations/LIVE_SOURCE_CATALOG.md new file mode 100644 index 0000000..7a5c628 --- /dev/null +++ b/docs/operations/LIVE_SOURCE_CATALOG.md @@ -0,0 +1,73 @@ +# Livebroncatalogus + +Reviewdatum: 2026-07-22 +Review vervalt: uiterlijk 2026-10-20 + +Deze catalogus bevat alleen publieke vacaturepagina's, officiële RSS-feeds en gedocumenteerde publieke ATS-feeds. Iedere actieve bron gebruikt uitsluitend `GET`, passeert bronbeleid, robotscontrole, TLS-controle en SSRF-validatie en benadert geen sollicitatie- of kandidaatendpoint. + +## Regionale scope + +Het actieve profiel gebruikt **postcode 2400**, gemeente **Mol**, en een maximale straal van **40 km**. Op 2026-07-22 is de officiële Belgische GeoNames-postcodedataset van 2026-04-03 lokaal geladen onder CC BY 4.0. Postcode 2400 is daarmee opgelost naar het benaderende postcodecentrum **51.1919, 5.1166**. Afstanden zijn bruikbaar voor regionale filtering, maar blijven indicatief omdat een postcodecentrum geen exact adres is. + +## Actieve werkgeversbronnen + +| Bron | Regionale reden en publieke ingang | Parser | Livevalidatie | +|---|---|---|---| +| Gemeente Mol | Mol; [officiële RSS-feed](https://jobs.gemeentemol.be/rss/vacatures.rss) | `rss` | 4 vacatures; twee geslaagde cycli, idempotent. | +| VITO | Mol; [eigen jobsite](https://jobs.vito.be/l/nl) | `ats-recruitee` | Bestaande bron; twee geslaagde cycli. | +| SCK CEN | Mol; [eigen vacaturelijst](https://www.sckcen.be/nl/carriere/vacatures) | `regional-mol-employers` | 6 kaarten; twee geslaagde cycli. | +| Ziekenhuis Geel | Geel; [eigen vacaturelijst](https://ziekenhuisgeel.careersite.be/nl/vacatures) | `regional-kempen-employers` | 14 kaarten; twee geslaagde cycli. | +| Lokaal bestuur Geel | Geel; [eigen jobsite](https://geel.hro.be/) | `regional-kempen-employers` | 2 kaarten; twee geslaagde cycli. | +| Group Renotec | Geel; [publieke vacaturelijst](https://jobs.renotec.be/) | `regional-kempen-employers` | 7 expliciet Geelse kaarten; twee geslaagde cycli. | +| Ravago | Arendonk/Olen; [publieke vacaturelijst](https://ravago.softgarden.io/en/vacancies) | `regional-kempen-employers` | 22 regionale kaarten; twee geslaagde cycli. | +| Sanofi | Geel; [Belgische carrièrepagina](https://jobs.sanofi.com/en/belgium) | `regional-kempen-employers` | 1 expliciet Geelse kaart; twee geslaagde cycli. | +| DAF Trucks | Westerlo; [publieke vacaturelijst](https://www.daf.be/nl-be/over-daf/werken-bij-daf-trucks/vacatures) | `regional-kempen-employers` | 1 expliciet Westerlose kaart; twee geslaagde cycli. | +| Cipal Schaubroeck | Westerlo/Geel; [publieke werkgeverlijst](https://cipalschaubroeck.teamtailor.com/jobs) | `regional-mol-employers` | 12 regionale kaarten; tweede cyclus correct als HTTP 304. | +| VanRoey | Turnhout/Geel; [eigen IT-vacaturelijst](https://www.vanroey.be/en/job-overview/) | `regional-mol-employers` | 23 kaarten; twee geslaagde cycli. | +| NTX (Netropolix) | Geel; [publieke werkgeverpagina](https://netropolix.recruitee.com/) | `regional-mol-employers` | 2 Geelse kaarten; twee geslaagde cycli. | +| Thomas More | Geel/Turnhout/Vorselaar; [publieke vacaturelijst](https://jobpage.cvwarehouse.com/?companyGuid=eab9ca13-ee10-4504-8a87-a785d0b037ef&lang=nl-BE) | `regional-mol-employers` | 5 regionale kaarten; twee geslaagde cycli. | +| Stad Turnhout | Turnhout; [eigen jobsite](https://jobs.turnhout.be/) | `regional-kempen-employers` | 2 kaarten; twee geslaagde cycli. | +| Universiteit Hasselt | Hasselt/Diepenbeek; [eigen vacaturelijst](https://www.uhasselt.be/vacatures) | `regional-local-employers` | 10 kaarten; twee geslaagde cycli. | +| Hogeschool PXL | Hasselt/Diepenbeek; [publieke jobsite](https://ses.pxl.be/) | `regional-local-employers` | 15 kaarten; twee geslaagde cycli. | +| Ziekenhuis Oost-Limburg | Genk/Lanaken/Maaseik; [publieke jobsite](https://ziekenhuis-oost-limburg.cvw.io/?lang=nl-BE) | `regional-local-employers` | 63 kaarten; twee geslaagde cycli. | +| Corda Campus werkgevers | Hasselt; [publieke Corda-joblijst](https://www.cordacampus.com/jobs/) | `regional-corda-campus` | Bestaande bron; twee geslaagde cycli. | +| RPM Europe Logistics NV | Genk; [publieke carrièrepagina](https://rpmeuropelogisticsnv.recruitee.com/) | `ats-recruitee` | Bestaande bron; twee geslaagde cycli. | +| ACA Group | Hasselt; [eigen vacaturelijst](https://acagroup.be/en/jobs) | `regional-local-employers` | 10 kaarten; twee geslaagde cycli. | +| Xplore Group | Hasselt; [eigen vacaturelijst](https://www.xploregroup.be/en/jobs) | `regional-local-employers` | 13 Hasselt-kaarten; twee geslaagde cycli. | + +De Kempen-adapter accepteert uitsluitend zeven vooraf gereviewde HTTPS-host/padcombinaties en same-host detail-URL's. Hij bewaart per werkgever alleen kaarten waarvan de gepubliceerde locatie binnen de afgesproken regionale vestiging valt. Gemeente Mol gebruikt een officiële RSS-feed en heeft daarom geen maatwerkparser nodig. Geen adapter benadert login-, message-, apply- of kandidaatendpoints. + +## Werkgeverswaaklijst + +Interessante regionale werkgevers zonder voldoende stabiele, publiek verifieerbare automatische ingang staan zichtbaar in de applicatiewaaklijst. Ze worden niet door bulk- of geplande scans meegenomen. Dat omvat ook werkgevers die bij de review geen lokale vacature open hadden. + +| Werkgever | Regio | Observatie op 2026-07-22 | +|---|---|---| +| Johnson & Johnson | Beerse | Actuele Belgische rollen; publieke lijst geeft een challenge/403, zonder omzeiling. | +| Nike European Logistics Campus | Laakdal | Actuele distributierollen; globale dynamische joblijst nog niet fail-closed regionaal parseerbaar. | +| AZ Turnhout | Turnhout | Actuele rollen; gedeelde dynamische CVWarehouse-lijst vraagt extra werkgever- en locatiefiltering. | +| Cartamundi | Turnhout | Interessante productie- en ondersteuningswerkgever; geen stabiele lokale publieke lijst bewezen. | +| Nyrstar | Balen/Pelt | Grote regionale industriële werkgever; geen actuele lokale vacature bevestigd. | +| Aurubis | Olen | Grote regionale werkgever; momenteel vooral spontane sollicitatie, geen regelmatige lokale vacaturelijst bevestigd. | +| Estée Lauder | Oevel/Westerlo | Grote regionale productievestiging; geen actuele lokale rol bevestigd. | +| Soudal | Turnhout | Grote regionale werkgever met eigen carrièreplatform; regionale route nog niet stabiel genoeg bewezen. | +| H.Essers | Tessenderlo | Logistieke werkgever met eigen jobsite; vestigingsfilter nog te herreviewen. | +| Cegeka | Hasselt | Relevante IT-werkgever; locatie is niet betrouwbaar aan elke kaart gekoppeld. | +| IOK / IOK Afvalbeheer | Kempen | Regionale publieke werkgever; geen stabiele vacaturefeed bewezen. | +| Jessa Ziekenhuis | Hasselt | Eigen jobs en jobmails; gedeeld ATS vereist nog fail-closed werkgeverafbakening. | + +## Uitgeschakeld wegens buiten regio + +Deliverect (Gent), YPTO, adesso Belgium, Monizze, Qover, SFEIR Belgium en ARHS Belgium zijn uitgeschakeld en uit het normale bronoverzicht verborgen. Historische documenten, vacature-aliassen en provenance blijven bewaard. Dit is geen denylistbeslissing: de bronnen zijn technisch toegestaan, maar vallen buiten de ingestelde regionale productscope. + +## Jobboards via officiële jobalerts + +Brede jobboards worden niet rechtstreeks gecrawld. De gebruiker activeert zelf een officiële alert en koppelt per platform een afzonderlijke mailbox. Ondersteund zijn VDAB, Indeed, LinkedIn, ictjob.be, Jobat, StepStone, Careerjet, Randstad en Robert Half. De parser accepteert per koppeling uitsluitend gelabelde HTTPS-links op het gekozen jobboarddomein en verwerpt externe tracking-, account-, privacy- en uitschrijflinks. Alle negen domeingroepen staan op de directe-fetchdenylist; loginbot, CAPTCHA-omzeiling, verborgen endpoints en jobboardscraping blijven uitgesloten. + +De officiële [VDAB Vacature API](https://extranet.vdab.be/api-center-excellence-coe/vacatures-ophalen-met-de-vacatures-api) blijft een mogelijke rechtstreekse synchronisatieroute, maar vereist goedkeuring, partnership en een samenwerkingsovereenkomst. Zonder die externe toelating wordt geen API-client geactiveerd. + +## TLS en herreview + +Gemeente Mol en Ziekenhuis Geel hebben op 2026-07-22 met de applicatie-CA-bundel een geldige TLS-keten en antwoordden beide met HTTP 200. De eerdere fout kwam door een verouderde lokale Windows-certificaatketen, niet door het uitschakelen van certificaatcontrole. VacatureRadar gebruikt expliciet de gepinde `certifi`-bundel en voert nooit `verify=False` uit. + +Uiterlijk 2026-10-20 controleert de beheerder opnieuw vacaturepagina's, robotsregels, responsevormen, regionale scope en twee proefcycli. Bij voorwaardenwijziging, parserdrift, onverwachte volumestijging, ontbrekende locatie of redirect naar een ander domein gaat de bron terug naar `trial`, `paused` of quarantaine; verlopen reviews blokkeren fail-closed. diff --git a/docs/operations/RUNBOOK.md b/docs/operations/RUNBOOK.md index 6570f7a..fecef72 100644 --- a/docs/operations/RUNBOOK.md +++ b/docs/operations/RUNBOOK.md @@ -56,11 +56,23 @@ celery -A config inspect reserved ## IMAP-import stopt -1. Zet `IMAP_ENABLED=0` om lockout of herhaalde fouten te beperken. -2. Controleer host, poort, TLS, mailboxnaam en app-password. -3. Verifieer dat de mailbox niet is hernoemd en het account niet is vergrendeld. -4. Test eerst read-only met `IMAP_MARK_SEEN=0`. -5. Een reeds verwerkte mail is idempotent via message identity; verwijder geen `EmailMessageRecord` om opnieuw te proberen zonder analyse. +1. Pauzeer alleen de falende platformmailbox op de bronpagina; andere mailboxen blijven werken. +2. Controleer provider, publieke IMAP-host, poort 993, mailboxmap, account en app-wachtwoord. +3. Verifieer dat `MAILBOX_CREDENTIAL_KEYS` aanwezig is en dat bij een rotatie de oude sleutel nog achter de nieuwe staat. +4. Verifieer dat de mailbox niet is hernoemd en het account niet is vergrendeld. +5. Alle polls selecteren de mailbox read-only; een reeds verwerkte mail is per mailbox idempotent via message identity. +6. Controleer het gekozen interval en `next_poll_at`; de scheduler zoekt iedere vijf minuten naar verschuldigde koppelingen. +7. Controleer `IMAP_MAX_MESSAGES_PER_POLL` en `IMAP_MAX_MESSAGE_BYTES`. Vergroot die niet om een onverwacht groot of verdacht bericht blind te verwerken. +8. Controleer bij een leeg resultaat of het jobboardmailformaat nog gelabelde HTTPS-vacaturelinks op het gekozen platformdomein bevat; versoepel de domeinfilter niet zonder fixture en securitytest. +9. Activeer de mailbox opnieuw, kies **Nu synchroniseren** en controleer `last_success_at`. Verwijder geen `EmailMessageRecord` om zonder analyse te herhalen. + +## Mailboxsleutel roteren + +1. Maak een nieuwe Fernet-sleutel en zet die als eerste in `MAILBOX_CREDENTIAL_KEYS`; behoud de oude sleutel erachter. +2. Herstart web, worker en scheduler zodat alle processen dezelfde sleutellijst gebruiken. +3. Draai `python manage.py rotate_mailbox_credentials`. +4. Synchroniseer één mailbox en controleer dat geen ciphertext, app-wachtwoord of volledige mailpayload in logs staat. +5. Verwijder daarna de oude sleutel en herstart opnieuw. ## Digest ontbreekt of dubbel diff --git a/docs/operations/SOURCE_ONBOARDING.md b/docs/operations/SOURCE_ONBOARDING.md index d173149..e156f99 100644 --- a/docs/operations/SOURCE_ONBOARDING.md +++ b/docs/operations/SOURCE_ONBOARDING.md @@ -1,5 +1,7 @@ # Bron onboarden +De actueel gereviewde werkgeverfeeds, proefresultaten en dekkingsgrenzen staan in [`LIVE_SOURCE_CATALOG.md`](LIVE_SOURCE_CATALOG.md). `config-data/seed_sources.yaml` installeert uitsluitend de daar vastgelegde bronnen en verbergt de oude fixturebron uit het normale overzicht. + Een nieuwe bron wordt nooit direct als onbeperkt actief beschouwd. Gebruik het traject **candidate → trial → active** en ga bij veiligheids- of kwaliteitsproblemen naar **quarantined**. ## Toegestane broncategorieën @@ -144,3 +146,21 @@ Voor het onboarden van publieke ATS-bronnen moet het bronrecord minimaal een van - Workable (`apply.workable.com`) — max 30 requests/min, alleen publieke vacaturepagina/feeds. Bij twijfel altijd op `review` blijven en eerst via een trialrun met gesloten evaluatiecriteria (gesloten job, lege joblijst, markeringswijziging) valideren. + +Meerdere werkgevers kunnen dezelfde ATS-host gebruiken. Registreer ze als afzonderlijke bronnen met een unieke exacte publieke `base_url`; gebruik het providerdomein nooit als werkgeveridentiteit. Rate limiting en leases blijven per origin gedeeld, dus laat opeenvolgende liveproeven op Lever of SmartRecruiters het ingestelde minimuminterval respecteren. + +## Brede jobboards via platformmailboxen + +VDAB, Indeed, LinkedIn, ictjob.be, Jobat, StepStone, Careerjet, Randstad en Robert Half blijven directe-fetchdenylistbronnen. Activeer op het jobboard zelf een officiële vacaturealert en koppel op de bronpagina per platform een afzonderlijke Gmail-, Outlook- of aangepaste IMAP-mailbox. Kies per mailbox een interval van 5 tot 360 minuten; de scheduler pakt alleen verschuldigde actieve koppelingen. De parser houdt per koppeling uitsluitend gelabelde HTTPS-links op het gekozen jobboarddomein over. Zonder mailboxcredentials gebruikt de beheerder handmatige URL- of tekstimport; dit is geen reden om platformpagina's automatisch te crawlen. + +`seed_sources` ondersteunt daarnaast `retire_source_urls`. Zo wordt een bron die buiten de gekozen regio valt uitgeschakeld en uit het normale bronoverzicht verborgen, terwijl eerdere documenten, vacature-aliassen en provenance behouden blijven. Gebruik dit mechanisme voor een regionale koerswijziging; verwijder bronhistoriek niet handmatig. + +Voor de Mol-regio geldt postcode 2400 met een straal van 40 km als cataloguscentrum. Leg per werkgever een expliciete gemeentemarker vast en behandel een ontbrekende concrete vacaturelocatie als onbekend. De `regional-mol-employers`-parser mag uitsluitend de vijf gereviewde routes uit de livecatalogus verwerken; een nieuwe host, pad, CVWarehouse-GUID of vestiging vereist eerst een nieuwe fixture, securitytest en bronreview. + +De aanvullende `regional-kempen-employers`-parser volgt dezelfde regel voor zeven exacte routes: Ziekenhuis Geel, lokaal bestuur Geel, Stad Turnhout, Renotec, Ravago, Sanofi en DAF Trucks Westerlo. Gemeente Mol gebruikt de officiële RSS-route. Controleer bij landelijke of gedeelde joblijsten dat iedere kaart de gereviewde gemeente expliciet noemt; een onduidelijke locatie wordt niet stil aan een vestiging toegewezen. + +Gebruik `metadata.watchlist: true` voor een interessante regionale werkgever die momenteel geen vacature heeft of nog geen stabiele publieke listing heeft. Zo'n bron staat op `paused`, verschijnt afzonderlijk in de UI en wordt niet door bulkacties of de scheduler gescand. Activeer hem pas na een nieuwe voorwaardenreview, fixture, negatieve host/padtest en twee geslaagde trialcycli. + +Een mailboxkoppeling vraagt alleen IMAP-gegevens. VacatureRadar vraagt of bewaart geen LinkedIn-, ictjob-, Jobat-, StepStone- of ander jobboardwachtwoord en activeert alerts niet automatisch. Wanneer een jobboard zijn mailformaat wijzigt en geen herkenbare platformlink resteert, werk dan eerst een gesaneerde fixture en parsertest bij. + +VDAB documenteert daarnaast een officiële Vacature API voor het ophalen en synchroniseren van vacatures. Registreer die pas als nieuwe adapter/brontype nadat VDAB het partnership schriftelijk heeft goedgekeurd, de samenwerkingsovereenkomst rond is en testcredentials beschikbaar zijn. Publieke developerportal-documentatie alleen is geen toestemming om de API te gebruiken. diff --git a/docs/operations/UNRAID_DEPLOYMENT.md b/docs/operations/UNRAID_DEPLOYMENT.md index acee432..56fb3e3 100644 --- a/docs/operations/UNRAID_DEPLOYMENT.md +++ b/docs/operations/UNRAID_DEPLOYMENT.md @@ -103,25 +103,30 @@ Wanneer een proxy op hetzelfde Docker-netwerk draait, publiceer poort 1226 allee ## Mailbox -Gebruik een aparte mailbox of alias, niet de hoofdmailbox. Begin met: +Gebruik per vacatureplatform een aparte mailbox of alias, niet de hoofdmailbox. Genereer eerst buiten Git een Fernet-sleutel: -```dotenv -IMAP_ENABLED=0 -IMAP_MARK_SEEN=0 +```bash +python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" ``` -Na een succesvolle fixture- en read-onlytest: +Plaats de uitvoer uitsluitend in de Unraid-secretconfiguratie: ```dotenv -IMAP_ENABLED=1 -IMAP_HOST= -IMAP_USER= -IMAP_PASSWORD= -IMAP_MAILBOX=INBOX -IMAP_USE_SSL=1 +MAILBOX_CREDENTIAL_KEYS= +IMAP_CONNECT_TIMEOUT_SECONDS=15 +IMAP_MAX_MESSAGES_PER_POLL=200 +IMAP_MAX_MESSAGE_BYTES=1000000 ``` -Het account heeft alleen mailboxrechten nodig. Gebruik waar beschikbaar een app-password en schakel interactieve login op die mailbox niet uit zolang herstel nodig is. +Herstart web, worker en scheduler. Open daarna **Bronnen → Platformmailboxen** en koppel ieder gewenst platform afzonderlijk: VDAB, Indeed, LinkedIn, ictjob.be, Jobat, StepStone, Careerjet, Randstad of Robert Half. Kies Gmail, Outlook / Microsoft 365 of een aangepaste publieke IMAP-host, gebruik een app-wachtwoord en laat `INBOX` geselecteerd tenzij de alert naar een aparte map wordt gefilterd. De scheduler controleert iedere vijf minuten welke mailbox volgens haar eigen interval verschuldigd is. + +Voor sleutelrotatie zet je `nieuwe-sleutel,oude-sleutel` in `MAILBOX_CREDENTIAL_KEYS`, herstart je de processen en draai je: + +```bash +python manage.py rotate_mailbox_credentials +``` + +Verwijder de oude sleutel pas na een geslaagde sync. Een rollback van migratie `0005` verwijdert de mailboxkoppelingen; maak daarom eerst een versleutelde databaseback-up en pauzeer de scheduler. ## Ollama diff --git a/docs/quality/THREAT_MODEL.md b/docs/quality/THREAT_MODEL.md index 0fa3337..64f2886 100644 --- a/docs/quality/THREAT_MODEL.md +++ b/docs/quality/THREAT_MODEL.md @@ -57,7 +57,7 @@ Django/Celery | T-09 | MIME/HTML-bom of malformed mail | resource-uitputting/parsercrash | IMAPbatchlimiet, bodylimiet bij vervolgtaak, robuuste parser, idempotente recordidentiteit | `VR-115`, e-mailfixtures | | T-10 | Cross-user objecttoegang | lek van sollicitaties/feedback | LoginRequired, user-scoped querysets/get_or_create, autorisatietests | integratie-viewtests | | T-11 | CSRF op feedback/profiel | ongewenste status- of profielmutatie | Django CSRF-middleware, POST-only acties, SameSitecookies | viewtests en frameworkcheck | -| T-12 | Credentiallek in Git/logs | account- of infrastructuurcompromis | env-only secrets, gitignore, beperkte logs, voorbeeldwaarden, secretrotatierunbook | CI-secret scan gepland `VR-115`; review | +| T-12 | Credentiallek in Git/logs/database/UI | account- of infrastructuurcompromis | app-wachtwoorden met MultiFernet en afzonderlijke environmentkey versleuteld; nooit terugtonen; generieke fouttekst; gitignore, secretrotatierunbook | `tests/unit/test_mailbox_credentials.py`, `tests/integration/test_mailbox_connections.py`, CI-secret scan `VR-115` | | T-13 | Onveilige reverse proxy | sessie over HTTP, host-header-/originproblemen | TLS, exact ALLOWED_HOSTS/CSRF origins, secure cookies, proxychecklist | deployment smoke-test | | T-14 | Redis als waarheid behandeld | dataverlies/dubbele verwerking | PostgreSQL-state, idempotente taken/outbox, acks-late | integratietests | | T-15 | Dubbele Celery-uitvoering | dubbele vacatures/mails | unieke keys, transacties, message identity, outbox-uniciteit | pipeline/digesttests | @@ -73,6 +73,13 @@ Django/Celery | T-31 | Sollicitatiedossier-export bevat niet-gewenste payload | data-lek of onbedoeld dossieroverdragen | export bevat alleen het gekozen dossier, ZIP-inhoud is beperkt tot snapshot/tijdlijn/print-HTML, bestandsnaam is geslugified zonder padseparators | `tests/integration/test_applications.py` | | T-32 | XML external entity/entity-expansion via sitemap | lokale data-uitlezing of parseruitputting | sitemap-XML wordt met `defusedxml` verwerkt; fetcherlimieten blijven vóór parsing gelden | `tests/unit/test_discovery_rss.py`, Ruff securitycheck | | T-33 | Externe UI-CDN of niet-vertrouwde Stitch-interactie | supply-chain-XSS, tracking of acties buiten Django-beleid | productie gebruikt alleen lokale CSS/JS en inline SVG-symbolen; Stitch-code is documentatiereferentie, alle mutaties blijven Django POST+CSRF; themavoorkeur bevat geen persoonsgegevens en opslagfouten worden afgevangen | `tests/e2e/test_vr114_browser_and_accessibility.py`, browserconsole- en assetcontrole VR-118 | +| T-34 | Publieke ATS-feed of robotsredirect wijzigt van host/identiteit | SSRF, verkeerd werkgeverslabel, ongewenst sollicitatie-endpoint of voortgezet fetchen na voorwaardenwijziging | iedere robotsredirect wordt opnieuw als publieke URL gevalideerd; feeds gebruiken alleen gereviewde GET-endpoints; werkgeveridentiteit komt uit source-config; reviews verlopen fail-closed en apply-endpoints zijn geen fetchbron | `tests/security/test_robots.py`, `tests/integration/test_pipeline.py`, `tests/unit/test_ats_adapters.py`, `VR-120` | +| T-35 | Regionale vacaturehub bevat een gemanipuleerde off-domain kaart of gedeelde ATS-host overschrijft een andere werkgever | phishinglink, foutieve herkomst, verlies van bronbeleid of verkeerde inventaris | regionale adapter accepteert alleen same-host `/job/`-links; bronuniciteit gebruikt type + exacte publieke base-URL; per-origin lease en minimuminterval blijven over alle ATS-tenants gedeeld | `tests/unit/test_regional_adapters.py`, `tests/unit/test_source_models.py`, `tests/integration/test_seed_sources.py`, `VR-121` | +| T-36 | Kwaadaardige/oversized platformmail of linkinjectie in een jobalert | parseruitputting, batchuitval, phishing, foutieve vacatures of verborgen scraping | IMAP-batch is begrensd; foutisolatie en idempotente identity per mailbox; platformparser accepteert alleen gelabelde HTTPS-links op het gekozen jobboarddomein en verwerpt externe/adminlinks; alle brede jobboards blijven op de directe-fetchdenylist | `tests/integration/test_email_import_notifications.py`, `tests/unit/test_email_adapter.py`, `tests/security/test_policy.py`, `VR-122/123/124/125` | +| T-37 | Lokale werkgeverslijst wijzigt markup, injecteert een externe kaart of vermengt functies buiten de regionale scope | phishing, foutieve locatie, te brede radar of onbetrouwbare ranking | lokale adapter is beperkt tot vijf gereviewde host/padcombinaties, accepteert alleen same-host details en past waar beschikbaar een expliciete Hasselt-filter toe; retire-seeding schakelt buitenregionale bronnen uit zonder provenance te wissen | `tests/unit/test_regional_adapters.py`, `tests/integration/test_seed_sources.py`, `VR-125` | +| T-38 | Cross-user mailboxbeheer, dubbele polls of IMAP naar private host | credentialmisbruik, duplicaten, toegang tot interne diensten | alle views scopen op gebruiker en vereisen CSRF; unieke user+platformkoppeling; transactionele lease; hostvorm en publieke DNS/IP-validatie vóór TLS-IMAP | `tests/integration/test_mailbox_connections.py`, `tests/unit/test_mailbox_credentials.py`, `apps/sources/services/imap_import.py`, `VR-123` | +| T-39 | Mol-/Kempenbron vermengt een andere werkgever, campus of vestiging | foutieve afstand, phishing of buitenregionale resultaten | exacte HTTPS-host/pad/GUID-routes; same-host details; expliciete plaatsfilters voor regionale en gedeelde lijsten; postcodecentra blijven als benadering herkenbaar; waaklijstbronnen worden niet gescand | `tests/unit/test_regional_adapters.py`, `tests/integration/test_tasks_and_more_views.py`, `VR-126/201` | +| T-40 | Verouderde lokale CA-store of ongeldige serverketen lokt een TLS-bypass uit | onderschepte vacature- of mailboxdata en credentialdiefstal | expliciet gepinde `certifi`-CA-bundel voor HTTP, robots en IMAP; hostnaamcontrole en `CERT_REQUIRED`; nooit `verify=False`; dependency- en negatieve TLS-tests bij update | `tests/unit/test_tls.py`, `tests/security/test_robots.py`, `apps/sources/services/tls.py`, `VR-201` | ## Misbruikscenario's @@ -86,7 +93,7 @@ De redirectlocatie wordt opnieuw gevalideerd. Het request naar het private adres ### E-mailalert bevat een platformlink en uitschrijflink -De adapter herkent vacaturekandidaten, negeert bekende unsubscribepatronen en slaat de platformlink hoogstens als alias op. Automatisch volgen vereist opnieuw een toegestane source policy; denylistplatformen worden geweigerd. +De adapter gebruikt de gekozen platformidentiteit, accepteert alleen gelabelde HTTPS-links op dat jobboarddomein en negeert externe tracking-, account-, privacy- en uitschrijflinks. De platformlink wordt hoogstens als alias opgeslagen. Automatisch volgen vereist opnieuw een toegestane source policy; denylistplatformen worden geweigerd. ### Aanvaller laat scheduler dezelfde job herhalen diff --git a/docs/quality/TRACEABILITY_MATRIX.md b/docs/quality/TRACEABILITY_MATRIX.md index 86a67f7..ebbe32c 100644 --- a/docs/quality/TRACEABILITY_MATRIX.md +++ b/docs/quality/TRACEABILITY_MATRIX.md @@ -17,7 +17,7 @@ Legenda: **I** = geïmplementeerd en getest in de huidige MVP; **P** = gedeeltel | PR-013 | I | `url_security.py`, redirectfetcher | security URL/fetchtests | VR-115 transporthardening | | PR-014 | I | `fetcher.py` | `test_fetcher.py` | — | | PR-015 | I | `SourceLease`, `SourceOriginState`, `apps/sources/services/scheduling.py`, `apps/sources/tasks.py` | `test_scheduling.py`, `test_fetcher.py`, `test_tasks_and_more_views.py` | `VR-103` afgerond | -| PR-016 | I | `email_import.py`, recordmodel | e-mailintegratietests | — | +| PR-016 | I | `email_import.py`, `MailboxConnection`, versleutelde platformmailboxservice en gereviewde jobboardcatalogus | e-mail- en mailboxintegratietests | VR-123/124 | | PR-017 | I | candidate/status + basisdiscovery | `apps/sources/services/discovery.py`, `tests/unit/test_discovery_rss.py`, `tests/integration/test_discovery_command.py`, `tests/integration/test_tasks_and_more_views.py` | VR-101/102 | | PR-018 | I | `apps/sources/services/health.py`, `apps/sources/tasks.py` | `tests/integration/test_source_health.py` | VR-112 | | PR-020 | I | `JsonLdJobPostingAdapter` | JSON-LD-tests/fixtures | — | @@ -39,8 +39,8 @@ Legenda: **I** = geïmplementeerd en getest in de huidige MVP; **P** = gedeeltel | PR-044 | I | profiel/modelversievelden | scoring/profiletests | VR-109 promptversie | | PR-045 | I | optionele Ollamaservice/fallback | `test_ai.py` | VR-109 | | PR-046 | I | geen tools + guardrails | AI/securitytests | VR-109/115 evaluaties | -| PR-050 | I | dashboard view/template + Stitch-cockpitshell | viewtests, VR-118 browsermatrix | VR-114/118 e2e/a11y | -| PR-051 | I | detailview/template/provenance + intelligence-layout | viewtests, VR-118 screenshots | VR-114/118 | +| PR-050 | I | dashboard view/template + Stitch-dominante cockpitshell/feed/rail | viewtests, VR-119 browsermatrix | VR-114/118/119 e2e/a11y | +| PR-051 | I | detailview/template/provenance + scorehero/intelligence-strip | viewtests, VR-119 screenshots | VR-114/118/119 | | PR-052 | I | feedbackendpoint/UI | feedback/viewtests | — | | PR-053 | I | applicationmodel/serviceflow | `tests/integration/test_dedupe_lifecycle_feedback.py`, `tests/integration/test_applications.py` | VR-113 uitgebreid dossier | | PR-054 | I | `notifications/models.py`, outbox/services/tasks | `tests/integration/test_email_import_notifications.py`, `tests/integration/test_reminders.py` | VR-111 | @@ -63,7 +63,7 @@ Legenda: **I** = geïmplementeerd en getest in de huidige MVP; **P** = gedeeltel | NFR-005 | I | domeinapps/services/adapters/ADRs | continue review | | NFR-006 | I | offline pytest + coveragegate | VR-114 verbreedt e2e | | NFR-007 | P | `scripts/benchmark.py`, `tests/benchmark/test_benchmark.py` | VR-116 | -| NFR-008 | I | semantische templates/designspec, zes viewports, dark/light en toetsenbordmenu | `tests/e2e/test_vr114_browser_and_accessibility.py`, `docs/design/validation/`, VR-118 | +| NFR-008 | I | semantische Stitch-dominante templates, zes viewports, dark/light en toetsenbordmenu | `tests/e2e/test_vr114_browser_and_accessibility.py`, `docs/design/validation/`, VR-118/119 | | NFR-009 | I | lokale SQLite en Docker/Unraidconfig, release-smoke en restore | VR-117 afgerond | | NFR-010 | I | `apps/sources/services/health.py`, `apps/sources/tasks.py`, `apps/core/views.py`, `config/settings.py` | `tests/integration/test_source_health.py` | VR-112 observability | @@ -88,5 +88,15 @@ Iedere backlogtaak die een requirement wijzigt, werkt in dezelfde commit deze ma - VR-107 is gerealiseerd met `apps/sources/views.py`, `apps/sources/services/manual_import.py`, `templates/sources/list.html`, `static/js/manual_import.js` en bijhorende unit/integration-tests. - VR-115 is afgerond met productie-hardening voor `config/settings.py`, DNS-rebindmitigatie in `apps/sources/services/fetcher.py`, rate limiting op login en handmatige import, CI-secret- en kwetsbaarheidsscans, non-root/read-only containerhardening en bijbehorende security/unit/integration-tests. - VR-118 vertaalt de aangeleverde Stitch Intelligence Cockpit naar de bestaande server-rendered routes. Bewijs staat in `docs/design/STITCH_INTEGRATION_PLAN.md`, `docs/design/validation/`, de view/a11ytests en de gedeelde lokale shell/assets. +- VR-119 herwerkt die vertaling na productfeedback zodat de Stitch-schermcompositie, feeddichtheid en hoofd-/zijpanelhiërarchie dominant zijn. Bewijs staat in `docs/design/validation/VR-119_REPORT.md`, de `vr119-*.png`-screenshots en de uitgebreide HTML/a11ytest. +- VR-120 vereenvoudigt thuislocatie tot postcode, vervangt JSON-/tekstinvoer door begeleide checkboxes en levert twee gereviewde livewerkgeverfeeds. Bewijs staat in `docs/operations/LIVE_SOURCE_CATALOG.md`, de profiel-/adapter-/pipelinetests en de bronruns in de lokale database. +- VR-121 breidt de catalogus uit tot tien livebronnen, ondersteunt meerdere werkgeverfeeds op gedeelde ATS-hosts, voegt een begrensde Corda Campus-adapter toe en houdt VDAB/Indeed uitsluitend op de bestaande veilige e-mail-/handmatige ingress. Bewijs staat in `docs/operations/LIVE_SOURCE_CATALOG.md`, de bronmigratie en de regionale/ATS/seedtests. +- VR-122 operationaliseert de veilige platformingress als dagelijkse, configureerbare en begrensde mailboxpoll. VDAB-alerts krijgen expliciete provenance, replay blijft idempotent en de bronpagina onderscheidt de werkende jobalertroute van de contractueel afgeschermde officiële Vacature API. Bewijs staat in de e-mailadapter-/importtests, `apps/sources/services/imap_import.py` en ADR-0005. +- VR-123 vervangt die globale poll door gebruikersgescopeerde VDAB- en Indeed-mailboxkoppelingen met versleutelde app-wachtwoorden, afzonderlijke intervallen, per-mailboxleases en UI-beheer. Bewijs staat in `tests/unit/test_mailbox_credentials.py`, `tests/integration/test_mailbox_connections.py`, de migratie en ADR-0009. +- VR-124 breidt dezelfde mailboxingang uit met LinkedIn, ictjob.be, Jobat, StepStone, EuroBrussels en BrusselsJobs. De platformgebonden parser accepteert uitsluitend gelabelde HTTPS-links op het gekozen jobboarddomein; fixtures, adapter-/integratietests en de centrale denylist bewijzen de afscherming. +- VR-125 vervangt de twee Brusselse nicheboards door Careerjet, Randstad en Robert Half, retireert buitenregionale werkgevers zonder provenanceverlies en voegt host- en padgebonden adapters toe voor ACA Group, Xplore Group, UHasselt, PXL en ZOL in de kern Hasselt-Genk. +- VR-126 herankert profiel en broncatalogus op postcode 2400 Mol en 40 km. De host-, pad- en locatiefilters voor SCK CEN, Cipal Schaubroeck, VanRoey, NTX en Thomas More worden gedekt door regionale adaptertests, source-viewtests, live bronruns en de bijgewerkte threat T-39. +- VR-201 activeert Gemeente Mol en zeven bijkomende Kempen-werkgevers na menselijke voorwaardenreview, fixtures, fail-closed locatie-/routefilters en twee livecycli. Twaalf interessante werkgevers zonder stabiele actieve ingang staan afzonderlijk en niet-scanbaar op de UI-waaklijst. Bewijs staat in de livecatalogus, seeddata, `KempenEmployerAdapter`, bronruns en adapter-/seed-/viewtests. +- VR-205 laadt de officiële Belgische GeoNames-postcodedataset 2026-04-03 onder CC BY 4.0, documenteert bron- en afgeleide checksums en lost 2400 Mol op naar het benaderende postcodecentrum. Bewijs staat in `config-data/geodata/README.local.md`, het importlog, geocodingtests en de opnieuw berekende profiel-vacaturescores. diff --git a/fixtures/ats/lever-public-postings.json b/fixtures/ats/lever-public-postings.json new file mode 100644 index 0000000..0900930 --- /dev/null +++ b/fixtures/ats/lever-public-postings.json @@ -0,0 +1,16 @@ +[ + { + "id": "deliverect-open-1", + "text": "Backend Engineer", + "hostedUrl": "https://jobs.lever.co/deliverect/deliverect-open-1", + "applyUrl": "https://jobs.lever.co/deliverect/deliverect-open-1/apply", + "categories": { + "location": "Ghent, Flemish Region", + "commitment": "Full-time Permanent", + "team": "Product" + }, + "country": "BE", + "description": "

Build reliable services.

", + "workplaceType": "hybrid" + } +] diff --git a/fixtures/ats/recruitee-public-offers.json b/fixtures/ats/recruitee-public-offers.json new file mode 100644 index 0000000..a8f1303 --- /dev/null +++ b/fixtures/ats/recruitee-public-offers.json @@ -0,0 +1,19 @@ +{ + "offers": [ + { + "id": 2680964, + "title": "Research Software Engineer", + "careers_url": "https://jobs.vito.be/o/research-software-engineer", + "company_name": "VITO", + "location": "Mol, Antwerpen, Belgium", + "state_name": "Antwerpen", + "postal_code": "2400", + "country": "Belgium", + "description": "

Ontwikkel software voor duurzaam onderzoek.

", + "employment_type_code": "fulltime_fixed_term", + "published_at": "2026-07-17 09:47:06 UTC", + "status": "published", + "hybrid": true + } + ] +} diff --git a/fixtures/ats/smartrecruiters-public-postings.json b/fixtures/ats/smartrecruiters-public-postings.json new file mode 100644 index 0000000..3270a77 --- /dev/null +++ b/fixtures/ats/smartrecruiters-public-postings.json @@ -0,0 +1,47 @@ +{ + "limit": 100, + "offset": 0, + "totalFound": 2, + "content": [ + { + "id": "744000137902340", + "name": "Forgerock IDM Developer", + "company": { + "identifier": "ARHS", + "name": "ARHS" + }, + "releasedDate": "2026-07-20T08:30:00Z", + "location": { + "country": "be", + "region": "Brussels", + "city": "Brussels", + "remote": false + }, + "typeOfEmployment": { + "id": "permanent", + "label": "Full-time" + }, + "ref": "https://api.smartrecruiters.com/v1/companies/ARHS/postings/744000137902340" + }, + { + "id": "744000136173889", + "name": "Cloud Engineer", + "company": { + "identifier": "ARHS", + "name": "ARHS" + }, + "releasedDate": "2026-07-19T08:30:00Z", + "location": { + "country": "be", + "region": "Vlaams-Brabant", + "city": "Leuven", + "remote": true + }, + "typeOfEmployment": { + "id": "permanent", + "label": "Permanent" + }, + "ref": "https://api.smartrecruiters.com/v1/companies/ARHS/postings/744000136173889" + } + ] +} diff --git a/fixtures/emails/careerjet_job_alert.eml b/fixtures/emails/careerjet_job_alert.eml new file mode 100644 index 0000000..3548c66 --- /dev/null +++ b/fixtures/emails/careerjet_job_alert.eml @@ -0,0 +1,13 @@ +From: Careerjet Job Alert +To: radar@example.invalid +Date: Wed, 22 Jul 2026 08:15:00 +0200 +Message-ID: +Subject: Nieuwe jobs rond Hasselt +MIME-Version: 1.0 +Content-Type: text/html; charset="utf-8" + +

Nieuwe jobs

+System Engineer +Externe advertentie +Alert beheren + diff --git a/fixtures/emails/ictjob_job_alert.eml b/fixtures/emails/ictjob_job_alert.eml new file mode 100644 index 0000000..7fc6cdb --- /dev/null +++ b/fixtures/emails/ictjob_job_alert.eml @@ -0,0 +1,14 @@ +From: ictjob.be Job Alert +To: radar@example.invalid +Date: Wed, 22 Jul 2026 08:05:00 +0200 +Message-ID: +Subject: Nieuwe Belgische IT-vacatures +MIME-Version: 1.0 +Content-Type: text/html; charset="utf-8" + + +

Je IT Job Alert

+Cloud Engineer +Advertentietracker +Voorkeuren beheren + diff --git a/fixtures/emails/jobat_job_alert.eml b/fixtures/emails/jobat_job_alert.eml new file mode 100644 index 0000000..7cf0fbc --- /dev/null +++ b/fixtures/emails/jobat_job_alert.eml @@ -0,0 +1,14 @@ +From: Jobat Job Alert +To: radar@example.invalid +Date: Wed, 22 Jul 2026 08:10:00 +0200 +Message-ID: +Subject: Jobs op maat in Limburg +MIME-Version: 1.0 +Content-Type: text/html; charset="utf-8" + + +

Jobs op maat

+Data Engineer +Partnerlink +Privacy + diff --git a/fixtures/emails/linkedin_job_alert.eml b/fixtures/emails/linkedin_job_alert.eml new file mode 100644 index 0000000..b264b83 --- /dev/null +++ b/fixtures/emails/linkedin_job_alert.eml @@ -0,0 +1,15 @@ +From: LinkedIn Jobs +To: radar@example.invalid +Date: Wed, 22 Jul 2026 08:00:00 +0200 +Message-ID: +Subject: Nieuwe jobs voor Software Engineer +MIME-Version: 1.0 +Content-Type: text/html; charset="utf-8" + + +

Nieuwe jobs voor jou

+Software Engineer bij Example +Externe trackingbestemming +Help en voorkeuren +Onveilige HTTP-link + diff --git a/fixtures/emails/randstad_job_alert.eml b/fixtures/emails/randstad_job_alert.eml new file mode 100644 index 0000000..898be3a --- /dev/null +++ b/fixtures/emails/randstad_job_alert.eml @@ -0,0 +1,13 @@ +From: Randstad Job Alert +To: radar@example.invalid +Date: Wed, 22 Jul 2026 08:15:00 +0200 +Message-ID: +Subject: Jobs in Limburg +MIME-Version: 1.0 +Content-Type: text/html; charset="utf-8" + +

Jobalert

+Infrastructure Engineer +Externe advertentie +Accountinstellingen + diff --git a/fixtures/emails/roberthalf_job_alert.eml b/fixtures/emails/roberthalf_job_alert.eml new file mode 100644 index 0000000..661cf63 --- /dev/null +++ b/fixtures/emails/roberthalf_job_alert.eml @@ -0,0 +1,13 @@ +From: Robert Half Job Alert +To: radar@example.invalid +Date: Wed, 22 Jul 2026 08:15:00 +0200 +Message-ID: +Subject: IT and digital jobs +MIME-Version: 1.0 +Content-Type: text/html; charset="utf-8" + +

Job recommendations

+Network Engineer +External advertisement +Account settings + diff --git a/fixtures/emails/stepstone_job_alert.eml b/fixtures/emails/stepstone_job_alert.eml new file mode 100644 index 0000000..bd71e69 --- /dev/null +++ b/fixtures/emails/stepstone_job_alert.eml @@ -0,0 +1,14 @@ +From: StepStone Job Alert +To: radar@example.invalid +Date: Wed, 22 Jul 2026 08:15:00 +0200 +Message-ID: +Subject: Nieuwe jobs voor developer +MIME-Version: 1.0 +Content-Type: text/html; charset="utf-8" + + +

Je dagelijkse Job Alert

+Software Developer +Externe advertentie +Accountinstellingen + diff --git a/fixtures/emails/vdab_job_alert.eml b/fixtures/emails/vdab_job_alert.eml new file mode 100644 index 0000000..6f0508c --- /dev/null +++ b/fixtures/emails/vdab_job_alert.eml @@ -0,0 +1,27 @@ +From: VDAB Job Alerts +To: radar@example.invalid +Date: Tue, 21 Jul 2026 05:30:00 +0200 +Message-ID: +Subject: Nieuwe jobs voor softwareontwikkeling in Limburg +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="vacatureradar-boundary" + +--vacatureradar-boundary +Content-Type: text/plain; charset="utf-8" + +Er zijn nieuwe vacatures voor je VDAB-jobalert. +https://www.vdab.be/vindeenjob/vacatures/70000001 +https://www.vdab.be/vindeenjob/vacatures/70000002 +https://www.vdab.be/jobs/job-alerts/uitschrijven + +--vacatureradar-boundary +Content-Type: text/html; charset="utf-8" + + +

Nieuwe vacatures voor je jobalert

+Python developer +Cloud engineer +Uitschrijven + + +--vacatureradar-boundary-- diff --git a/fixtures/pages/aca-group-jobs.html b/fixtures/pages/aca-group-jobs.html new file mode 100644 index 0000000..52c2d2e --- /dev/null +++ b/fixtures/pages/aca-group-jobs.html @@ -0,0 +1,7 @@ +
+ +

AWS Cloud & DevOps Engineer

+

Design and maintain secure cloud infrastructure.

+ +

External recruiter

 +
diff --git a/fixtures/pages/cipal-schaubroeck-jobs.html b/fixtures/pages/cipal-schaubroeck-jobs.html new file mode 100644 index 0000000..c4fd454 --- /dev/null +++ b/fixtures/pages/cipal-schaubroeck-jobs.html @@ -0,0 +1,5 @@ + diff --git a/fixtures/pages/corda-campus-jobs.html b/fixtures/pages/corda-campus-jobs.html new file mode 100644 index 0000000..cc24dc2 --- /dev/null +++ b/fixtures/pages/corda-campus-jobs.html @@ -0,0 +1,21 @@ + + + +
+ + Voorbeeld IT + System Engineer +
21 juli 2026
+ + + Voorbeeld Data + Data Engineer +
20 juli 2026
+ + + Onbetrouwbaar + Nepjob + +
+ + diff --git a/fixtures/pages/daf-westerlo-jobs.html b/fixtures/pages/daf-westerlo-jobs.html new file mode 100644 index 0000000..7eda936 --- /dev/null +++ b/fixtures/pages/daf-westerlo-jobs.html @@ -0,0 +1,5 @@ + diff --git a/fixtures/pages/netropolix-jobs.html b/fixtures/pages/netropolix-jobs.html new file mode 100644 index 0000000..9844443 --- /dev/null +++ b/fixtures/pages/netropolix-jobs.html @@ -0,0 +1,5 @@ +
+
System Engineer - GeelHybride · Geel, Antwerpen · Infrastructuur
+
System Engineer - KontichKontich, Antwerpen
+ +
diff --git a/fixtures/pages/pxl-jobs.html b/fixtures/pages/pxl-jobs.html new file mode 100644 index 0000000..f404def --- /dev/null +++ b/fixtures/pages/pxl-jobs.html @@ -0,0 +1,10 @@ +
+
+
Lector netwerkbeheer
+ +
PXL-Digital
Campus Diepenbeek
+
+
+
Onveilige vacature
+
+
diff --git a/fixtures/pages/ravago-kempen-jobs.html b/fixtures/pages/ravago-kempen-jobs.html new file mode 100644 index 0000000..ad1cbb0 --- /dev/null +++ b/fixtures/pages/ravago-kempen-jobs.html @@ -0,0 +1,5 @@ + + + + + diff --git a/fixtures/pages/renotec-geel-jobs.html b/fixtures/pages/renotec-geel-jobs.html new file mode 100644 index 0000000..9c6c897 --- /dev/null +++ b/fixtures/pages/renotec-geel-jobs.html @@ -0,0 +1,4 @@ +
+

Technisch commercieel verantwoordelijke

Andere functies · Brussel, Geel, Waremme

Lees meer
+ +
diff --git a/fixtures/pages/sanofi-geel-jobs.html b/fixtures/pages/sanofi-geel-jobs.html new file mode 100644 index 0000000..1c0e56c --- /dev/null +++ b/fixtures/pages/sanofi-geel-jobs.html @@ -0,0 +1,5 @@ +
+Maintenance EngineerGeel, Belgium +TraineeDiegem, Belgium +OnveiligGeel, Belgium +
diff --git a/fixtures/pages/sck-cen-jobs.html b/fixtures/pages/sck-cen-jobs.html new file mode 100644 index 0000000..916a34b --- /dev/null +++ b/fixtures/pages/sck-cen-jobs.html @@ -0,0 +1,7 @@ +
+ + +
diff --git a/fixtures/pages/stad-geel-jobs.html b/fixtures/pages/stad-geel-jobs.html new file mode 100644 index 0000000..acb15a4 --- /dev/null +++ b/fixtures/pages/stad-geel-jobs.html @@ -0,0 +1,6 @@ + +
+

Administratief medewerker digitale dienstverlening

Onbepaalde duur

+
+

Niets gevonden?

+ diff --git a/fixtures/pages/stad-turnhout-jobs.html b/fixtures/pages/stad-turnhout-jobs.html new file mode 100644 index 0000000..bf175f5 --- /dev/null +++ b/fixtures/pages/stad-turnhout-jobs.html @@ -0,0 +1,4 @@ + +
Coördinator digitale projecten

Stad Turnhout

 +Onveilig + diff --git a/fixtures/pages/thomas-more-jobs.html b/fixtures/pages/thomas-more-jobs.html new file mode 100644 index 0000000..e3d749e --- /dev/null +++ b/fixtures/pages/thomas-more-jobs.html @@ -0,0 +1,5 @@ +
+ Een medewerker lokale IT - Campus Geel + IT developer - Campus Mechelen + Fake IT - Campus Geel +
diff --git a/fixtures/pages/uhasselt-jobs.html b/fixtures/pages/uhasselt-jobs.html new file mode 100644 index 0000000..b24c511 --- /dev/null +++ b/fixtures/pages/uhasselt-jobs.html @@ -0,0 +1,9 @@ +
+

Expert systeembeheerder (100%)

+

Vacaturenummer: 256/ATP/999

+ Meer info +
+

Onveilige kaart

+ Meer info +
+
diff --git a/fixtures/pages/vanroey-jobs.html b/fixtures/pages/vanroey-jobs.html new file mode 100644 index 0000000..b57bb3b --- /dev/null +++ b/fixtures/pages/vanroey-jobs.html @@ -0,0 +1,5 @@ +
+

Network Engineer

Bouw veilige netwerken voor klanten vanuit de Kempense vestigingen.

+

Oost-Vlaanderen Consultant

+

Fake Engineer

+
diff --git a/fixtures/pages/xplore-group-jobs.html b/fixtures/pages/xplore-group-jobs.html new file mode 100644 index 0000000..21447cf --- /dev/null +++ b/fixtures/pages/xplore-group-jobs.html @@ -0,0 +1,8 @@ +
+

Platform Engineer

+

Locations: Hasselt, Kontich

+

Data Engineer

+

Locations: Kontich

+

Fake

+

Locations: Hasselt

+
diff --git a/fixtures/pages/ziekenhuis-geel-jobs.html b/fixtures/pages/ziekenhuis-geel-jobs.html new file mode 100644 index 0000000..9246306 --- /dev/null +++ b/fixtures/pages/ziekenhuis-geel-jobs.html @@ -0,0 +1,11 @@ + +
+
ICT Support Engineer
+

Ondersteunende en technische diensten · 100%

+ Lees meer +
+ + diff --git a/fixtures/pages/zol-jobs.html b/fixtures/pages/zol-jobs.html new file mode 100644 index 0000000..06c30e3 --- /dev/null +++ b/fixtures/pages/zol-jobs.html @@ -0,0 +1,8 @@ +
+ +
(Senior) IT Engineer Infrastructuur
+ + +
Onveilige vacature
+ +
diff --git a/pyproject.toml b/pyproject.toml index a587467..abe4c73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "celery[redis]==5.6.3", "psycopg[binary]==3.3.4", "httpx==0.28.1", + "certifi==2026.6.17", "beautifulsoup4==4.15.0", "bleach==6.4.0", "lxml==6.1.1", @@ -20,6 +21,7 @@ dependencies = [ "defusedxml==0.7.1", "gunicorn==26.0.0", "whitenoise==6.11.0", + "cryptography==49.0.0", ] [project.optional-dependencies] diff --git a/scripts/codex_bootstrap.sh b/scripts/codex_bootstrap.sh index fcde560..b714d4f 100644 --- a/scripts/codex_bootstrap.sh +++ b/scripts/codex_bootstrap.sh @@ -16,7 +16,10 @@ CELERY_TASK_ALWAYS_EAGER=1 VACATURERADAR_ADMIN_USERNAME=admin VACATURERADAR_ADMIN_PASSWORD=codex-local-only EMAIL_BACKEND=django.core.mail.backends.console.EmailBackend -IMAP_ENABLED=0 +MAILBOX_CREDENTIAL_KEYS= +IMAP_CONNECT_TIMEOUT_SECONDS=15 +IMAP_MAX_MESSAGES_PER_POLL=200 +IMAP_MAX_MESSAGE_BYTES=1000000 SOURCE_POLICY_MODE=strict OLLAMA_ENABLED=0 ENV diff --git a/scripts/codex_verify.sh b/scripts/codex_verify.sh index 8b3233e..3ccb598 100644 --- a/scripts/codex_verify.sh +++ b/scripts/codex_verify.sh @@ -3,10 +3,27 @@ set -euo pipefail cd "$(dirname "$0")/.." mkdir -p local -uv run ruff check . -uv run python manage.py check -uv run python manage.py makemigrations --check --dry-run -uv run pytest --cov=apps --cov=config --cov-report=term-missing -python scripts/validate_task_ledger.py -python scripts/validate_repository.py +if command -v uv >/dev/null 2>&1; then + PYTHON=(uv run python) + RUFF=(uv run ruff) + PYTEST=(uv run pytest) +elif [[ -x .venv/Scripts/python.exe ]]; then + PYTHON=(.venv/Scripts/python.exe) + RUFF=(.venv/Scripts/python.exe -m ruff) + PYTEST=(.venv/Scripts/python.exe -m pytest) +elif [[ -x .venv/bin/python ]]; then + PYTHON=(.venv/bin/python) + RUFF=(.venv/bin/python -m ruff) + PYTEST=(.venv/bin/python -m pytest) +else + printf 'Geen uv of project-venv gevonden.\n' >&2 + exit 1 +fi + +"${RUFF[@]}" check . +"${PYTHON[@]}" manage.py check +"${PYTHON[@]}" manage.py makemigrations --check --dry-run +"${PYTEST[@]}" --cov=apps --cov=config --cov-report=term-missing +"${PYTHON[@]}" scripts/validate_task_ledger.py +"${PYTHON[@]}" scripts/validate_repository.py printf '\nAlle kwaliteitsgates geslaagd.\n' diff --git a/scripts/release_verify.sh b/scripts/release_verify.sh index 0efe0ff..6e173d3 100644 --- a/scripts/release_verify.sh +++ b/scripts/release_verify.sh @@ -72,8 +72,10 @@ VACATURERADAR_ADMIN_PASSWORD=${admin_password} VACATURERADAR_ADMIN_EMAIL=release-smoke@invalid EMAIL_BACKEND=django.core.mail.backends.locmem.EmailBackend DIGEST_RECIPIENT=release-smoke@invalid -IMAP_ENABLED=0 -IMAP_MARK_SEEN=0 +MAILBOX_CREDENTIAL_KEYS= +IMAP_CONNECT_TIMEOUT_SECONDS=15 +IMAP_MAX_MESSAGES_PER_POLL=200 +IMAP_MAX_MESSAGE_BYTES=1000000 SOURCE_POLICY_MODE=strict OLLAMA_ENABLED=0 SESSION_COOKIE_SECURE=0 diff --git a/scripts/validate_repository.py b/scripts/validate_repository.py index 86d1cbb..d92355e 100644 --- a/scripts/validate_repository.py +++ b/scripts/validate_repository.py @@ -18,7 +18,10 @@ REQUIRED_ENV_KEYS = { "POSTGRES_PASSWORD", "REDIS_URL", "SOURCE_POLICY_MODE", - "IMAP_ENABLED", + "MAILBOX_CREDENTIAL_KEYS", + "IMAP_CONNECT_TIMEOUT_SECONDS", + "IMAP_MAX_MESSAGES_PER_POLL", + "IMAP_MAX_MESSAGE_BYTES", "OLLAMA_ENABLED", } REQUIRED_DOCS = { diff --git a/static/css/app.css b/static/css/app.css index fd407c0..2454e65 100644 --- a/static/css/app.css +++ b/static/css/app.css @@ -1,21 +1,21 @@ :root, :root[data-theme="dark"] { color-scheme: dark; - --bg: #081120; - --bg-deep: #050c18; - --surface: #111b2e; - --surface-low: #0c1627; - --surface-high: #19253a; - --surface-bright: #243149; - --text: #e7efff; - --text-soft: #c7d3e7; - --muted: #8fa1b8; - --border: #2b3a50; - --border-strong: #42536b; - --primary: #16e5ee; - --primary-strong: #7cf8ff; - --primary-ink: #002f32; - --primary-soft: rgba(22, 229, 238, .1); + --bg: #0b1326; + --bg-deep: #060e20; + --surface: #171f33; + --surface-low: #131b2e; + --surface-high: #222a3d; + --surface-bright: #2d3449; + --text: #dae2fd; + --text-soft: #b9caca; + --muted: #849495; + --border: #3a494a; + --border-strong: #59696a; + --primary: #00f5ff; + --primary-strong: #63f7ff; + --primary-ink: #003739; + --primary-soft: rgba(0, 245, 255, .1); --positive: #4edea3; --positive-soft: rgba(78, 222, 163, .11); --warning: #ffca69; @@ -25,10 +25,10 @@ --indigo: #9ea5ff; --shadow: 0 18px 55px rgba(0, 0, 0, .24); --signal-glow: 0 0 0 1px rgba(22, 229, 238, .12), 0 0 28px rgba(22, 229, 238, .07); - --radius: 9px; - --radius-sm: 5px; - --sidebar: 272px; - --utility: 66px; + --radius: 8px; + --radius-sm: 4px; + --sidebar: 256px; + --utility: 64px; } :root[data-theme="light"] { @@ -143,10 +143,10 @@ button.utility-link { width: auto; } :root[data-theme="light"] .theme-icon-dark { display: none; } :root[data-theme="light"] .theme-icon-light { display: block; } .nav-scrim { display: none; } -.main-content { width: 100%; max-width: 1600px; margin: 0 auto; padding: 34px clamp(18px, 3.2vw, 52px) 72px; } +.main-content { width: 100%; max-width: 1280px; margin: 0 auto; padding: 32px 24px 72px; } .page-header { display: flex; align-items: flex-end; justify-content: space-between; gap: 26px; margin-bottom: 28px; } -.page-header h1, .job-hero h1 { margin: 0; font-size: clamp(30px, 3.7vw, 48px); line-height: 1.06; letter-spacing: -.042em; } +.page-header h1, .job-hero h1 { margin: 0; font-size: clamp(30px, 3.2vw, 42px); line-height: 1.08; letter-spacing: -.035em; } .page-header p { max-width: 760px; margin: 9px 0 0; color: var(--muted); } .eyebrow { margin: 0 0 9px !important; color: var(--primary) !important; font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: .13em; } .page-actions { display: flex; flex-wrap: wrap; gap: 9px; } @@ -223,6 +223,7 @@ button.utility-link { width: auto; } .filter-bar .grow { flex: 1; } .field-label { color: var(--muted); font: 10px ui-monospace, SFMono-Regular, Consolas, monospace; text-transform: uppercase; letter-spacing: .07em; } input, select, textarea { width: 100%; min-height: 43px; padding: 9px 11px; border: 1px solid var(--border); border-radius: 5px; background: var(--surface-low); color: var(--text); } +input[type="checkbox"], input[type="radio"] { width: 22px; min-height: 22px; height: 22px; padding: 0; accent-color: var(--primary); } input::placeholder, textarea::placeholder { color: color-mix(in srgb, var(--muted) 78%, transparent); } textarea { resize: vertical; } .table-wrap { overflow-x: auto; } @@ -282,6 +283,13 @@ textarea { resize: vertical; } .form-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 17px; } .form-grid label { display: grid; align-content: start; gap: 6px; color: var(--muted); font-size: 13px; font-weight: 650; } .form-grid label > input, .form-grid label > select, .form-grid label > textarea { color: var(--text); font-weight: 400; } +.choice-field { min-width: 0; margin: 0; padding: 0; border: 0; } +.choice-field legend { margin-bottom: 9px; color: var(--muted); font-size: 13px; font-weight: 700; } +.choice-field > div { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 9px; } +.choice-field > div > div { min-width: 0; } +.choice-field label { display: flex; align-items: center; gap: 9px; min-height: 44px; padding: 9px 11px; border: 1px solid var(--border); border-radius: 10px; background: color-mix(in srgb, var(--surface) 92%, transparent); color: var(--text); font-size: 13px; font-weight: 600; cursor: pointer; } +.choice-field label:has(input:checked) { border-color: color-mix(in srgb, var(--primary) 62%, var(--border)); background: color-mix(in srgb, var(--primary) 10%, var(--surface)); } +.choice-field small { display: block; margin-top: 7px; } .form-section { grid-column: 1 / -1; margin: 12px 0 -2px; padding-top: 18px; border-top: 1px solid var(--border); } .form-section h2 { margin: 0; font-size: 17px; } .form-section p { margin: 4px 0 0; color: var(--muted); font-size: 12px; } @@ -290,6 +298,18 @@ textarea { resize: vertical; } .form-actions { position: sticky; bottom: 12px; z-index: 5; display: flex; justify-content: flex-end; gap: 10px; margin-top: 24px; padding: 12px; border: 1px solid var(--border); border-radius: 7px; background: color-mix(in srgb, var(--surface) 92%, transparent); backdrop-filter: blur(12px); } .source-overview { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-bottom: 20px; } +.source-alert-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; margin-top: 16px; } +.source-alert-card { display: flex; align-items: flex-end; justify-content: space-between; gap: 18px; padding: 16px; border: 1px solid var(--border); border-radius: var(--radius-sm); background: var(--surface-soft); } +.source-alert-card p { margin: 6px 0 0; color: var(--muted); } +.source-alert-card .button { flex: 0 0 auto; } +.source-alert-card small { display: block; margin-top: 8px; color: var(--muted); } +.source-alert-card > .job-actions { flex: 0 0 auto; justify-content: flex-end; } +.mailbox-disclosure { margin-top: 16px; } +.mailbox-form-grid { margin-bottom: 14px; } +.checkbox-row { display: inline-flex; align-items: center; gap: 9px; color: var(--text); font-size: 14px; font-weight: 700; } +.checkbox-row input { width: 18px; height: 18px; accent-color: var(--accent); } +.platform-help-grid { margin-top: 12px; } +.button:disabled { cursor: not-allowed; opacity: .48; } .source-card { overflow: hidden; box-shadow: none; } .source-card-head { display: flex; align-items: flex-start; justify-content: space-between; gap: 20px; padding: 18px; } .source-identity { min-width: 0; display: flex; gap: 13px; } @@ -354,7 +374,8 @@ textarea { resize: vertical; } .page-header, .profile-card, .application-card, .job-hero { align-items: stretch; flex-direction: column; } .page-header .button, .page-actions { width: 100%; } .page-actions .button { flex: 1; } - .stats-grid, .two-columns, .form-grid, .definition-grid, .source-overview, .source-details-grid, .import-layout, .dashboard-rail { grid-template-columns: 1fr; } + .stats-grid, .two-columns, .form-grid, .definition-grid, .source-overview, .source-alert-grid, .source-details-grid, .import-layout, .dashboard-rail { grid-template-columns: 1fr; } + .source-alert-card { align-items: flex-start; flex-direction: column; } .filter-bar { align-items: stretch; flex-direction: column; } .span-2, .form-section { grid-column: auto; } .job-card-head { grid-template-columns: 1fr; } @@ -366,6 +387,7 @@ textarea { resize: vertical; } .source-list li { flex-direction: column; gap: 3px; } .source-card-head { flex-direction: column; } .source-actions { width: 100%; justify-content: space-between; } + .choice-field > div { grid-template-columns: 1fr; } .form-actions { bottom: 8px; } } @@ -382,3 +404,188 @@ textarea { resize: vertical; } @media (prefers-reduced-motion: reduce) { *, *::before, *::after { scroll-behavior: auto !important; transition: none !important; animation: none !important; } } + +/* VR-119: Stitch-compositie is leidend voor shell, feeds en intelligencepanelen. */ +.utilitybar-start { min-width: 0; flex: 1; gap: 18px; } +.global-search { position: relative; width: min(440px, 44vw); } +.global-search .icon { position: absolute; left: 13px; top: 50%; z-index: 1; width: 17px; height: 17px; transform: translateY(-50%); color: var(--muted); pointer-events: none; } +.global-search input { min-height: 42px; padding-left: 42px; background: var(--surface-low); } +.utility-radar { white-space: nowrap; } +.utility-cta { min-height: 40px; font-size: 13px; } +.section-link, .view-indicator { display: inline-flex; align-items: center; gap: 8px; } +.section-link .icon, .view-indicator .icon { width: 17px; height: 17px; } +.view-indicator { min-height: 42px; padding: 8px 13px; border: 1px solid var(--border); border-radius: var(--radius-sm); background: var(--surface-low); color: var(--text-soft); font: 12px ui-monospace, SFMono-Regular, Consolas, monospace; text-transform: uppercase; letter-spacing: .06em; } + +.dashboard-header { align-items: end; } +.dashboard-header strong { color: var(--primary); } +.radar-status { display: inline-flex; align-items: center; gap: 9px; max-width: 430px; padding: 10px 14px; border: 1px solid var(--border); border-radius: 999px; background: var(--surface-low); color: var(--text-soft); font: 11px ui-monospace, SFMono-Regular, Consolas, monospace; } +.radar-status a { margin-left: 3px; } +.dashboard-grid { grid-template-columns: minmax(0, 1fr) 310px; gap: 28px; } +.dashboard-matches { min-width: 0; } +.job-feed, .results-feed { display: grid; gap: 12px; } +.feed-card { display: grid; grid-template-columns: minmax(0, 1fr) auto; align-items: stretch; padding: 0; gap: 0; overflow: hidden; } +.job-card-main { min-width: 0; display: grid; gap: 14px; padding: 20px; } +.feed-card-actions { min-width: 152px; display: flex; flex-direction: column; align-items: stretch; justify-content: center; gap: 9px; padding: 18px; border-left: 1px solid var(--border); background: var(--surface-low); } +.feed-card-actions form { margin: 0; } +.feed-card-actions .save-button { width: 100%; border-color: var(--border); } +.confidence-label { color: var(--muted); font: 10px ui-monospace, SFMono-Regular, Consolas, monospace; text-transform: uppercase; letter-spacing: .06em; } +.signal-list { display: grid; gap: 7px; margin: 0; padding: 0; list-style: none; color: var(--text-soft); } +.signal-list li { position: relative; padding-left: 20px; } +.signal-list li::before { content: ""; position: absolute; left: 2px; top: .62em; width: 6px; height: 6px; border-radius: 50%; background: var(--positive); } +.rail-panel { padding: 22px; } +.rail-panel h2 { margin: 8px 0 10px; font-size: 20px; } +.rail-application-list { display: grid; margin: 8px 0 16px; border-top: 1px solid var(--border); } +.rail-application { min-width: 0; display: flex; align-items: center; justify-content: space-between; gap: 12px; padding: 12px 0; border-bottom: 1px solid var(--border); color: var(--text); } +.rail-application:hover { color: var(--primary-strong); text-decoration: none; } +.rail-application > span { min-width: 0; display: grid; } +.rail-application strong, .rail-application small { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.rail-application small { margin-top: 3px; color: var(--muted); } +.rail-application time { color: var(--warning); font: 11px ui-monospace, SFMono-Regular, Consolas, monospace; } +.compact-empty { min-height: 235px; display: grid; place-content: center; } + +.filter-console { display: grid; grid-template-columns: minmax(250px, 1fr) 170px 170px auto; align-items: end; gap: 10px; padding: 16px; margin-bottom: 26px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--surface); } +.filter-console label { display: grid; gap: 6px; } +.input-with-icon { position: relative; display: block; } +.input-with-icon .icon { position: absolute; left: 12px; top: 50%; width: 16px; height: 16px; transform: translateY(-50%); color: var(--muted); } +.input-with-icon input { padding-left: 38px; } +.results-toolbar { display: flex; align-items: end; justify-content: space-between; gap: 16px; margin: 0 0 14px; } +.results-toolbar h2 { margin: 4px 0 0; font-size: 20px; } +.result-card { position: relative; min-width: 0; display: grid; grid-template-columns: 4px minmax(0, 1fr) auto; overflow: hidden; border: 1px solid var(--border); border-radius: var(--radius); background: var(--surface); transition: border-color .16s ease, transform .16s ease; } +.result-card:hover { border-color: var(--border-strong); transform: translateY(-1px); } +.result-card.strong-match { border-color: color-mix(in srgb, var(--primary) 38%, var(--border)); box-shadow: var(--signal-glow); } +.result-accent { background: var(--surface-bright); } +.result-card.strong-match .result-accent { background: var(--primary); } +.result-body { min-width: 0; display: grid; gap: 13px; padding: 19px 22px; } +.result-heading { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 18px; } +.result-heading h2 { margin: 7px 0 2px; font-size: 21px; line-height: 1.25; } +.result-heading h2 a { color: var(--text); } +.result-heading p, .result-summary { margin: 0; color: var(--muted); } +.result-summary { max-width: 78ch; font-size: 13px; } +.result-actions { min-width: 174px; display: grid; place-content: center; padding: 18px; border-left: 1px solid var(--border); background: var(--surface-low); } + +.intelligence-strip { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin: 0 0 18px; } +.intelligence-strip > div { min-height: 110px; display: grid; align-content: center; gap: 4px; padding: 18px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--surface); } +.intelligence-strip strong { color: var(--primary-strong); font-size: 24px; letter-spacing: -.025em; } +.intelligence-strip small { color: var(--muted); } + +.pipeline-page-header { align-items: end; } +.pipeline-page-header h1 { font-size: clamp(30px, 3vw, 40px); } +.pipeline { min-height: calc(100vh - 250px); grid-template-columns: repeat(6, minmax(270px, 1fr)); gap: 16px; } +.pipeline-column { min-height: 460px; padding: 14px; background: color-mix(in srgb, var(--surface-low) 72%, transparent); } +.pipeline-head h2 { display: flex; align-items: center; gap: 8px; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-weight: 500; } +.column-dot { width: 8px; height: 8px; border-radius: 50%; background: var(--border-strong); } +.pipeline-column:nth-child(2) .column-dot, .pipeline-column:nth-child(3) .column-dot { background: var(--primary); } +.pipeline-column:nth-child(4) .column-dot { background: var(--positive); } +.pipeline-column:nth-child(5) .column-dot, .pipeline-column:nth-child(6) .column-dot { background: var(--danger); } +.pipeline-card { padding: 17px; background: var(--surface); } + +.editor-layout { display: grid; grid-template-columns: minmax(0, 1fr) 300px; gap: 22px; align-items: start; } +.profile-editor { max-width: none; padding: 28px; } +.profile-editor .form-section { display: flex; align-items: center; gap: 12px; margin-top: 20px; } +.profile-editor .form-section:first-child { margin-top: 0; padding-top: 0; border-top: 0; } +.form-section-icon { width: 38px; height: 38px; display: grid; place-items: center; flex: 0 0 auto; border: 1px solid var(--border); border-radius: var(--radius-sm); background: var(--surface-high); color: var(--primary); } +.editor-rail { position: sticky; top: calc(var(--utility) + 20px); } +.radar-preview { position: relative; overflow: hidden; } +.radar-preview::after { content: ""; position: absolute; right: -45px; top: -45px; width: 150px; height: 150px; border: 1px solid color-mix(in srgb, var(--primary) 15%, transparent); border-radius: 50%; box-shadow: inset 0 0 0 32px color-mix(in srgb, var(--primary) 3%, transparent), inset 0 0 0 62px color-mix(in srgb, var(--primary) 3%, transparent); pointer-events: none; } +.radar-preview > * { position: relative; z-index: 1; } +.preview-score { display: grid; margin: 20px 0; padding: 17px; border: 1px solid color-mix(in srgb, var(--primary) 30%, var(--border)); border-radius: var(--radius); background: var(--bg-deep); } +.preview-score strong { color: var(--primary); font-size: 36px; line-height: 1; } +.preview-score span { margin-top: 5px; color: var(--muted); font: 10px ui-monospace, SFMono-Regular, Consolas, monospace; text-transform: uppercase; } + +.import-disclosure { padding: 0; overflow: hidden; } +.import-disclosure > summary { min-height: 82px; display: grid; grid-template-columns: 44px minmax(0, 1fr) auto; align-items: center; gap: 14px; padding: 16px 20px; cursor: pointer; list-style: none; } +.import-disclosure > summary::-webkit-details-marker { display: none; } +.import-disclosure > summary > span:nth-child(2) { display: grid; gap: 3px; } +.import-disclosure > summary strong { font-size: 17px; } +.import-disclosure > summary small { color: var(--muted); } +.import-disclosure[open] > summary { border-bottom: 1px solid var(--border); } +.import-disclosure[open] > summary .disclosure-icon { transform: rotate(180deg); } +.import-disclosure-body { padding: 20px; } +.source-list-heading { align-items: end; } +.inline-bulk-action { display: flex; align-items: end; gap: 8px; } +.inline-bulk-action select { min-width: 170px; } +.source-card { border-left: 3px solid var(--border-strong); } +.source-card:has(.pill-success) { border-left-color: var(--positive); } +.source-card:has(.pill-failed) { border-left-color: var(--danger); } +.source-card:has(.pill-review), .source-card:has(.pill-partial) { border-left-color: var(--warning); } + +.system-hero { display: grid; grid-template-columns: minmax(300px, 1.25fr) minmax(300px, 1fr); gap: 14px; } +.system-readiness { min-height: 134px; display: grid; grid-template-columns: 48px minmax(0, 1fr) 10px; align-items: center; gap: 16px; padding: 22px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--surface); } +.system-readiness.is-healthy { border-color: color-mix(in srgb, var(--positive) 35%, var(--border)); } +.system-readiness h2 { margin: 5px 0 2px; font-size: 22px; } +.system-readiness p { margin: 0; color: var(--muted); } +.system-counts { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; } +.system-counts article { display: grid; align-content: center; gap: 6px; padding: 17px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--surface-low); } +.system-counts strong { font-size: 25px; } +.health-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; } +.health-card { padding: 19px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--surface); } +.health-card-head { display: flex; align-items: start; justify-content: space-between; gap: 12px; } +.health-card h3 { margin: 5px 0 0; font-size: 17px; } +.health-card dl { display: grid; grid-template-columns: repeat(2, 1fr); gap: 8px; margin: 17px 0 0; } +.health-card dl div { padding: 9px; border: 1px solid var(--border); border-radius: var(--radius-sm); background: var(--surface-low); } +.health-card dt { color: var(--muted); font: 9px ui-monospace, SFMono-Regular, Consolas, monospace; text-transform: uppercase; } +.health-card dd { margin: 3px 0 0; font-weight: 750; } +.command-panel { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 14px; } +.command-panel > div { display: grid; gap: 6px; } +.command-panel pre { height: 100%; } + +@media (max-width: 1180px) { + .utility-radar { display: none; } + .dashboard-grid { grid-template-columns: minmax(0, 1fr) 280px; } + .dashboard-rail { position: sticky; display: block; } + .editor-layout { grid-template-columns: 1fr; } + .editor-rail { position: static; display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 14px; } + .health-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } +} + +@media (max-width: 900px) { + .main-content { padding-inline: 20px; } + .global-search { width: min(420px, 55vw); } + .utility-cta { width: 42px; padding: 0; } + .dashboard-grid { grid-template-columns: 1fr; } + .dashboard-rail { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); position: static; } + .feed-card { grid-template-columns: 1fr; } + .feed-card-actions { min-width: 0; flex-direction: row; justify-content: flex-start; border-top: 1px solid var(--border); border-left: 0; } + .feed-card-actions .save-button { width: 42px; } + .filter-console { grid-template-columns: minmax(0, 1fr) repeat(2, 150px); } + .filter-console .button { grid-column: 1 / -1; } + .system-hero { grid-template-columns: 1fr; } +} + +@media (max-width: 680px) { + .main-content { padding-inline: 14px; } + .utilitybar { gap: 8px; padding-inline: 12px; } + .utilitybar-start { gap: 8px; } + .global-search { width: min(100%, 260px); } + .global-search input { min-height: 40px; } + .dashboard-header { align-items: stretch; } + .radar-status { width: 100%; max-width: none; flex-wrap: wrap; border-radius: var(--radius); } + .dashboard-rail, .editor-rail, .health-grid { grid-template-columns: 1fr; } + .feed-card-actions .button { flex: 1; } + .filter-console { grid-template-columns: 1fr; } + .filter-console .button { grid-column: auto; } + .results-toolbar, .source-list-heading { align-items: stretch; flex-direction: column; } + .result-card { grid-template-columns: 3px minmax(0, 1fr); } + .result-actions { grid-column: 2; min-width: 0; place-content: stretch; padding-top: 0; border-top: 0; border-left: 0; background: var(--surface); } + .result-heading { grid-template-columns: 1fr; } + .result-heading .score-block { justify-items: start; } + .intelligence-strip { grid-template-columns: 1fr; } + .intelligence-strip > div { min-height: 92px; } + .pipeline { grid-template-columns: repeat(6, 258px); min-height: 410px; } + .pipeline-column { min-height: 380px; } + .profile-editor { padding: 18px; } + .inline-bulk-action { align-items: stretch; flex-direction: column; } + .inline-bulk-action select { min-width: 0; } + .system-counts, .command-panel { grid-template-columns: 1fr; } +} + +@media (max-width: 420px) { + .global-search { width: 46vw; } + .global-search input::placeholder { color: transparent; } + .utility-actions { gap: 2px; } + .utility-actions .icon-button, .utility-actions .utility-link, .utility-cta { width: 38px; min-height: 38px; } + .radar-status span:nth-child(3), .radar-status a { display: none; } + .feed-card-actions { padding: 13px; } + .result-body { padding: 17px; } + .result-actions { padding: 0 17px 17px; } +} diff --git a/static/favicon.svg b/static/favicon.svg new file mode 100644 index 0000000..6c73d5e --- /dev/null +++ b/static/favicon.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/templates/applications/list.html b/templates/applications/list.html index f47edac..491db83 100644 --- a/templates/applications/list.html +++ b/templates/applications/list.html @@ -1,18 +1,18 @@ {% extends "base.html" %} {% block title %}Sollicitaties · VacatureRadar{% endblock %} {% block content %} -

Persoonlijke opvolging

Sollicitatiepipeline

Vacaturesnapshots blijven beschikbaar wanneer de oorspronkelijke bron offline gaat.

+

Persoonlijke opvolging

Sollicitatiepipeline

Beheer je actieve trajecten, deadlines en bewaarde vacaturesnapshots.

KanbanVacature toevoegen
{% for column in pipeline_columns %}
-

{{ column.label }}

{{ column.applications|length }}
+

{{ column.label }}

{{ column.applications|length }}
{% for application in column.applications %} -
{{ application.get_status_display }}

{{ application.job.original_title }}

{{ application.job.employer_name }}{% if application.job.raw_location %} · {{ application.job.raw_location }}{% endif %}

{% if application.applied_at %}Verzonden {{ application.applied_at|date:"d/m/Y" }}{% endif %}{% if application.follow_up_date %}Opvolgen {{ application.follow_up_date|date:"d/m/Y" }}{% endif %}
Dossier
+
{{ application.get_status_display }}

{{ application.job.original_title }}

{{ application.job.employer_name }}{% if application.job.raw_location %} · {{ application.job.raw_location }}{% endif %}

{% if application.applied_at %}Verzonden {{ application.applied_at|date:"d/m/Y" }}{% endif %}{% if application.follow_up_date %}Opvolgen {{ application.follow_up_date|date:"d/m/Y" }}{% endif %}
Dossier
{% empty %}
Geen dossiers in deze fase.
{% endfor %}
{% endfor %}
-{% if not applications %}

Nog geen sollicitatiedossiers

Markeer een vacature als Gesolliciteerd om automatisch een persoonlijk dossier te maken.

Vacatures bekijken
{% endif %} +{% if not applications %}

Nog geen sollicitatiedossiers

Markeer een vacature als Gesolliciteerd om automatisch een persoonlijk dossier te maken.

Vacatures bekijken
{% endif %} {% endblock %} diff --git a/templates/base.html b/templates/base.html index af41d6b..1e098ae 100644 --- a/templates/base.html +++ b/templates/base.html @@ -7,6 +7,7 @@ {% block title %}{{ app_name }}{% endblock %} + @@ -60,14 +61,19 @@ -
Persoonlijke radar
+ +
Radar actief
- Zoeken + {% if navigation_profile %}Zoekprofiel aanpassen{% endif %}
{% csrf_token %}
diff --git a/templates/dashboard/today.html b/templates/dashboard/today.html index ec8724b..75fa9c3 100644 --- a/templates/dashboard/today.html +++ b/templates/dashboard/today.html @@ -1,10 +1,20 @@ {% extends "base.html" %} {% block title %}Vandaag · VacatureRadar{% endblock %} {% block content %} -
-

Intelligence cockpit

Goedemorgen, {{ user.first_name|default:user.username }}

Je persoonlijke radar brengt alleen actuele signalen en concrete volgende stappen samen.

-
Vacatures verkennen
+
+
+

Intelligence cockpit

+

Goedemorgen, {{ user.first_name|default:user.username }}

+

Er {% if new_today == 1 %}is {{ new_today }} nieuw signaal{% else %}zijn {{ new_today }} nieuwe signalen{% endif %} die je aandacht verdienen.

+
+
+ + Persoonlijke radar actief + + Systeem bekijken +
+
Nieuwe signalen{{ new_today }}vandaag gevonden
Actieve vacatures{{ active_jobs }}actueel in de radar
@@ -13,32 +23,42 @@
-
-
Gerangschikt voor jou

Beste matches

Matchscore en dataconfidence blijven bewust afzonderlijk zichtbaar.

Alles bekijken →
-
+
+
+
Gerangschikt voor jou

Beste nieuwe matches

Matchscore en datakwaliteit blijven afzonderlijk zichtbaar.

+ Alles bekijken +
+
{% for score in score_cards %} -
-
-
{{ score.get_recommendation_display }}

{{ score.job.original_title }}

{{ score.job.employer_name }}

-
{{ score.score|floatformat:0 }}%confidence {{ score.confidence|floatformat:2 }}
+
+
+
+
+
{{ score.get_recommendation_display }}Datakwaliteit {{ score.confidence|floatformat:2 }}
+

{{ score.job.original_title }}

+

{{ score.job.employer_name }}{% if score.job.raw_location %} · {{ score.job.raw_location }}{% endif %}

+
+
{{ score.score|floatformat:0 }}%match
+
+
{{ score.job.raw_location|default:"Locatie onbekend" }}{{ score.job.get_workplace_type_display }}
+ {% if score.positives %}
    {% for item in score.positives|slice:":2" %}
  • {{ item }}
    • {% endfor %}
{% endif %} + {% if score.concerns %}

Aandachtspunt: {{ score.concerns.0 }}

{% endif %}
-
{{ score.job.raw_location|default:"Locatie onbekend" }}{{ score.job.get_workplace_type_display }}
- {% if score.positives %}
    {% for item in score.positives|slice:":2" %}
  • {{ item }}
    • {% endfor %}
{% endif %} - {% if score.concerns %}

Aandachtspunt: {{ score.concerns.0 }}

{% endif %} -
-
{% csrf_token %}
-
{% csrf_token %}
- Analyse +
+ Bekijk analyse +
{% csrf_token %}
{% empty %} -

Nog geen beoordeelde vacatures

Importeer vacaturedata of activeer een toegestane bron. De eerste betrouwbare resultaten verschijnen hier automatisch.

Bronnen beheren
+

Nog geen beoordeelde vacatures

Importeer vacaturedata of activeer een toegestane bron. Betrouwbare resultaten verschijnen hier automatisch.

Bronnen beheren
{% endfor %}
-
{% endblock %} diff --git a/templates/jobs/detail.html b/templates/jobs/detail.html index da5215d..334f656 100644 --- a/templates/jobs/detail.html +++ b/templates/jobs/detail.html @@ -8,6 +8,11 @@

{{ job.get_status_display }}{% if job.date_posted %} · gepubliceerd {{ job.date_posted|date:"d/m/Y" }}{% endif %}

{{ job.original_title }}

{{ job.employer_name }}{% if job.raw_location %} · {{ job.raw_location }}{% endif %}

{{ job.get_workplace_type_display }}{% for type in job.employment_types %}{{ type }}{% endfor %}{% if job.hours_text %}{{ job.hours_text }}{% endif %}{% if job.valid_through %}sluit {{ job.valid_through|date:"d/m/Y" }}{% endif %}
{% if score %}
{{ score.score|floatformat:0 }}%{{ score.get_recommendation_display }}confidence {{ score.confidence|floatformat:2 }}
{% endif %}
+
+
Matchkwaliteit{% if score %}{{ score.score|floatformat:0 }}%{{ score.get_recommendation_display }}{% else %}—Nog niet beoordeeld{% endif %}
+
Datakwaliteit{{ job.extraction_confidence|floatformat:2 }}Extractieconfidence
+
Werkmodel{{ job.get_workplace_type_display }}{{ job.hours_text|default:"Uren onbekend" }}
+
{% if score %}
Deterministische analyse

Waarom deze score?

De inhoudelijke match en betrouwbaarheid worden afzonderlijk beoordeeld.

{{ score.get_recommendation_display }}
{% for name,value in score.components.items %}
{{ name|capfirst }}
{{ value|floatformat:1 }}
{% endfor %}

Waarom passend

    {% for item in score.positives %}
  • {{ item }}
    • {% empty %}
  • Nog onvoldoende positieve signalen.
    • {% endfor %}

Aandachtspunten

    {% for item in score.concerns %}
  • {{ item }}
    • {% empty %}
  • Geen specifieke aandachtspunten gedetecteerd.
    • {% endfor %}
{% if score.hard_exclusions %}
Harde uitsluitingen: {{ score.hard_exclusions|join:", " }}
{% endif %}
{% else %}

Nog geen persoonlijke score

Deze vacature heeft nog geen score voor je actieve zoekprofiel. De broninformatie en vacaturetekst blijven wel beschikbaar.

{% endif %} diff --git a/templates/jobs/list.html b/templates/jobs/list.html index f21c25d..7dc14c4 100644 --- a/templates/jobs/list.html +++ b/templates/jobs/list.html @@ -1,23 +1,36 @@ {% extends "base.html" %} {% block title %}Vacatures · VacatureRadar{% endblock %} {% block content %} -

Nieuwe matches & archief

Vacature-intelligentie

Doorzoek actuele en gearchiveerde vacatures zonder de herkomst of status uit het oog te verliezen.

-
- +
+

Nieuwe matches & archief

Vacature-intelligentie

Doorzoek actuele en gearchiveerde vacatures met score, herkomst en status in één scanbare feed.

+
+ + - +
-
Resultaten

{{ page_obj.paginator.count|default:jobs|length }} vacatures

-
+ +
Resultaten

{{ page_obj.paginator.count }} vacatures gevonden

Lijstweergave
+
{% for job in jobs %} -
-
{{ job.get_status_display }}

{{ job.original_title }}

{{ job.employer_name }}

-
{{ job.raw_location|default:"Locatie onbekend" }}{{ job.get_workplace_type_display }}gevonden {{ job.first_seen|date:"d/m/Y" }}
- {% if job.skills_required %}

Vereiste skills: {{ job.skills_required|slice:":3"|join:", " }}

{% endif %} -
Bekijk analyse
+
+ +
+
+
+
{{ job.get_status_display }}{% if job.match_recommendation %}{% if job.match_recommendation == 'strong' %}Sterke match{% elif job.match_recommendation == 'possible' %}Mogelijke match{% elif job.match_recommendation == 'weak' %}Lage match{% else %}Verborgen{% endif %}{% endif %}
+

{{ job.original_title }}

+

{{ job.employer_name }}

+
+ {% if job.match_score is not None %}
{{ job.match_score|floatformat:0 }}%confidence {{ job.match_confidence|floatformat:2 }}
{% endif %} +
+
{{ job.raw_location|default:"Locatie onbekend" }}{{ job.get_workplace_type_display }}gevonden {{ job.first_seen|date:"d/m/Y" }}{% if job.direct_employer %}Directe werkgever{% endif %}
+ {% if job.skills_required %}

Herkenbare skills: {{ job.skills_required|slice:":4"|join:", " }}

{% elif job.description_text %}

{{ job.description_text|truncatewords:24 }}

{% endif %} +
+
Bekijk analyse
-{% empty %}

Geen vacatures gevonden

Pas de zoekterm of filters aan. Gearchiveerde resultaten verschijnen wanneer je de statusfilter leegmaakt.

{% endfor %} +{% empty %}

Geen vacatures gevonden

Pas de zoekterm of filters aan. Gearchiveerde resultaten verschijnen wanneer je de statusfilter leegmaakt.

{% endfor %}
{% if is_paginated %}{% endif %} {% endblock %} diff --git a/templates/profiles/edit.html b/templates/profiles/edit.html index 0ead213..83c24db 100644 --- a/templates/profiles/edit.html +++ b/templates/profiles/edit.html @@ -1,11 +1,21 @@ {% extends "base.html" %} {% block title %}{{ object.name }} bewerken · VacatureRadar{% endblock %} {% block content %} -

Zoekprofiel · versie {{ object.version }}

{{ object.name }} bewerken

Je voorkeuren sturen ranking en uitsluiting; bronbeleid en veiligheidsregels blijven automatisch bewaakt.

{% if object.is_active %}Actief profiel{% else %}Inactief profiel{% endif %}
-
{% csrf_token %}{% if form.non_field_errors %}
{{ form.non_field_errors }}
{% endif %}
-{% for field in form %} - {% if field.name == 'name' %}

Profielbasis

Naam, status en thuislocatie voor afstandsbepaling.

{% elif field.name == 'desired_titles_text' %}

Functies en vaardigheden

Eén titel of skill per regel; uitsluitingen hebben voorrang op voorkeuren.

{% elif field.name == 'allowed_employment_types' %}

Werkvorm en regio

Lijstvelden gebruiken JSON-notatie, bijvoorbeeld ["hybrid", "on_site"].

{% elif field.name == 'recommendation_threshold' %}

Scoring en automatisering

Topmatch moet minstens gelijk zijn aan de aanbevelingsdrempel.

{% endif %} - -{% endfor %} -
Annuleren
+

Zoekprofiel · versie {{ object.version }}

Configureer je radar

Verfijn {{ object.name }} zodat alleen relevante, uitlegbare kansen bovenaan verschijnen.

{% if object.is_active %}Actief profiel{% else %}Inactief profiel{% endif %}
+
+
{% csrf_token %} + {% if form.non_field_errors %}
{{ form.non_field_errors }}
{% endif %} +
+ {% for field in form %} + {% if field.name == 'name' %}

Profielbasis

Je postcode volstaat; technische locatievelden worden op de achtergrond afgeleid.

{% elif field.name == 'desired_titles' %}

Gewenste rollen & technologie

Kies herkenbare opties. Uitsluitingen blijven altijd doorslaggevend.

{% elif field.name == 'allowed_employment_types' %}

Werkvorm, regio & reistijd

Selecteer contractvormen, werkmodellen en regio's zonder technische notatie.

{% elif field.name == 'recommendation_threshold' %}

Scoring & automatisering

Een topmatch ligt altijd boven de gewone aanbevelingsdrempel.

{% endif %} + {% if field.field.widget.allow_multiple_selected %}
{{ field.label }}{{ field }}{% if field.help_text %}{{ field.help_text }}{% endif %}{% for error in field.errors %}{{ error }}{% endfor %}
{% else %}{% endif %} + {% endfor %} +
+
Annuleren
+
+ +
{% endblock %} diff --git a/templates/profiles/list.html b/templates/profiles/list.html index b28de07..4d30fbe 100644 --- a/templates/profiles/list.html +++ b/templates/profiles/list.html @@ -2,5 +2,5 @@ {% block title %}Zoekprofielen · VacatureRadar{% endblock %} {% block content %}

Persoonlijke regels

Zoekprofiel

Harde grenzen veranderen nooit automatisch; zachte voorkeuren kunnen alleen gecontroleerd leren.

-
{% for profile in profiles %}
{% if profile.is_active %}Actief{% else %}Inactief{% endif %}

{{ profile.name }}

{{ profile.desired_titles|join:", "|default:"Nog geen functietitels" }}

max. {{ profile.max_distance_km }} kmdigest {{ profile.digest_time|time:"H:i" }}versie {{ profile.version }}leren {% if profile.learning_enabled %}aan{% else %}uit{% endif %}
Profiel bewerken{% if not profile.is_active %}
{% csrf_token %}
{% endif %}
{% empty %}

Geen profiel beschikbaar

Voer de bootstrapopdracht uit om het standaardprofiel aan te maken.

{% endfor %}
+
{% for profile in profiles %}
{% if profile.is_active %}Actief{% else %}Inactief{% endif %}

{{ profile.name }}

{{ profile.desired_titles|join:", "|default:"Nog geen functietitels" }}

{% if profile.home_postal_code %}{{ profile.home_postal_code }}{% if profile.home_municipality %} · {{ profile.home_municipality }}{% endif %}{% else %}geen thuispostcode{% endif %}max. {{ profile.max_distance_km }} kmdigest {{ profile.digest_time|time:"H:i" }}versie {{ profile.version }}leren {% if profile.learning_enabled %}aan{% else %}uit{% endif %}
Profiel bewerken{% if not profile.is_active %}
{% csrf_token %}
{% endif %}
{% empty %}

Geen profiel beschikbaar

Voer de bootstrapopdracht uit om het standaardprofiel aan te maken.

{% endfor %}
{% endblock %} diff --git a/templates/sources/list.html b/templates/sources/list.html index a0d1ba0..1a16fa9 100644 --- a/templates/sources/list.html +++ b/templates/sources/list.html @@ -6,12 +6,68 @@
Actieve bronnen{{ active_source_count }}van {{ source_total_count }} geregistreerd
-
Policyreview{{ review_pending_count }}bronnen te beoordelen
+
Werkgeverswaaklijst{{ watchlist_count }}interessante regionale werkgevers
Veilige modusAANpolicy + SSRF-validatie
-
-
Gecontroleerde invoer

Handmatige import

Gebruik een publieke URL of plak vacaturetekst. URL-import passeert altijd het bronbeleid.

+
+
Regionale werkgeversradar

2400 Mol · straal 40 km

De werkgeverscatalogus is rond Mol gereviewd. Afstanden gebruiken lokale postcodecoördinaten en blijven afhankelijk van de concrete werklocatie in de vacature.

{% if active_search_profile.home_latitude != None and active_search_profile.home_longitude != None %}Afstanden actief{% else %}Profiel controleren{% endif %}
+ {% if active_search_profile %}

Actief profiel: postcode {{ active_search_profile.home_postal_code|default:"niet ingevuld" }} · {{ active_search_profile.home_municipality|default:"gemeente onbekend" }} · maximaal {{ active_search_profile.max_distance_km }} km.{% if active_search_profile.home_latitude == None or active_search_profile.home_longitude == None %} Coördinaten ontbreken nog; vacatureafstanden worden niet geschat.{% else %} Postcodecentrum gecontroleerd via de lokale GeoNames-dataset.{% endif %}

{% endif %} +
+ +{% if watchlist_sources %}
+
Ook volgen zonder vacature

Interessante werkgeverswaaklijst

Deze werkgevers liggen in de ruime regio en blijven zichtbaar, ook wanneer er nu geen geschikte vacature is. Ze worden pas automatisch gescand na een stabiele, veilige parserproef.

{{ watchlist_count }} gevolgd
+
+ {% for source in watchlist_sources %}
{{ source.name }}Waaklijst

{{ source.metadata.municipality_scope }} · {{ source.metadata.watchlist_reason }}

{{ source.metadata.current_observation }}
Open carrièrepagina
{% endfor %} +
+
{% endif %} + +
+
Brede marktdekking

Platformmailboxen

Koppel per jobboard een aparte alertmailbox. De scheduler controleert elke vijf minuten welke mailbox volgens haar eigen interval aan de beurt is.

{{ mailbox_connections|length }} gekoppeld
+ + {% if not mailbox_credential_key_ready %}

Credentialopslag is nog vergrendeld. Stel eerst MAILBOX_CREDENTIAL_KEYS in; app-wachtwoorden worden nooit onversleuteld opgeslagen.

{% endif %} + +
+ {% for connection in mailbox_connections %} +
+
{{ connection.get_platform_display }}-mailbox{% if connection.enabled %}{% if connection.last_error_category %}Aandacht nodig{% else %}Actief{% endif %}{% else %}Gepauzeerd{% endif %}
+

{{ connection.get_provider_display }} · {{ connection.username }} · {{ connection.get_poll_interval_minutes_display|lower }}

+ {% if connection.last_success_at %}Laatste import {{ connection.last_success_at|date:"d/m/Y H:i" }}.{% else %}Nog geen geslaagde import.{% endif %} {% if connection.enabled %}Volgende controle {{ connection.next_poll_at|date:"d/m/Y H:i" }}.{% endif %} + {% if connection.last_error_message %}

{{ connection.last_error_message }}

{% endif %}
+
{% if connection.enabled %}
{% csrf_token %}
{% endif %}Wijzigen
{% csrf_token %}
+
+ {% empty %} +
Nog geen mailbox gekoppeld

Koppel hieronder per gewenst jobboard de mailbox die de bijbehorende vacaturealerts ontvangt.

+ {% endfor %} +
+ +
+ Versleutelde IMAP-koppeling{% if editing_mailbox %}{{ editing_mailbox.get_platform_display }}-mailbox wijzigen{% else %}Platformmailbox koppelen{% endif %}Gmail, Outlook / Microsoft 365 of aangepaste IMAP +
{% csrf_token %} +
+ + + + + + + + +
+ + {% for error in mailbox_form.non_field_errors %}
{{ error }}
{% endfor %} +
{% if editing_mailbox %}Annuleren{% endif %}
+
+
+ +
+ {% for platform in platform_alerts %}
{{ platform.label }}-alert instellen

{{ platform.help_text }} VacatureRadar verwerkt alleen links van {{ platform.label }} en logt niet in op het platform.

Open {{ platform.label }}
{% endfor %} +
+
+ +
+ Gecontroleerde invoerHandmatige importPublieke URL of vacaturetekst toevoegen +

URL-import passeert altijd het bronbeleid en de SSRF-validatie.

{% csrf_token %}
@@ -20,19 +76,17 @@ {% for error in manual_import_form.non_field_errors %}
{{ error }}
{% endfor %}
Bookmarklet installeren
- -
+
+ {% if manual_import_result %}
Importresultaat

Importresultaat

Bron: {{ manual_import_result.source_name }} ({{ manual_import_result.source_id }})

Modus
Modus: {{ manual_import_result.mode }}
Herkenbare items
Herkenbare items: {{ manual_import_result.extracted_count }}
Nieuwe vacatures
Nieuwe vacatures: {{ manual_import_result.created_count }}
Duplicaten
Duplicaten: {{ manual_import_result.duplicate_count }}
{% if manual_import_result.warnings %}

Waarschuwingen

    {% for warning in manual_import_result.warnings %}
  • {{ warning }}
    • {% endfor %}
{% endif %}{% if manual_import_result.jobs %}

Geïmporteerde vacatures

    {% for job in manual_import_result.jobs %}
  • {% if job.id %}{{ job.title }}{% else %}{{ job.title }}{% endif %}{{ job.employer }}
    • {% endfor %}
{% endif %}
{% endif %} -
{% csrf_token %}
- -
Endpoints

{{ source_total_count }} geregistreerde bronnen

Status is gebaseerd op bronbeleid en de meest recente run, indien beschikbaar.

+
Scan-endpoints

{{ source_total_count }} geregistreerde bronnen

Status is gebaseerd op bronbeleid en de meest recente run.

{% csrf_token %}
{% for source in sources %}{% with latest_run=source.runs.all|first %}
-
{% if source.policy != 'deny' %}{% endif %}

{{ source.name }}

{{ source.get_source_type_display }} · {{ source.domain|default:"Geen domein" }}

{{ source.get_policy_display }}{% if latest_run %}laatste run {{ latest_run.started_at|date:"d/m/Y H:i" }}{{ latest_run.extracted_count }} gevonden{% else %}nog niet gestart{% endif %}
+
{% if source.policy != 'deny' %}{% endif %}

{{ source.name }}

{{ source.get_source_type_display }} · {{ source.domain|default:"Geen domein" }}

{{ source.get_policy_display }}{% if latest_run %}laatste run {{ latest_run.started_at|date:"d/m/Y H:i" }}{{ source.job_count }} vacatures in radar{% else %}nog niet gestart{% endif %}
{% if source.policy == 'deny' %}Geblokkeerd{% elif source.status == 'quarantined' %}Quarantaine{% elif latest_run %}{{ latest_run.get_status_display }}{% else %}Geen run{% endif %}
{% if source.discovery_evidence %}
Ontdekkingsevidence
    {% for evidence in source.discovery_evidence %}
  • {{ evidence.label|default:"Ontdekte bron" }}: {{ evidence.url }}
    • {% endfor %}
{% endif %} diff --git a/templates/system/status.html b/templates/system/status.html index c033f42..51dd32b 100644 --- a/templates/system/status.html +++ b/templates/system/status.html @@ -1,11 +1,15 @@ {% extends "base.html" %} {% block title %}Systeemstatus · VacatureRadar{% endblock %} {% block content %} -

Automatisering centrum

Systeemstatus

Readiness en echte database-aantallen. Secrets, fictieve uptime en niet-gemeten latency worden nooit getoond.

{% if health.ok %}Operationeel{% else %}Aandacht nodig{% endif %}
-
Readiness{% if health.ok %}OK{% else %}FOUT{% endif %}database: {{ health.checks.database }}
{% for row in source_summary %}
Bronnen · {{ row.status }}{{ row.total }}geregistreerde bronnen
{% endfor %}{% for row in job_summary %}
Vacatures · {{ row.status }}{{ row.total }}vacatures in database
{% endfor %}
-
Operaties

Veilige beheeropdrachten

Voer deze uitsluitend uit vanaf een vertrouwde shell op de server.

-
make verify
-./scripts/backup.sh
-./scripts/restore.sh /pad/naar/backup.sql.gz
-uv run python manage.py seed_sources
+

Automatisering centrum

Radarbesturing

Readiness, bronruns en veilige beheeracties op basis van actuele systeemdata.

{% if health.ok %}Operationeel{% else %}Aandacht nodig{% endif %}
+
+
Systeemreadiness

{% if health.ok %}Radar is operationeel{% else %}Controle vereist{% endif %}

Databasecontrole: {{ health.checks.database }}

+
{% for row in source_summary %}
Bronnen · {{ row.status }}{{ row.total }}
{% endfor %}{% for row in job_summary %}
Vacatures · {{ row.status }}{{ row.total }}
{% endfor %}
+
+ +
Brontelemetrie

Recente rungezondheid

Meetwaarden uit maximaal de laatste 30 runs per bron.

Bronnen beheren
+
{% for row in source_health %}
{{ row.source_type }}

{{ row.source_name }}

{% if row.monitored_runs == 0 %}Geen runs{% else %}Ratio {{ row.success_ratio|floatformat:2 }}{% endif %}
Runs
{{ row.monitored_runs }}
Gevonden
{{ row.extracted_count }}
Bijgewerkt
{{ row.updated_count }}
Latency
{% if row.avg_latency_ms is not None %}{{ row.avg_latency_ms|floatformat:0 }} ms{% else %}—{% endif %}
{% if row.last_health_reason %}

{{ row.last_health_reason }}

{% endif %}
{% empty %}

Nog geen brontelemetrie

Bronruns verschijnen hier zodra een toegestane bron is uitgevoerd.

{% endfor %}
+ +
Operaties

Veilige beheeropdrachten

Uitsluitend uitvoeren vanaf een vertrouwde shell op de server.

+
Verificatie
make verify
Back-up
./scripts/backup.sh
Herstel
./scripts/restore.sh /pad/naar/backup.sql.gz
Bronnen initialiseren
uv run python manage.py seed_sources
{% endblock %} diff --git a/tests/e2e/test_vr114_browser_and_accessibility.py b/tests/e2e/test_vr114_browser_and_accessibility.py index 1855878..b2a8d2f 100644 --- a/tests/e2e/test_vr114_browser_and_accessibility.py +++ b/tests/e2e/test_vr114_browser_and_accessibility.py @@ -85,10 +85,14 @@ def _run_html_accessibility_probe(client, user, profile, job) -> None: response = client.get(reverse("dashboard:today")) assert response.status_code == 200 _assert_basic_html_accessibility(response.content, page_name="dashboard") + assert "Zoek door vacatures" in response.content.decode("utf-8") response = client.get(reverse("jobs:list"), {"q": job.original_title}) assert response.status_code == 200 _assert_basic_html_accessibility(response.content, page_name="jobs-list") + jobs_body = response.content.decode("utf-8") + assert "1 vacatures gevonden" in jobs_body + assert "88%" in jobs_body response = client.get(reverse("jobs:detail", kwargs={"pk": job.pk})) assert response.status_code == 200 @@ -186,12 +190,12 @@ def test_vr114_browser_flow(viewport, live_server, client, tmp_path, user, profi page.get_by_label("Wachtwoord").fill("correct-horse-battery-staple") page.get_by_role("button", name="Aanmelden").click() page.wait_for_url(dashboard_url) - _assert_playwright_accessibility(page, expected_h1="Vandaag") + _assert_playwright_accessibility(page, expected_h1="Goedemorgen") # Dashboard -> filter -> detail. page.get_by_role("link", name="Vacatures").click() page.get_by_role("textbox", name="Zoeken").fill(job.original_title) - page.get_by_role("button", name="Filter").click() + page.get_by_role("button", name="Filters toepassen").click() assert page.url.startswith(jobs_url) page.get_by_role("link", name=job.original_title).first.click() page.wait_for_url(detail_url) @@ -209,13 +213,12 @@ def test_vr114_browser_flow(viewport, live_server, client, tmp_path, user, profi page.wait_for_selector(".message.success") # Profiel aanpassen. - page.get_by_role("link", name="Profiel").click() + page.get_by_role("link", name="Zoekprofiel").click() page.wait_for_url(profile_list_url) - page.get_by_role("link", name="Bewerken").click() - page.get_by_label("Gewenste functietitels").fill( - "Infrastructure Engineer\nSysteembeheerder" - ) - page.get_by_role("button", name="Profiel opslaan").click() + page.get_by_role("link", name="Profiel bewerken").click() + page.get_by_label("Infrastructure engineer").check() + page.get_by_label("System engineer").check() + page.get_by_role("button", name="Instellingen opslaan").click() page.wait_for_url(profile_list_url) # Sollicitatie dossier openen en bijwerken. @@ -231,12 +234,13 @@ def test_vr114_browser_flow(viewport, live_server, client, tmp_path, user, profi (timezone.localdate() + timedelta(days=7)).isoformat() ) page.locator("textarea[name='notes']").fill("Eerste follow-up ingepland.") - page.get_by_role("button", name="Opslaan").click() + page.get_by_role("button", name="Dossier opslaan").click() page.wait_for_url(applications_url) # Handmatige import via plaktekst. - page.get_by_role("link", name="Brongezondheid").click() + page.get_by_role("link", name="Bronnen").click() page.wait_for_url(sources_url) + page.locator(".import-disclosure > summary").click() page.locator("textarea[name='pasted_text']").fill( "Vacaturetitel: Infrastructure Engineer\nWerkgever: Example IT\nLocatie: Hasselt" ) @@ -249,7 +253,7 @@ def test_vr114_browser_flow(viewport, live_server, client, tmp_path, user, profi page.keyboard.press("Tab") page.keyboard.press("Tab") page.keyboard.press("Tab") - _assert_playwright_accessibility(page, expected_h1="Brongezondheid") + _assert_playwright_accessibility(page, expected_h1="Bronnen") assert Feedback.objects.filter(user=user, job=job).count() >= 4 except Exception: diff --git a/tests/integration/test_email_import_notifications.py b/tests/integration/test_email_import_notifications.py index fef8084..6bccad8 100644 --- a/tests/integration/test_email_import_notifications.py +++ b/tests/integration/test_email_import_notifications.py @@ -3,6 +3,7 @@ from pathlib import Path from zoneinfo import ZoneInfo import pytest +from cryptography.fernet import Fernet from django.core import mail from django.test import override_settings @@ -10,8 +11,10 @@ from apps.jobs.models import Feedback, ScoreRun from apps.jobs.services.scoring import score_and_save from apps.notifications.models import DigestOutbox from apps.notifications.services import build_digest_payload, create_daily_outbox, send_digest -from apps.sources.models import EmailMessageRecord, RawDocument +from apps.sources.models import EmailMessageRecord, MailboxConnection, RawDocument from apps.sources.services.email_import import ingest_email, message_identity +from apps.sources.services.imap_import import poll_imap_mailbox +from apps.sources.services.mailbox_connections import encrypt_mailbox_password @pytest.mark.django_db @@ -27,6 +30,132 @@ def test_email_import_is_idempotent(profile): assert len(first.links) == 2 +@pytest.mark.django_db +def test_vdab_alert_has_provider_provenance_and_replays_idempotently(profile): + raw = Path("fixtures/emails/vdab_job_alert.eml").read_bytes() + + first = ingest_email(raw) + second = ingest_email(raw) + + assert second.pk == first.pk + assert first.processed is True + assert first.raw_document.metadata["alert_provider"] == "vdab" + assert len(first.links) == 2 + assert all("vdab.be/vindeenjob/vacatures/" in link for link in first.links) + + +@pytest.mark.django_db +@pytest.mark.parametrize( + ("platform", "fixture", "expected_domain"), + ( + (MailboxConnection.Platform.LINKEDIN, "linkedin_job_alert.eml", "linkedin.com"), + (MailboxConnection.Platform.ICTJOB, "ictjob_job_alert.eml", "ictjob.be"), + (MailboxConnection.Platform.JOBAT, "jobat_job_alert.eml", "jobat.be"), + (MailboxConnection.Platform.STEPSTONE, "stepstone_job_alert.eml", "stepstone.be"), + (MailboxConnection.Platform.CAREERJET, "careerjet_job_alert.eml", "careerjet.be"), + (MailboxConnection.Platform.RANDSTAD, "randstad_job_alert.eml", "randstad.be"), + ( + MailboxConnection.Platform.ROBERTHALF, + "roberthalf_job_alert.eml", + "roberthalf.com", + ), + ), +) +def test_jobboard_mail_import_keeps_platform_provenance_and_rejects_external_links( + profile, platform, fixture, expected_domain +): + raw = (Path("fixtures/emails") / fixture).read_bytes() + + record = ingest_email(raw, platform=platform) + + assert record.processed is True + assert len(record.links) == 1 + assert expected_domain in record.links[0] + assert "outside.example.invalid" not in record.links[0] + assert record.raw_document.metadata["alert_provider"] == platform + assert record.raw_document.source.name == f"{platform.label} vacaturemailbox" + + +@pytest.mark.django_db +def test_imap_poll_is_bounded_and_one_bad_message_does_not_abort(profile): + valid = Path("fixtures/emails/vdab_job_alert.eml").read_bytes() + oversized = b"x" * (len(valid) + 1) + + class FakeImapClient: + def __init__(self, host, port, timeout=None): + self.host = host + self.port = port + self.timeout = timeout + self.logged_out = False + + def login(self, user, password): + return "OK", [] + + def select(self, mailbox, readonly=True): + assert readonly is True + return "OK", [] + + def uid(self, command, *args): + if command == "search": + return "OK", [b"1 2 3"] + uid = args[0] + if uid == b"1": + return "OK", [(b"RFC822", valid)] + if uid == b"2": + return "OK", [(b"RFC822", oversized)] + return "NO", [] + + def logout(self): + self.logged_out = True + + clients = [] + + def factory(host, port, timeout=None): + client = FakeImapClient(host, port, timeout=timeout) + clients.append(client) + return client + + key = Fernet.generate_key().decode("ascii") + with override_settings( + MAILBOX_CREDENTIAL_KEYS=[key], + IMAP_MAX_MESSAGES_PER_POLL=3, + IMAP_MAX_MESSAGE_BYTES=len(valid), + ): + connection = MailboxConnection.objects.create( + user=profile.user, + platform=MailboxConnection.Platform.VDAB, + provider=MailboxConnection.Provider.CUSTOM, + custom_host="imap.example.invalid", + username="radar@example.invalid", + encrypted_password=encrypt_mailbox_password("test-only"), + ) + first = poll_imap_mailbox( + connection, client_factory=factory, host_validator=lambda url: url + ) + second = poll_imap_mailbox( + connection, client_factory=factory, host_validator=lambda url: url + ) + + assert first == { + "status": "ok", + "imported": 1, + "duplicates": 0, + "skipped_oversized": 1, + "failed": 1, + } + assert second == { + "status": "ok", + "imported": 0, + "duplicates": 1, + "skipped_oversized": 1, + "failed": 1, + } + assert all(client.logged_out for client in clients) + record = EmailMessageRecord.objects.get(mailbox_connection=connection) + assert record.raw_document.metadata["alert_provider"] == "vdab" + assert record.raw_document.metadata["mailbox_connection_id"] == connection.pk + + @pytest.mark.django_db def test_digest_payload_outbox_deduplication_and_send(profile, job): score = score_and_save(job, profile) diff --git a/tests/integration/test_mailbox_connections.py b/tests/integration/test_mailbox_connections.py new file mode 100644 index 0000000..86021a1 --- /dev/null +++ b/tests/integration/test_mailbox_connections.py @@ -0,0 +1,205 @@ +from datetime import timedelta + +import pytest +from cryptography.fernet import Fernet +from django.contrib.auth import get_user_model +from django.test import Client, override_settings +from django.urls import reverse +from django.utils import timezone + +from apps.sources.models import MailboxConnection +from apps.sources.services.mailbox_connections import ( + claim_mailbox_connection, + decrypt_mailbox_password, +) +from apps.sources.tasks import poll_mailbox, schedule_mailbox_polls + + +def _payload(**overrides): + payload = { + "platform": MailboxConnection.Platform.VDAB, + "provider": MailboxConnection.Provider.GMAIL, + "custom_host": "", + "port": 993, + "username": "vdab-alerts@example.invalid", + "password": "test-app-password", + "mailbox": "INBOX", + "poll_interval_minutes": 15, + "enabled": "on", + } + payload.update(overrides) + return payload + + +@pytest.mark.integration +@pytest.mark.django_db +def test_user_can_link_edit_pause_and_sync_separate_platform_mailboxes(client, user, monkeypatch): + key = Fernet.generate_key().decode("ascii") + calls = [] + monkeypatch.setattr( + "apps.sources.views.poll_mailbox.delay", + lambda connection_id, force=False: calls.append((connection_id, force)), + ) + client.force_login(user) + + with override_settings(MAILBOX_CREDENTIAL_KEYS=[key]): + response = client.post(reverse("sources:mailbox_add"), _payload()) + assert response.status_code == 302 + vdab = MailboxConnection.objects.get(user=user, platform="vdab") + assert decrypt_mailbox_password(vdab) == "test-app-password" + assert "test-app-password" not in vdab.encrypted_password + + response = client.post( + reverse("sources:mailbox_add"), + _payload( + platform=MailboxConnection.Platform.INDEED, + provider=MailboxConnection.Provider.OUTLOOK, + username="indeed-alerts@example.invalid", + password="second-app-password", + poll_interval_minutes=30, + ), + ) + assert response.status_code == 302 + indeed = MailboxConnection.objects.get(user=user, platform="indeed") + + edit_response = client.get(reverse("sources:mailbox_edit", args=[vdab.pk])) + assert edit_response.status_code == 200 + assert b"test-app-password" not in edit_response.content + assert vdab.encrypted_password.encode() not in edit_response.content + + original_ciphertext = vdab.encrypted_password + response = client.post( + reverse("sources:mailbox_edit", args=[vdab.pk]), + _payload( + username="vdab-updated@example.invalid", + password="", + poll_interval_minutes=5, + ), + ) + assert response.status_code == 302 + vdab.refresh_from_db() + assert vdab.username == "vdab-updated@example.invalid" + assert vdab.poll_interval_minutes == 5 + assert vdab.encrypted_password == original_ciphertext + assert decrypt_mailbox_password(vdab) == "test-app-password" + + response = client.post(reverse("sources:mailbox_toggle", args=[indeed.pk])) + assert response.status_code == 302 + indeed.refresh_from_db() + assert indeed.enabled is False + + response = client.post(reverse("sources:sync_platform_alerts", args=[vdab.pk])) + assert response.status_code == 302 + assert calls == [ + (vdab.pk, True), + (indeed.pk, True), + (vdab.pk, True), + (vdab.pk, True), + ] + + +@pytest.mark.integration +@pytest.mark.django_db +def test_sources_page_offers_all_reviewed_jobboard_mailboxes(client, user): + client.force_login(user) + + response = client.get(reverse("sources:list")) + + assert response.status_code == 200 + for platform in MailboxConnection.Platform: + assert f'value="{platform.value}"'.encode() in response.content + assert f"{platform.label}-alert instellen".encode() in response.content + + +@pytest.mark.integration +@pytest.mark.django_db +def test_mailbox_views_are_user_scoped_and_require_encryption_key(client, user, monkeypatch): + other = get_user_model().objects.create_user(username="other", password="test-only-password") + other_connection = MailboxConnection.objects.create( + user=other, + platform=MailboxConnection.Platform.VDAB, + provider=MailboxConnection.Provider.GMAIL, + username="private@example.invalid", + encrypted_password="opaque-ciphertext", + ) + client.force_login(user) + monkeypatch.setattr("apps.sources.views.poll_mailbox.delay", lambda *args, **kwargs: None) + + assert ( + client.get(reverse("sources:mailbox_edit", args=[other_connection.pk])).status_code == 404 + ) + assert ( + client.post(reverse("sources:sync_platform_alerts", args=[other_connection.pk])).status_code + == 404 + ) + assert ( + client.post(reverse("sources:mailbox_toggle", args=[other_connection.pk])).status_code + == 404 + ) + + with override_settings(MAILBOX_CREDENTIAL_KEYS=[]): + response = client.post(reverse("sources:mailbox_add"), _payload(password="visible-once")) + assert response.status_code == 200 + assert MailboxConnection.objects.filter(user=user).count() == 0 + assert b"visible-once" not in response.content + assert b"Credentialopslag is niet geconfigureerd" in response.content + + anonymous_response = Client().post(reverse("sources:mailbox_add"), _payload()) + assert anonymous_response.status_code == 302 + csrf_client = Client(enforce_csrf_checks=True) + csrf_client.force_login(user) + assert csrf_client.post(reverse("sources:mailbox_add"), _payload()).status_code == 403 + + +@pytest.mark.integration +@pytest.mark.django_db +def test_scheduler_queues_only_due_active_mailboxes_and_claim_is_exclusive(user, monkeypatch): + now = timezone.now() + due = MailboxConnection.objects.create( + user=user, + platform=MailboxConnection.Platform.VDAB, + provider=MailboxConnection.Provider.GMAIL, + username="due@example.invalid", + encrypted_password="opaque", + next_poll_at=now - timedelta(minutes=1), + ) + MailboxConnection.objects.create( + user=user, + platform=MailboxConnection.Platform.INDEED, + provider=MailboxConnection.Provider.OUTLOOK, + username="later@example.invalid", + encrypted_password="opaque", + next_poll_at=now + timedelta(minutes=20), + ) + calls = [] + monkeypatch.setattr( + "apps.sources.tasks.poll_mailbox.delay", lambda connection_id: calls.append(connection_id) + ) + + assert schedule_mailbox_polls() == {"scheduled": 1} + assert calls == [due.pk] + assert claim_mailbox_connection(connection_id=due.pk, worker_token="worker-a") is not None + assert claim_mailbox_connection(connection_id=due.pk, worker_token="worker-b") is None + + +@pytest.mark.integration +@pytest.mark.django_db +def test_mailbox_task_does_not_persist_or_return_secret_on_imap_error(user, monkeypatch): + connection = MailboxConnection.objects.create( + user=user, + platform=MailboxConnection.Platform.VDAB, + provider=MailboxConnection.Provider.GMAIL, + username="errors@example.invalid", + encrypted_password="opaque", + next_poll_at=timezone.now() - timedelta(minutes=1), + ) + monkeypatch.setattr( + "apps.sources.tasks.poll_imap_mailbox", + lambda connection: (_ for _ in ()).throw(RuntimeError("leaked-app-password")), + ) + + result = poll_mailbox(connection.pk) + connection.refresh_from_db() + assert "leaked-app-password" not in str(result) + assert "leaked-app-password" not in connection.last_error_message + assert result == {"status": "failed", "error_category": "imap", "imported": 0} diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index a3652c0..9e9e9f5 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -34,6 +34,40 @@ def test_pipeline_is_idempotent_and_scores(source, profile): assert ScoreRun.objects.filter(profile=profile).count() == 2 +@pytest.mark.django_db +def test_pipeline_uses_reviewed_employer_identity_for_public_ats_feed(profile): + source = Source.objects.create( + name="Deliverect", + source_type=Source.Type.ATS, + base_url="https://api.lever.co/v0/postings/deliverect?mode=json", + domain="api.lever.co", + status=Source.Status.ACTIVE, + policy=Source.Policy.ALLOW, + metadata={"employer_name": "Deliverect", "direct_employer": True}, + ) + content = Path("fixtures/ats/lever-public-postings.json").read_text(encoding="utf-8") + document = RawDocument.objects.create( + source=source, + url=source.base_url, + final_url=source.base_url, + kind=RawDocument.Kind.JSON, + content_type="application/json", + content_hash=hashlib.sha256(content.encode()).hexdigest(), + body_text=content, + byte_length=len(content.encode()), + retain_until=timezone.now() + timedelta(days=7), + ) + + result = process_raw_document(document) + + assert result["created"] == 1 + job = JobPosting.objects.get() + assert job.employer_name == "Deliverect" + assert job.direct_employer is True + evidence = job.source_aliases.get().field_evidence.get(field_name="employer_name") + assert evidence.extraction_method == "reviewed-source-config" + + @pytest.mark.django_db def test_pipeline_resolves_recruiter_alias_to_direct_employer(): source = Source.objects.create( diff --git a/tests/integration/test_profile_preferences.py b/tests/integration/test_profile_preferences.py new file mode 100644 index 0000000..27b3877 --- /dev/null +++ b/tests/integration/test_profile_preferences.py @@ -0,0 +1,106 @@ +from decimal import Decimal + +import pytest +from django.urls import reverse + +from apps.jobs.models import GeocodeLocationLookup +from apps.profiles.forms import SearchProfileForm + + +def _profile_post_data(**overrides): + data = { + "name": "Testprofiel", + "is_active": "on", + "home_postal_code": "3500", + "max_distance_km": "45", + "desired_titles": ["infrastructure engineer", "cloud engineer"], + "excluded_titles": ["sales"], + "desired_skills": ["azure", "terraform"], + "excluded_skills": ["cold calling"], + "allowed_employment_types": ["full_time", "permanent"], + "preferred_workplace": ["hybrid"], + "preferred_regions": ["Limburg"], + "excluded_regions": ["West-Vlaanderen"], + "recommendation_threshold": "65", + "top_match_threshold": "90", + "digest_time": "07:30", + } + data.update(overrides) + return data + + +@pytest.mark.django_db +def test_profile_form_uses_guided_choices_and_preserves_existing_custom_values(profile): + form = SearchProfileForm(instance=profile) + + assert form.fields["home_postal_code"].widget.attrs["maxlength"] == 4 + assert "home_municipality" not in form.fields + assert "home_latitude" not in form.fields + assert "home_longitude" not in form.fields + assert form.fields["desired_titles"].widget.allow_multiple_selected is True + assert ("systeembeheerder", "systeembeheerder") in form.fields["desired_titles"].choices + assert ("VMware", "VMware") in form.fields["desired_skills"].choices + + +@pytest.mark.django_db +def test_profile_form_rejects_non_belgian_postcode_and_conflicting_choices(profile): + invalid_postcode = SearchProfileForm( + data=_profile_post_data(home_postal_code="3500 Hasselt"), instance=profile + ) + assert invalid_postcode.is_valid() is False + assert "home_postal_code" in invalid_postcode.errors + + conflicting = SearchProfileForm( + data=_profile_post_data(preferred_regions=["Limburg"], excluded_regions=["Limburg"]), + instance=profile, + ) + assert conflicting.is_valid() is False + assert "excluded_regions" in conflicting.errors + + +@pytest.mark.django_db +def test_profile_update_derives_home_location_from_postcode(client, user, profile): + GeocodeLocationLookup.objects.create( + source_name="test-geodata", + source_version="2026-01", + query_kind=GeocodeLocationLookup.QueryKind.POSTAL, + query_value="3500", + postal_code="3500", + municipality="Hasselt", + region="Limburg", + latitude=Decimal("50.930700"), + longitude=Decimal("5.332500"), + confidence=Decimal("1.00"), + ) + client.force_login(user) + + response = client.post( + reverse("profiles:edit", kwargs={"pk": profile.pk}), _profile_post_data() + ) + + assert response.status_code == 302 + profile.refresh_from_db() + assert profile.home_postal_code == "3500" + assert profile.home_municipality == "Hasselt" + assert profile.home_latitude == Decimal("50.930700") + assert profile.desired_titles == ["infrastructure engineer", "cloud engineer"] + assert profile.preferred_workplace == ["hybrid"] + + +@pytest.mark.django_db +def test_profile_update_clears_stale_coordinates_when_postcode_is_unknown(client, user, profile): + client.force_login(user) + + response = client.post( + reverse("profiles:edit", kwargs={"pk": profile.pk}), + _profile_post_data(home_postal_code="9999"), + follow=True, + ) + + assert response.status_code == 200 + assert "De postcode is bewaard" in response.content.decode("utf-8") + profile.refresh_from_db() + assert profile.home_postal_code == "9999" + assert profile.home_municipality == "" + assert profile.home_latitude is None + assert profile.home_longitude is None diff --git a/tests/integration/test_seed_sources.py b/tests/integration/test_seed_sources.py new file mode 100644 index 0000000..06645da --- /dev/null +++ b/tests/integration/test_seed_sources.py @@ -0,0 +1,91 @@ +from datetime import timedelta + +import pytest +import yaml +from django.core.management import call_command +from django.utils import timezone + +from apps.sources.models import Source, SourcePolicyReview +from apps.sources.views import _source_list_queryset + + +@pytest.mark.django_db +def test_seed_sources_registers_reviewed_live_feed_and_hides_demo(tmp_path): + Source.objects.create( + name="Voorbeeldwerkgever (fixture)", + source_type=Source.Type.EMPLOYER, + base_url="https://jobs.example.org/vacatures/", + domain="jobs.example.org", + ) + expires_at = (timezone.now() + timedelta(days=30)).isoformat() + retired_source = Source.objects.create( + name="Brusselse werkgever", + source_type=Source.Type.ATS, + base_url="https://brussels-employer.example/jobs", + domain="brussels-employer.example", + status=Source.Status.ACTIVE, + policy=Source.Policy.ALLOW, + next_run_at=timezone.now(), + ) + payload = { + "disable_demo_sources": True, + "retire_source_urls": [retired_source.base_url], + "sources": [ + { + "name": "VITO", + "url": "https://vito.recruitee.com/api/offers/", + "type": "ats", + "status": "trial", + "policy": "review", + "parser": "ats-recruitee", + "allow_public_endpoint": True, + "metadata": {"country_scope": "BE"}, + "review": { + "decision": "trial", + "reason": "Publieke feed", + "evidence_link": "https://docs.recruitee.com/reference/offers", + "expires_at": expires_at, + }, + }, + { + "name": "Lever werkgever A", + "url": "https://api.lever.co/v0/postings/werkgever-a?mode=json", + "type": "ats", + "status": "trial", + "policy": "allow", + "parser": "ats-lever", + "allow_public_endpoint": True, + }, + { + "name": "Lever werkgever B", + "url": "https://api.lever.co/v0/postings/werkgever-b?mode=json", + "type": "ats", + "status": "trial", + "policy": "allow", + "parser": "ats-lever", + "allow_public_endpoint": True, + }, + ], + } + seed_path = tmp_path / "sources.yaml" + seed_path.write_text(yaml.safe_dump(payload), encoding="utf-8") + + call_command("seed_sources", str(seed_path)) + call_command("seed_sources", str(seed_path)) + + demo = Source.objects.get(domain="jobs.example.org") + assert demo.status == Source.Status.DISABLED + assert demo.metadata["demo"] is True + retired_source.refresh_from_db() + assert retired_source.status == Source.Status.DISABLED + assert retired_source.next_run_at is None + assert retired_source.metadata["hidden_from_source_list"] is True + assert retired_source.metadata["retired_reason"] == "Buiten de ingestelde regionale dekking" + source = Source.objects.get(name="VITO") + assert retired_source not in _source_list_queryset() + assert source in _source_list_queryset() + assert source.allow_public_endpoint is True + assert source.parser_key == "ats-recruitee" + assert source.latest_policy_review.decision == SourcePolicyReview.Decision.TRIAL + assert source.policy_reviews.count() == 1 + assert Source.objects.filter(domain="api.lever.co", source_type=Source.Type.ATS).count() == 2 diff --git a/tests/integration/test_tasks_and_more_views.py b/tests/integration/test_tasks_and_more_views.py index 0d232ad..a5a007c 100644 --- a/tests/integration/test_tasks_and_more_views.py +++ b/tests/integration/test_tasks_and_more_views.py @@ -177,7 +177,20 @@ def test_job_profile_source_and_application_views(client, user, profile, job, so @pytest.mark.integration @pytest.mark.django_db -def test_sources_candidate_bulk_actions_and_evidence_view(client, user): +def test_sources_candidate_bulk_actions_and_evidence_view(client, user, profile): + profile.home_postal_code = "2400" + profile.max_distance_km = 40 + profile.home_latitude = None + profile.home_longitude = None + profile.save( + update_fields=[ + "home_postal_code", + "max_distance_km", + "home_latitude", + "home_longitude", + "updated_at", + ] + ) candidate = Source.objects.create( name="Discoverd candidate", source_type=Source.Type.EMPLOYER, @@ -206,12 +219,37 @@ def test_sources_candidate_bulk_actions_and_evidence_view(client, user): status=Source.Status.CANDIDATE, policy=Source.Policy.REVIEW, ) + watchlist = Source.objects.create( + name="Interessante werkgever zonder stabiele feed", + source_type=Source.Type.EMPLOYER, + base_url="https://watch.example.org/jobs/", + domain="watch.example.org", + status=Source.Status.PAUSED, + policy=Source.Policy.REVIEW, + metadata={ + "watchlist": True, + "municipality_scope": "Olen", + "watchlist_reason": "Regionaal interessante werkgever", + "current_observation": "Momenteel geen lokale vacature bevestigd.", + "public_job_page": "https://watch.example.org/jobs/", + }, + ) client.force_login(user) response = client.get(reverse("sources:list")) assert response.status_code == 200 content = response.content.decode("utf-8") assert "jobs.example.org/careers" in content + assert "Platformmailboxen" in content + assert "Mailbox koppelen en testen" in content + assert "elke vijf minuten" in content + assert "2400 Mol" in content + assert "maximaal 40 km" in content + assert "Coördinaten ontbreken nog" in content + assert "Interessante werkgeverswaaklijst" in content + assert watchlist.name in content + assert response.context["watchlist_count"] == 1 + assert {item.pk for item in response.context["sources"]} == {candidate.pk, other.pk} response = client.post( reverse("sources:bulk"), @@ -229,3 +267,12 @@ def test_sources_candidate_bulk_actions_and_evidence_view(client, user): other.refresh_from_db() assert other.status == Source.Status.DISABLED assert other.policy == Source.Policy.DENY + + response = client.post( + reverse("sources:bulk"), + {"action": "dismiss", "source_ids": [str(watchlist.pk)]}, + ) + assert response.status_code == 302 + watchlist.refresh_from_db() + assert watchlist.status == Source.Status.PAUSED + assert watchlist.policy == Source.Policy.REVIEW diff --git a/tests/security/test_policy.py b/tests/security/test_policy.py index 9e621a7..d828e1b 100644 --- a/tests/security/test_policy.py +++ b/tests/security/test_policy.py @@ -9,10 +9,20 @@ from apps.sources.services.robots import RobotsDecision def test_platform_domains_are_denied_including_subdomains(): - assert is_denied_domain("www.linkedin.com") - assert is_denied_domain("be.indeed.com") + for domain in ( + "www.linkedin.com", + "be.indeed.com", + "www.ictjob.be", + "www.jobat.be", + "www.stepstone.be", + "www.careerjet.be", + "jobs.randstad.be", + "www.roberthalf.com", + ): + assert is_denied_domain(domain) assert not is_denied_domain("jobs.example.org") assert not assess_url("https://www.linkedin.com/jobs/view/123").allowed + assert not assess_url("https://www.ictjob.be/nl/it-vacatures-zoeken/example").allowed @pytest.mark.security diff --git a/tests/security/test_robots.py b/tests/security/test_robots.py index c8f9cc2..91b07d9 100644 --- a/tests/security/test_robots.py +++ b/tests/security/test_robots.py @@ -92,3 +92,61 @@ def test_robots_cache_refreshes_after_ttl(db, settings, monkeypatch): assert calls["count"] == 2 assert SourceRobotsCache.objects.filter(origin__contains="jobs.example.org").exists() client.close() + + +@pytest.mark.security +def test_robots_follows_and_revalidates_redirects(db, monkeypatch): + validated_urls = [] + + def validate(url, **kwargs): + validated_urls.append(url) + if "127.0.0.1" in url: + from apps.sources.services.url_security import UnsafeUrlError + + raise UnsafeUrlError("Privéadres") + return ValidatedUrl( + url=url, + hostname="jobs.example.org", + port=443, + addresses=("93.184.216.34",), + ) + + monkeypatch.setattr("apps.sources.services.robots.validate_public_url", validate) + source = Source.objects.create( + name="Redirect jobs", + source_type=Source.Type.ATS, + base_url="https://ats.example.org/jobs", + domain="ats.example.org", + status=Source.Status.ACTIVE, + policy=Source.Policy.ALLOW, + ) + + def safe_handler(request: httpx.Request) -> httpx.Response: + if request.url.host == "ats.example.org": + return httpx.Response( + 302, + headers={"location": "https://jobs.example.org/robots.txt"}, + request=request, + ) + return httpx.Response(200, text="User-agent: *\nAllow: /", request=request) + + client = httpx.Client(transport=httpx.MockTransport(safe_handler)) + assert assess_robots(source.base_url, source=source, client=client).allowed is True + client.close() + assert validated_urls == [ + "https://ats.example.org/robots.txt", + "https://jobs.example.org/robots.txt", + ] + + SourceRobotsCache.objects.all().delete() + + def unsafe_handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 302, headers={"location": "http://127.0.0.1/robots.txt"}, request=request + ) + + client = httpx.Client(transport=httpx.MockTransport(unsafe_handler)) + blocked = assess_robots(source.base_url, source=source, client=client) + client.close() + assert blocked.allowed is False + assert "Privéadres" in blocked.reason diff --git a/tests/unit/test_ats_adapters.py b/tests/unit/test_ats_adapters.py index a5d4b8d..10ad9d1 100644 --- a/tests/unit/test_ats_adapters.py +++ b/tests/unit/test_ats_adapters.py @@ -89,6 +89,22 @@ def test_lever_adapter_detail_supports_changed_markup(): assert job.location_text == "Aalst" +def test_lever_adapter_supports_documented_public_postings_api_shape(): + adapter = LeverAdapter() + result = adapter.extract( + _load_fixture("fixtures/ats/lever-public-postings.json"), + url="https://api.lever.co/v0/postings/deliverect?mode=json", + ) + + assert len(result.jobs) == 1 + job = result.jobs[0] + assert job.title == "Backend Engineer" + assert job.url == "https://jobs.lever.co/deliverect/deliverect-open-1" + assert job.location_text == "Ghent, Flemish Region" + assert job.employment_types == ["Full-time Permanent"] + assert job.workplace_type == "hybrid" + + def test_recruitee_adapter_listing_extracts_open_jobs_and_skips_closed(): adapter = RecruiteeAdapter() result = adapter.extract( @@ -114,6 +130,23 @@ def test_recruitee_adapter_detail_supports_changed_markup(): assert job.employment_types == ["permanent"] +def test_recruitee_adapter_supports_documented_public_offers_shape(): + adapter = RecruiteeAdapter() + result = adapter.extract( + _load_fixture("fixtures/ats/recruitee-public-offers.json"), + url="https://vito.recruitee.com/api/offers/", + ) + + assert len(result.jobs) == 1 + job = result.jobs[0] + assert job.title == "Research Software Engineer" + assert job.employer_name == "VITO" + assert job.url == "https://jobs.vito.be/o/research-software-engineer" + assert job.postal_code == "2400" + assert job.employment_types == ["fulltime_fixed_term"] + assert job.workplace_type == "hybrid" + + def test_smartrecruiters_adapter_listing_extracts_open_jobs_and_skips_closed(): adapter = SmartRecruitersAdapter() result = adapter.extract( @@ -139,6 +172,24 @@ def test_smartrecruiters_adapter_detail_supports_changed_markup(): assert job.workplace_type in {"remote", "hybrid", ""} +def test_smartrecruiters_adapter_supports_documented_public_postings_shape(): + adapter = SmartRecruitersAdapter() + result = adapter.extract( + _load_fixture("fixtures/ats/smartrecruiters-public-postings.json"), + url=("https://api.smartrecruiters.com/v1/companies/ARHS/postings?limit=100&country=be"), + ) + + assert len(result.jobs) == 2 + job = result.jobs[1] + assert job.title == "Cloud Engineer" + assert job.employer_name == "ARHS" + assert job.url == "https://jobs.smartrecruiters.com/ARHS/744000136173889" + assert job.location_text == "Leuven" + assert job.country == "be" + assert job.employment_types == ["Permanent"] + assert job.workplace_type == "remote" + + def test_workable_adapter_listing_extracts_open_jobs_and_skips_closed(): adapter = WorkableAdapter() result = adapter.extract( diff --git a/tests/unit/test_deployment_configuration.py b/tests/unit/test_deployment_configuration.py index 3e6afec..d11e2c8 100644 --- a/tests/unit/test_deployment_configuration.py +++ b/tests/unit/test_deployment_configuration.py @@ -24,3 +24,18 @@ def test_runtime_is_compatible_with_read_only_container() -> None: assert '"--no-control-socket"' in dockerfile assert "exec -T web python manage.py collectstatic" not in deploy_script assert "for required_host in 127.0.0.1 localhost" in deploy_script + + +def test_deployment_examples_use_per_platform_mailbox_configuration() -> None: + required_keys = { + "MAILBOX_CREDENTIAL_KEYS", + "IMAP_CONNECT_TIMEOUT_SECONDS", + "IMAP_MAX_MESSAGES_PER_POLL", + "IMAP_MAX_MESSAGE_BYTES", + } + for filename in (".env.example", "scripts/codex_bootstrap.sh", "scripts/release_verify.sh"): + content = (ROOT / filename).read_text(encoding="utf-8") + + assert "IMAP_ENABLED" not in content + for key in required_keys: + assert f"{key}=" in content diff --git a/tests/unit/test_email_adapter.py b/tests/unit/test_email_adapter.py index ddc5c7f..b706397 100644 --- a/tests/unit/test_email_adapter.py +++ b/tests/unit/test_email_adapter.py @@ -1,6 +1,10 @@ from pathlib import Path +import pytest + from apps.sources.adapters.email_alert import EmailAlertAdapter +from apps.sources.models import MailboxConnection +from apps.sources.platforms import PLATFORM_ALERTS def test_email_adapter_extracts_job_links_and_skips_unsubscribe(): @@ -10,3 +14,43 @@ def test_email_adapter_extracts_job_links_and_skips_unsubscribe(): assert "https://jobs.example.org/vacatures/infrastructure-engineer" in urls assert "https://www.linkedin.com/jobs/view/123456" in urls assert all("unsubscribe" not in url for url in urls) + + +def test_email_adapter_recognizes_sanitized_vdab_alert(): + raw = Path("fixtures/emails/vdab_job_alert.eml").read_bytes() + result = EmailAlertAdapter().extract_message(raw) + + assert {job.title for job in result.jobs} == {"Python developer", "Cloud engineer"} + assert all(job.raw["alert_provider"] == "vdab" for job in result.jobs) + assert all(job.raw["target_domain"] == "www.vdab.be" for job in result.jobs) + assert all("uitschrijven" not in job.url for job in result.jobs) + + +@pytest.mark.parametrize( + ("provider", "fixture", "title", "domain"), + ( + ("linkedin", "linkedin_job_alert.eml", "Software Engineer bij Example", "linkedin.com"), + ("ictjob", "ictjob_job_alert.eml", "Cloud Engineer", "ictjob.be"), + ("jobat", "jobat_job_alert.eml", "Data Engineer", "jobat.be"), + ("stepstone", "stepstone_job_alert.eml", "Software Developer", "stepstone.be"), + ("careerjet", "careerjet_job_alert.eml", "System Engineer", "careerjet.be"), + ("randstad", "randstad_job_alert.eml", "Infrastructure Engineer", "randstad.be"), + ("roberthalf", "roberthalf_job_alert.eml", "Network Engineer", "roberthalf.com"), + ), +) +def test_platform_bound_alert_extracts_only_expected_https_jobboard_link( + provider, fixture, title, domain +): + raw = (Path("fixtures/emails") / fixture).read_bytes() + + result = EmailAlertAdapter().extract_message(raw, expected_provider=provider) + + assert [job.title for job in result.jobs] == [title] + assert result.jobs[0].raw["alert_provider"] == provider + assert result.jobs[0].raw["target_domain"].endswith(domain) + assert result.jobs[0].url.startswith("https://") + assert "outside.example.invalid" not in result.jobs[0].url + + +def test_mailbox_platform_choices_and_alert_catalog_stay_in_sync(): + assert set(MailboxConnection.Platform.values) == set(PLATFORM_ALERTS) diff --git a/tests/unit/test_mailbox_credentials.py b/tests/unit/test_mailbox_credentials.py new file mode 100644 index 0000000..c6de5fe --- /dev/null +++ b/tests/unit/test_mailbox_credentials.py @@ -0,0 +1,110 @@ +import pytest +from cryptography.fernet import Fernet +from django.core.exceptions import ValidationError +from django.test import override_settings + +from apps.sources.models import MailboxConnection +from apps.sources.services.imap_import import poll_imap_mailbox +from apps.sources.services.mailbox_connections import ( + MailboxCredentialError, + decrypt_mailbox_password, + encrypt_mailbox_password, + rotate_mailbox_credentials, +) +from apps.sources.services.url_security import UnsafeUrlError + + +def test_mailbox_password_is_encrypted_and_supports_key_rotation(): + old_key = Fernet.generate_key().decode("ascii") + new_key = Fernet.generate_key().decode("ascii") + with override_settings(MAILBOX_CREDENTIAL_KEYS=[old_key]): + ciphertext = encrypt_mailbox_password("app-password-value") + + connection = MailboxConnection(encrypted_password=ciphertext) + assert "app-password-value" not in ciphertext + with override_settings(MAILBOX_CREDENTIAL_KEYS=[new_key, old_key]): + assert decrypt_mailbox_password(connection) == "app-password-value" + rotated = encrypt_mailbox_password("replacement-value") + assert Fernet(new_key.encode("ascii")).decrypt(rotated.encode("ascii")) == b"replacement-value" + + +def test_mailbox_password_fails_closed_without_valid_key(): + with ( + override_settings(MAILBOX_CREDENTIAL_KEYS=[]), + pytest.raises(MailboxCredentialError, match="niet geconfigureerd"), + ): + encrypt_mailbox_password("never-stored") + with ( + override_settings(MAILBOX_CREDENTIAL_KEYS=["invalid"]), + pytest.raises(MailboxCredentialError, match="ongeldige sleutel"), + ): + encrypt_mailbox_password("never-stored") + + +@pytest.mark.parametrize("host", ["", "https://imap.example.org", "imap.example.org/path", "a b"]) +def test_custom_mailbox_host_rejects_non_hostname_values(host): + connection = MailboxConnection( + platform=MailboxConnection.Platform.VDAB, + provider=MailboxConnection.Provider.CUSTOM, + custom_host=host, + port=993, + username="mailbox@example.invalid", + encrypted_password="ciphertext", + ) + with pytest.raises(ValidationError): + connection.clean() + + +@pytest.mark.parametrize("port", [0, 65536]) +def test_mailbox_rejects_invalid_port(port): + connection = MailboxConnection( + platform=MailboxConnection.Platform.VDAB, + provider=MailboxConnection.Provider.GMAIL, + port=port, + username="mailbox@example.invalid", + encrypted_password="ciphertext", + ) + with pytest.raises(ValidationError): + connection.clean() + + +@pytest.mark.django_db +def test_rotation_reencrypts_existing_credentials_with_primary_key(user): + old_key = Fernet.generate_key().decode("ascii") + new_key = Fernet.generate_key().decode("ascii") + with override_settings(MAILBOX_CREDENTIAL_KEYS=[old_key]): + ciphertext = encrypt_mailbox_password("rotate-me") + connection = MailboxConnection.objects.create( + user=user, + platform=MailboxConnection.Platform.VDAB, + provider=MailboxConnection.Provider.GMAIL, + username="rotate@example.invalid", + encrypted_password=ciphertext, + ) + + with override_settings(MAILBOX_CREDENTIAL_KEYS=[new_key, old_key]): + assert rotate_mailbox_credentials() == 1 + connection.refresh_from_db() + with override_settings(MAILBOX_CREDENTIAL_KEYS=[new_key]): + assert decrypt_mailbox_password(connection) == "rotate-me" + + +@pytest.mark.django_db +def test_private_imap_host_is_blocked_before_connection(user): + key = Fernet.generate_key().decode("ascii") + with override_settings(MAILBOX_CREDENTIAL_KEYS=[key]): + connection = MailboxConnection.objects.create( + user=user, + platform=MailboxConnection.Platform.VDAB, + provider=MailboxConnection.Provider.CUSTOM, + custom_host="127.0.0.1", + username="private@example.invalid", + encrypted_password=encrypt_mailbox_password("never-sent"), + ) + with pytest.raises(UnsafeUrlError, match="Niet-publiek"): + poll_imap_mailbox( + connection, + client_factory=lambda *args, **kwargs: pytest.fail( + "Private IMAP-host had niet geconnecteerd mogen worden" + ), + ) diff --git a/tests/unit/test_regional_adapters.py b/tests/unit/test_regional_adapters.py new file mode 100644 index 0000000..f6b8476 --- /dev/null +++ b/tests/unit/test_regional_adapters.py @@ -0,0 +1,253 @@ +from pathlib import Path + +import pytest + +from apps.sources.adapters.kempen import KempenEmployerAdapter +from apps.sources.adapters.mol_region import MolRegionEmployerAdapter +from apps.sources.adapters.regional import CordaCampusAdapter, LocalEmployerListingAdapter + + +def test_corda_adapter_extracts_local_employer_cards_and_rejects_off_domain_links(): + content = Path("fixtures/pages/corda-campus-jobs.html").read_text(encoding="utf-8") + + result = CordaCampusAdapter().extract( + content, + url="https://www.cordacampus.com/jobs/", + ) + + assert result.parser_key == "regional-corda-campus" + assert len(result.jobs) == 2 + assert {job.title for job in result.jobs} == {"System Engineer", "Data Engineer"} + assert {job.employer_name for job in result.jobs} == { + "Voorbeeld IT", + "Voorbeeld Data", + } + assert all(job.region == "Limburg" for job in result.jobs) + assert all("evil.example" not in job.url for job in result.jobs) + + +def test_corda_adapter_does_not_capture_individual_detail_pages(): + result = CordaCampusAdapter().extract( + "

System Engineer

", + url="https://www.cordacampus.com/job/system-engineer/", + ) + + assert result.jobs == [] + assert result.confidence == 0.0 + + +@pytest.mark.parametrize( + ("fixture_name", "url", "expected_title", "expected_employer"), + [ + ( + "aca-group-jobs.html", + "https://acagroup.be/en/jobs", + "AWS Cloud & DevOps Engineer", + "ACA Group", + ), + ( + "xplore-group-jobs.html", + "https://www.xploregroup.be/en/jobs", + "Platform Engineer", + "Xplore Group", + ), + ( + "uhasselt-jobs.html", + "https://www.uhasselt.be/vacatures", + "Expert systeembeheerder (100%)", + "Universiteit Hasselt", + ), + ( + "pxl-jobs.html", + "https://ses.pxl.be/", + "Lector netwerkbeheer", + "Hogeschool PXL", + ), + ( + "zol-jobs.html", + "https://ziekenhuis-oost-limburg.cvw.io/?lang=nl-BE", + "(Senior) IT Engineer Infrastructuur", + "Ziekenhuis Oost-Limburg", + ), + ], +) +def test_local_employer_adapter_extracts_reviewed_routes_only( + fixture_name, url, expected_title, expected_employer +): + content = Path(f"fixtures/pages/{fixture_name}").read_text(encoding="utf-8") + + result = LocalEmployerListingAdapter().extract(content, url=url) + + assert result.parser_key == "regional-local-employers" + assert len(result.jobs) == 1 + assert result.jobs[0].title == expected_title + assert result.jobs[0].employer_name == expected_employer + assert result.jobs[0].region == "Limburg" + assert "evil.example" not in result.jobs[0].url + + +def test_local_employer_adapter_fails_closed_for_unreviewed_host_or_path(): + adapter = LocalEmployerListingAdapter() + + assert adapter.extract("

Fake

", url="https://evil.example/jobs").jobs == [] + assert ( + adapter.extract( + "

Detail

", + url="https://www.uhasselt.be/vacatures/detail/3999", + ).jobs + == [] + ) + + +@pytest.mark.parametrize( + ("fixture_name", "url", "expected_title", "expected_employer"), + [ + ( + "sck-cen-jobs.html", + "https://www.sckcen.be/nl/carriere/vacatures", + "Automatisatie Ingenieur PLC & OT-netwerken", + "SCK CEN", + ), + ( + "cipal-schaubroeck-jobs.html", + "https://cipalschaubroeck.teamtailor.com/jobs", + "Network & Security Engineer", + "Cipal Schaubroeck", + ), + ( + "vanroey-jobs.html", + "https://www.vanroey.be/en/job-overview/", + "Network Engineer", + "VanRoey", + ), + ( + "netropolix-jobs.html", + "https://netropolix.recruitee.com/", + "System Engineer - Geel", + "NTX (Netropolix)", + ), + ( + "thomas-more-jobs.html", + "https://jobpage.cvwarehouse.com/?companyGuid=eab9ca13-ee10-4504-8a87-a785d0b037ef&lang=nl-BE", + "Een medewerker lokale IT - Campus Geel", + "Thomas More", + ), + ], +) +def test_mol_region_adapter_extracts_only_reviewed_local_cards( + fixture_name, url, expected_title, expected_employer +): + content = Path(f"fixtures/pages/{fixture_name}").read_text(encoding="utf-8") + + result = MolRegionEmployerAdapter().extract(content, url=url) + + assert result.parser_key == "regional-mol-employers" + assert result.warnings == [] + assert len(result.jobs) == 1 + assert result.jobs[0].title == expected_title + assert result.jobs[0].employer_name == expected_employer + assert result.jobs[0].region == "Antwerpen" + assert "evil.example" not in result.jobs[0].url + + +def test_mol_region_adapter_fails_closed_for_unreviewed_route_and_company(): + adapter = MolRegionEmployerAdapter() + + assert adapter.extract("

Fake

", url="https://evil.example/jobs").jobs == [] + assert ( + adapter.extract( + "

Detail

", + url="https://www.sckcen.be/nl/carriere/vacatures/detail", + ).jobs + == [] + ) + + +@pytest.mark.parametrize( + ("fixture_name", "url", "expected_title", "expected_employer", "expected_postal"), + [ + ( + "ziekenhuis-geel-jobs.html", + "https://ziekenhuisgeel.careersite.be/nl/vacatures", + "ICT Support Engineer", + "Ziekenhuis Geel", + "2440", + ), + ( + "stad-geel-jobs.html", + "https://geel.hro.be/", + "Administratief medewerker digitale dienstverlening", + "Lokaal bestuur Geel", + "2440", + ), + ( + "stad-turnhout-jobs.html", + "https://jobs.turnhout.be/", + "Coördinator digitale projecten", + "Stad Turnhout", + "2300", + ), + ( + "renotec-geel-jobs.html", + "https://jobs.renotec.be/nl/alle-jobs/", + "Technisch commercieel verantwoordelijke", + "Group Renotec", + "2440", + ), + ( + "ravago-kempen-jobs.html", + "https://ravago.softgarden.io/en/vacancies", + "SAP Operations Analyst", + "Ravago", + "2370", + ), + ( + "sanofi-geel-jobs.html", + "https://jobs.sanofi.com/en/belgium", + "Maintenance Engineer", + "Sanofi", + "2440", + ), + ( + "daf-westerlo-jobs.html", + "https://www.daf.com/nl-nl/werken-bij-daf/vacatures", + "Technieker Facilitaire Diensten", + "DAF Trucks", + "2260", + ), + ], +) +def test_kempen_adapter_extracts_only_reviewed_local_cards( + fixture_name, url, expected_title, expected_employer, expected_postal +): + content = Path(f"fixtures/pages/{fixture_name}").read_text(encoding="utf-8") + + result = KempenEmployerAdapter().extract(content, url=url) + + assert result.parser_key == "regional-kempen-employers" + assert result.warnings == [] + assert len(result.jobs) == 1 + assert result.jobs[0].title == expected_title + assert result.jobs[0].employer_name == expected_employer + assert result.jobs[0].postal_code == expected_postal + assert "evil.example" not in result.jobs[0].url + + +def test_kempen_adapter_fails_closed_for_unreviewed_route(): + adapter = KempenEmployerAdapter() + + assert adapter.extract("

Fake

", url="https://evil.example/jobs").jobs == [] + assert ( + adapter.extract( + "

Detail

", + url="https://jobs.sanofi.com/en/search-jobs", + ).jobs + == [] + ) + assert ( + adapter.extract( + "

Andere werkgever

", + url="https://jobpage.cvwarehouse.com/?companyGuid=other&lang=nl-BE", + ).jobs + == [] + ) diff --git a/tests/unit/test_settings_security.py b/tests/unit/test_settings_security.py index 6726bd4..c1c9681 100644 --- a/tests/unit/test_settings_security.py +++ b/tests/unit/test_settings_security.py @@ -3,7 +3,20 @@ from __future__ import annotations import pytest from django.core.exceptions import ImproperlyConfigured -from config.settings import _validate_production_security +from config.settings import _validate_production_security, env_int_range + + +def test_env_int_range_accepts_default_and_rejects_invalid_values(monkeypatch): + monkeypatch.delenv("TEST_BOUNDED_SETTING", raising=False) + assert env_int_range("TEST_BOUNDED_SETTING", 6, minimum=0, maximum=23) == 6 + + monkeypatch.setenv("TEST_BOUNDED_SETTING", "24") + with pytest.raises(ImproperlyConfigured): + env_int_range("TEST_BOUNDED_SETTING", 6, minimum=0, maximum=23) + + monkeypatch.setenv("TEST_BOUNDED_SETTING", "morgen") + with pytest.raises(ImproperlyConfigured): + env_int_range("TEST_BOUNDED_SETTING", 6, minimum=0, maximum=23) def test_validate_production_security_skips_when_debug_mode_is_enabled(): diff --git a/tests/unit/test_source_models.py b/tests/unit/test_source_models.py index a520fcc..ac09a0e 100644 --- a/tests/unit/test_source_models.py +++ b/tests/unit/test_source_models.py @@ -2,6 +2,7 @@ from datetime import timedelta import pytest from django.core.exceptions import ValidationError +from django.db import IntegrityError from django.utils import timezone from apps.sources.models import Source, SourceRun @@ -40,3 +41,27 @@ def test_source_run_finish_updates_metrics(source): assert run.finished_at is not None assert run.http_status == 200 assert run.extracted_count == 3 + + +@pytest.mark.django_db(transaction=True) +def test_shared_ats_domain_accepts_distinct_public_feeds_but_not_duplicate_urls(): + Source.objects.create( + name="Lever A", + source_type=Source.Type.ATS, + base_url="https://api.lever.co/v0/postings/a?mode=json", + domain="api.lever.co", + ) + Source.objects.create( + name="Lever B", + source_type=Source.Type.ATS, + base_url="https://api.lever.co/v0/postings/b?mode=json", + domain="api.lever.co", + ) + + with pytest.raises(IntegrityError): + Source.objects.create( + name="Lever A dubbel", + source_type=Source.Type.ATS, + base_url="https://api.lever.co/v0/postings/a?mode=json", + domain="api.lever.co", + ) diff --git a/tests/unit/test_tls.py b/tests/unit/test_tls.py new file mode 100644 index 0000000..bfbc914 --- /dev/null +++ b/tests/unit/test_tls.py @@ -0,0 +1,15 @@ +import ssl +from pathlib import Path + +import certifi + +from apps.sources.services.tls import trusted_tls_context + + +def test_application_tls_context_uses_strict_pinned_ca_bundle(): + context = trusted_tls_context() + + assert Path(certifi.where()).is_file() + assert context.verify_mode == ssl.CERT_REQUIRED + assert context.check_hostname is True + assert context.cert_store_stats()["x509_ca"] > 0 diff --git a/uv.lock b/uv.lock index 1a2622c..c60ade9 100644 --- a/uv.lock +++ b/uv.lock @@ -104,6 +104,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" }, ] +[[package]] +name = "cffi" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/85/990925db5df586ec90beb97529c853497e7f85ba0234830447faf41c3057/cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f", size = 184829, upload-time = "2026-07-06T21:32:44.324Z" }, + { url = "https://files.pythonhosted.org/packages/4b/92/e7bb136ad6b5352603732cf907ef862ca103f20f2031c1735a46300c20c9/cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde", size = 184728, upload-time = "2026-07-06T21:32:45.683Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" }, + { url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" }, + { url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" }, + { url = "https://files.pythonhosted.org/packages/58/0c/f528df19cc94b675087324d4760d9e6d5bfae97d6217aa4fac43de4f5fcc/cffi-2.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7", size = 208859, upload-time = "2026-07-06T21:32:52.512Z" }, + { url = "https://files.pythonhosted.org/packages/62/f2/c9522a81c32132799a1972c39f5c5f8b4c8b9f00488a23feaa6c06f07741/cffi-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66", size = 221844, upload-time = "2026-07-06T21:32:53.704Z" }, + { url = "https://files.pythonhosted.org/packages/6e/28/bd53988b9833e8f8ad539d26f4c07a6b3f6bcb1e9e02e7ca038250b3428d/cffi-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe", size = 225287, upload-time = "2026-07-06T21:32:54.907Z" }, + { url = "https://files.pythonhosted.org/packages/79/99/0d0fd37f055224085f42bbb2c022d002e17dde4a97972822327b07d84101/cffi-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b", size = 223681, upload-time = "2026-07-06T21:32:56.329Z" }, + { url = "https://files.pythonhosted.org/packages/b0/80/c138990aa2a70b1a269f6e06348729836d733d6f970867943f61d367f8cc/cffi-2.1.0-cp312-cp312-win32.whl", hash = "sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a", size = 175269, upload-time = "2026-07-06T21:32:57.777Z" }, + { url = "https://files.pythonhosted.org/packages/a8/eb/f636456ff21a83fc13c032b58cc5dde061691546ac79efa284b2989b7982/cffi-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384", size = 185881, upload-time = "2026-07-06T21:32:59.253Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/400ea43e721727dca8a65c4521390e9196757caba4a45643acb2b63271b8/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6", size = 180088, upload-time = "2026-07-06T21:33:02.278Z" }, + { url = "https://files.pythonhosted.org/packages/96/88/a996879e2eeccb815f6e3a5967b12a308257412acec882039d386bd2aa7b/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda", size = 194331, upload-time = "2026-07-06T21:33:03.697Z" }, + { url = "https://files.pythonhosted.org/packages/58/85/7ae00d5c8dd6266f4e944c3db630f3c5c9a98b61d469c714d848b1d8138a/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b", size = 196966, upload-time = "2026-07-06T21:33:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e9/45c3a76ad8d43ad9261f4c95436da61128d3ca545d72b9612c0ab5be0b1c/cffi-2.1.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a", size = 184795, upload-time = "2026-07-06T21:33:06.699Z" }, + { url = "https://files.pythonhosted.org/packages/84/4c/82f132cb4418ee6d953d982b19191e87e2a6372c8a4ce36e50b69d6ade4a/cffi-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea", size = 184746, upload-time = "2026-07-06T21:33:08.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1c/4ed5a0e5bdca6cbc275556de3328dd1b76fd0c11cc13c88fe66d1d8715f2/cffi-2.1.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db", size = 214747, upload-time = "2026-07-06T21:33:09.671Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a6/e879bb68cc23a2bc9ba8f4b7d8019f0c2694bad2ab6c4a3701d429439f58/cffi-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f", size = 222392, upload-time = "2026-07-06T21:33:10.896Z" }, + { url = "https://files.pythonhosted.org/packages/88/f6/01890cfd63c08f8eb96a8319b0443690197d240a8bd6346048cf7bde9190/cffi-2.1.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d", size = 210285, upload-time = "2026-07-06T21:33:12.251Z" }, + { url = "https://files.pythonhosted.org/packages/a6/cf/2b684132056f438567b61e19d690dd31cd0921ace051e0a458be6074369e/cffi-2.1.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0", size = 208801, upload-time = "2026-07-06T21:33:13.617Z" }, + { url = "https://files.pythonhosted.org/packages/6f/08/f2e7d62c460faae0926f2d6e423694aa409ced3bc1fe2927a0a6e5f05416/cffi-2.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224", size = 221808, upload-time = "2026-07-06T21:33:15.466Z" }, + { url = "https://files.pythonhosted.org/packages/38/37/04f54b8e63a02f3d908332c9effbf8c366167c6f733ed8a3d4f79b7e2a1e/cffi-2.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c", size = 225241, upload-time = "2026-07-06T21:33:16.869Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d6/c72eecca433cd3e681c65ed313ab4835d9d4a379704d0f628a6a05f51c2e/cffi-2.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a", size = 223588, upload-time = "2026-07-06T21:33:18.239Z" }, + { url = "https://files.pythonhosted.org/packages/c6/4b/e706f67279140f92939da3475ad610df18bfd52d50f14953a8e5fede71d5/cffi-2.1.0-cp313-cp313-win32.whl", hash = "sha256:db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2", size = 175248, upload-time = "2026-07-06T21:33:19.799Z" }, + { url = "https://files.pythonhosted.org/packages/5a/47/59eb7975cb0e4ef0afa764ea945b29a5bb4537a9f771cb7d6c8a5dd74c95/cffi-2.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512", size = 185717, upload-time = "2026-07-06T21:33:21.47Z" }, + { url = "https://files.pythonhosted.org/packages/5a/af/34fee85c48f8d94efc8597bc09470c9dd274c145f1c12e0fbc6ab6d38d74/cffi-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f", size = 180114, upload-time = "2026-07-06T21:33:22.515Z" }, +] + [[package]] name = "click" version = "8.4.2" @@ -201,6 +238,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/82/32e3bd191d498e64f6f911ad55d14006a0861e54869d2d32452326399e65/coverage-7.15.2-py3-none-any.whl", hash = "sha256:eb6bcae8d1a9d305351ecb108232441d11c5cfe9de840a04388ba5d2db8d735c", size = 213375, upload-time = "2026-07-15T18:56:17.305Z" }, ] +[[package]] +name = "cryptography" +version = "49.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/22/adf66990e63584a68dfb50c24f48a125c07b1699899381c8151e63ed458c/cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db", size = 4032100, upload-time = "2026-06-12T20:02:32.143Z" }, + { url = "https://files.pythonhosted.org/packages/09/41/3797cfaf69cae04a13ee78ebd83f0678d9c02b4779d21ce24445326f1a69/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db", size = 4692978, upload-time = "2026-06-12T20:01:21.305Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8b/43011f7ebe515a8aa20d61f290a326cd890c2e738e16e59eaff8d9c3a412/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325", size = 4716422, upload-time = "2026-06-12T20:01:48.566Z" }, + { url = "https://files.pythonhosted.org/packages/4a/91/01ce7303a4579e6d3a6abef01bd322848e9ea7a219adcabc5048b9033571/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2", size = 4700503, upload-time = "2026-06-12T20:02:47.091Z" }, + { url = "https://files.pythonhosted.org/packages/62/99/a2c95cf8293f07491e9e27c20cc4dcd18176d944e674679adeb1d0173fd6/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b", size = 5309779, upload-time = "2026-06-12T20:02:08.987Z" }, + { url = "https://files.pythonhosted.org/packages/20/2c/0622f20ff02b2ef32558733443805dc82fd4c275be01b2d19d14676f3a1b/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6", size = 4749683, upload-time = "2026-06-12T20:02:03.335Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5b/c5246635d5fd3b64e0d45ae10e99fd32fe9676a79915ccfe5a61ba9af1a5/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c", size = 4337874, upload-time = "2026-06-12T20:02:54.323Z" }, + { url = "https://files.pythonhosted.org/packages/6d/88/05563c7fe2e914e87d1a536d06fe83e66b4e1d95cb593e05aea375531da8/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7", size = 4700283, upload-time = "2026-06-12T20:01:34.822Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b6/d7696e4e890d6ae1469935164c9e5215c557671cb78d6e3f458ccceaa632/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68", size = 5265844, upload-time = "2026-06-12T20:01:24.09Z" }, + { url = "https://files.pythonhosted.org/packages/a9/3c/f3ad17eecc1a57b0ba236dc01f90e783c51f4a2f35f64777cc4f47a184b2/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9", size = 4749290, upload-time = "2026-06-12T20:01:30.848Z" }, + { url = "https://files.pythonhosted.org/packages/4f/01/339573cf1023163a400b0b5d16f6d507de413b9f60be6fd1b77feeaf6737/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f", size = 4834612, upload-time = "2026-06-12T20:01:29.246Z" }, + { url = "https://files.pythonhosted.org/packages/71/fd/577302e213a1be9468f92d1afef66fcf1ef83d516819d9992ca547f592bd/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459", size = 4980804, upload-time = "2026-06-12T20:01:42.853Z" }, + { url = "https://files.pythonhosted.org/packages/1f/09/f42b1d190c5ba75f72062a387f8030d1d75f6ab035788f1d9c4b01de6525/cryptography-49.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e", size = 3810026, upload-time = "2026-06-12T20:02:39.262Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/5bb823f5bedcf80718cea7fbc95ec5515cca3769633c4b01a32be7f30e7c/cryptography-49.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a", size = 4025947, upload-time = "2026-06-12T20:01:25.745Z" }, + { url = "https://files.pythonhosted.org/packages/3d/df/40577043ca124e17012f408ddddaeb213b856336ac82ddb3bc915f39e29f/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4", size = 4692429, upload-time = "2026-06-12T20:01:53.628Z" }, + { url = "https://files.pythonhosted.org/packages/2c/99/2d13299eb3dd27b02dcfaafcc91d6b5cb3329f7cbd6d8f51921acd566c1a/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18", size = 4700968, upload-time = "2026-06-12T20:02:45.383Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4d/9c0cd02f95e2602dd5e563da149ee0830abef3537be8b34dc56281ebe27a/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69", size = 4697758, upload-time = "2026-06-12T20:01:41.13Z" }, + { url = "https://files.pythonhosted.org/packages/24/01/186c825898477d77e2324d5360fefe622ff1d8d1963ec0554e2cada8ec77/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64", size = 5298863, upload-time = "2026-06-12T20:02:24.579Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7b/62cbbab75d0659865bf0273790031544a0b16c8072d258f9428dcd8190dc/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21", size = 4735983, upload-time = "2026-06-12T20:01:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/6c/72/3e798c064bc39e471008075d0f9bc9daf77a80879c092e4a8e170c585ed4/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9", size = 4334173, upload-time = "2026-06-12T20:01:44.743Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ee/6fca21d1ac73e06f8bef71940abfd4d2f6472b4bca284d770f32bd4086f6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc", size = 4697298, upload-time = "2026-06-12T20:02:20.918Z" }, + { url = "https://files.pythonhosted.org/packages/67/d0/a5fcd3515f0bae49a7b6d0413cc1bdccdcc1fc0047037a0d480642cdc5d6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8", size = 5254338, upload-time = "2026-06-12T20:02:22.737Z" }, + { url = "https://files.pythonhosted.org/packages/a0/84/84fe36f19caf857d61cb7fc9c63035a47ffabd84ea12d1d393148efa3615/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36", size = 4735650, upload-time = "2026-06-12T20:02:41.389Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, + { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, +] + [[package]] name = "defusedxml" version = "0.7.1" @@ -491,6 +565,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" }, ] +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + [[package]] name = "pyee" version = "13.0.1" @@ -699,6 +782,8 @@ dependencies = [ { name = "beautifulsoup4" }, { name = "bleach" }, { name = "celery", extra = ["redis"] }, + { name = "certifi" }, + { name = "cryptography" }, { name = "defusedxml" }, { name = "django" }, { name = "feedparser" }, @@ -735,6 +820,8 @@ requires-dist = [ { name = "beautifulsoup4", specifier = "==4.15.0" }, { name = "bleach", specifier = "==6.4.0" }, { name = "celery", extras = ["redis"], specifier = "==5.6.3" }, + { name = "certifi", specifier = "==2026.6.17" }, + { name = "cryptography", specifier = "==49.0.0" }, { name = "defusedxml", specifier = "==0.7.1" }, { name = "django", specifier = "==5.2.16" }, { name = "feedparser", specifier = "==6.0.12" },