feat: release regional radar and mailbox integrations
deploy / deploy (push) Canceled after 0s

This commit is contained in:
Jens
2026-07-22 05:12:07 +02:00
parent 598d3ec18a
commit 551d0f46c2
131 changed files with 6209 additions and 336 deletions
@@ -3,6 +3,7 @@ from pathlib import Path
from zoneinfo import ZoneInfo
import pytest
from cryptography.fernet import Fernet
from django.core import mail
from django.test import override_settings
@@ -10,8 +11,10 @@ from apps.jobs.models import Feedback, ScoreRun
from apps.jobs.services.scoring import score_and_save
from apps.notifications.models import DigestOutbox
from apps.notifications.services import build_digest_payload, create_daily_outbox, send_digest
from apps.sources.models import EmailMessageRecord, RawDocument
from apps.sources.models import EmailMessageRecord, MailboxConnection, RawDocument
from apps.sources.services.email_import import ingest_email, message_identity
from apps.sources.services.imap_import import poll_imap_mailbox
from apps.sources.services.mailbox_connections import encrypt_mailbox_password
@pytest.mark.django_db
@@ -27,6 +30,132 @@ def test_email_import_is_idempotent(profile):
assert len(first.links) == 2
@pytest.mark.django_db
def test_vdab_alert_has_provider_provenance_and_replays_idempotently(profile):
raw = Path("fixtures/emails/vdab_job_alert.eml").read_bytes()
first = ingest_email(raw)
second = ingest_email(raw)
assert second.pk == first.pk
assert first.processed is True
assert first.raw_document.metadata["alert_provider"] == "vdab"
assert len(first.links) == 2
assert all("vdab.be/vindeenjob/vacatures/" in link for link in first.links)
@pytest.mark.django_db
@pytest.mark.parametrize(
("platform", "fixture", "expected_domain"),
(
(MailboxConnection.Platform.LINKEDIN, "linkedin_job_alert.eml", "linkedin.com"),
(MailboxConnection.Platform.ICTJOB, "ictjob_job_alert.eml", "ictjob.be"),
(MailboxConnection.Platform.JOBAT, "jobat_job_alert.eml", "jobat.be"),
(MailboxConnection.Platform.STEPSTONE, "stepstone_job_alert.eml", "stepstone.be"),
(MailboxConnection.Platform.CAREERJET, "careerjet_job_alert.eml", "careerjet.be"),
(MailboxConnection.Platform.RANDSTAD, "randstad_job_alert.eml", "randstad.be"),
(
MailboxConnection.Platform.ROBERTHALF,
"roberthalf_job_alert.eml",
"roberthalf.com",
),
),
)
def test_jobboard_mail_import_keeps_platform_provenance_and_rejects_external_links(
profile, platform, fixture, expected_domain
):
raw = (Path("fixtures/emails") / fixture).read_bytes()
record = ingest_email(raw, platform=platform)
assert record.processed is True
assert len(record.links) == 1
assert expected_domain in record.links[0]
assert "outside.example.invalid" not in record.links[0]
assert record.raw_document.metadata["alert_provider"] == platform
assert record.raw_document.source.name == f"{platform.label} vacaturemailbox"
@pytest.mark.django_db
def test_imap_poll_is_bounded_and_one_bad_message_does_not_abort(profile):
valid = Path("fixtures/emails/vdab_job_alert.eml").read_bytes()
oversized = b"x" * (len(valid) + 1)
class FakeImapClient:
def __init__(self, host, port, timeout=None):
self.host = host
self.port = port
self.timeout = timeout
self.logged_out = False
def login(self, user, password):
return "OK", []
def select(self, mailbox, readonly=True):
assert readonly is True
return "OK", []
def uid(self, command, *args):
if command == "search":
return "OK", [b"1 2 3"]
uid = args[0]
if uid == b"1":
return "OK", [(b"RFC822", valid)]
if uid == b"2":
return "OK", [(b"RFC822", oversized)]
return "NO", []
def logout(self):
self.logged_out = True
clients = []
def factory(host, port, timeout=None):
client = FakeImapClient(host, port, timeout=timeout)
clients.append(client)
return client
key = Fernet.generate_key().decode("ascii")
with override_settings(
MAILBOX_CREDENTIAL_KEYS=[key],
IMAP_MAX_MESSAGES_PER_POLL=3,
IMAP_MAX_MESSAGE_BYTES=len(valid),
):
connection = MailboxConnection.objects.create(
user=profile.user,
platform=MailboxConnection.Platform.VDAB,
provider=MailboxConnection.Provider.CUSTOM,
custom_host="imap.example.invalid",
username="radar@example.invalid",
encrypted_password=encrypt_mailbox_password("test-only"),
)
first = poll_imap_mailbox(
connection, client_factory=factory, host_validator=lambda url: url
)
second = poll_imap_mailbox(
connection, client_factory=factory, host_validator=lambda url: url
)
assert first == {
"status": "ok",
"imported": 1,
"duplicates": 0,
"skipped_oversized": 1,
"failed": 1,
}
assert second == {
"status": "ok",
"imported": 0,
"duplicates": 1,
"skipped_oversized": 1,
"failed": 1,
}
assert all(client.logged_out for client in clients)
record = EmailMessageRecord.objects.get(mailbox_connection=connection)
assert record.raw_document.metadata["alert_provider"] == "vdab"
assert record.raw_document.metadata["mailbox_connection_id"] == connection.pk
@pytest.mark.django_db
def test_digest_payload_outbox_deduplication_and_send(profile, job):
score = score_and_save(job, profile)
@@ -0,0 +1,205 @@
from datetime import timedelta
import pytest
from cryptography.fernet import Fernet
from django.contrib.auth import get_user_model
from django.test import Client, override_settings
from django.urls import reverse
from django.utils import timezone
from apps.sources.models import MailboxConnection
from apps.sources.services.mailbox_connections import (
claim_mailbox_connection,
decrypt_mailbox_password,
)
from apps.sources.tasks import poll_mailbox, schedule_mailbox_polls
def _payload(**overrides):
payload = {
"platform": MailboxConnection.Platform.VDAB,
"provider": MailboxConnection.Provider.GMAIL,
"custom_host": "",
"port": 993,
"username": "vdab-alerts@example.invalid",
"password": "test-app-password",
"mailbox": "INBOX",
"poll_interval_minutes": 15,
"enabled": "on",
}
payload.update(overrides)
return payload
@pytest.mark.integration
@pytest.mark.django_db
def test_user_can_link_edit_pause_and_sync_separate_platform_mailboxes(client, user, monkeypatch):
key = Fernet.generate_key().decode("ascii")
calls = []
monkeypatch.setattr(
"apps.sources.views.poll_mailbox.delay",
lambda connection_id, force=False: calls.append((connection_id, force)),
)
client.force_login(user)
with override_settings(MAILBOX_CREDENTIAL_KEYS=[key]):
response = client.post(reverse("sources:mailbox_add"), _payload())
assert response.status_code == 302
vdab = MailboxConnection.objects.get(user=user, platform="vdab")
assert decrypt_mailbox_password(vdab) == "test-app-password"
assert "test-app-password" not in vdab.encrypted_password
response = client.post(
reverse("sources:mailbox_add"),
_payload(
platform=MailboxConnection.Platform.INDEED,
provider=MailboxConnection.Provider.OUTLOOK,
username="indeed-alerts@example.invalid",
password="second-app-password",
poll_interval_minutes=30,
),
)
assert response.status_code == 302
indeed = MailboxConnection.objects.get(user=user, platform="indeed")
edit_response = client.get(reverse("sources:mailbox_edit", args=[vdab.pk]))
assert edit_response.status_code == 200
assert b"test-app-password" not in edit_response.content
assert vdab.encrypted_password.encode() not in edit_response.content
original_ciphertext = vdab.encrypted_password
response = client.post(
reverse("sources:mailbox_edit", args=[vdab.pk]),
_payload(
username="vdab-updated@example.invalid",
password="",
poll_interval_minutes=5,
),
)
assert response.status_code == 302
vdab.refresh_from_db()
assert vdab.username == "vdab-updated@example.invalid"
assert vdab.poll_interval_minutes == 5
assert vdab.encrypted_password == original_ciphertext
assert decrypt_mailbox_password(vdab) == "test-app-password"
response = client.post(reverse("sources:mailbox_toggle", args=[indeed.pk]))
assert response.status_code == 302
indeed.refresh_from_db()
assert indeed.enabled is False
response = client.post(reverse("sources:sync_platform_alerts", args=[vdab.pk]))
assert response.status_code == 302
assert calls == [
(vdab.pk, True),
(indeed.pk, True),
(vdab.pk, True),
(vdab.pk, True),
]
@pytest.mark.integration
@pytest.mark.django_db
def test_sources_page_offers_all_reviewed_jobboard_mailboxes(client, user):
client.force_login(user)
response = client.get(reverse("sources:list"))
assert response.status_code == 200
for platform in MailboxConnection.Platform:
assert f'value="{platform.value}"'.encode() in response.content
assert f"{platform.label}-alert instellen".encode() in response.content
@pytest.mark.integration
@pytest.mark.django_db
def test_mailbox_views_are_user_scoped_and_require_encryption_key(client, user, monkeypatch):
other = get_user_model().objects.create_user(username="other", password="test-only-password")
other_connection = MailboxConnection.objects.create(
user=other,
platform=MailboxConnection.Platform.VDAB,
provider=MailboxConnection.Provider.GMAIL,
username="private@example.invalid",
encrypted_password="opaque-ciphertext",
)
client.force_login(user)
monkeypatch.setattr("apps.sources.views.poll_mailbox.delay", lambda *args, **kwargs: None)
assert (
client.get(reverse("sources:mailbox_edit", args=[other_connection.pk])).status_code == 404
)
assert (
client.post(reverse("sources:sync_platform_alerts", args=[other_connection.pk])).status_code
== 404
)
assert (
client.post(reverse("sources:mailbox_toggle", args=[other_connection.pk])).status_code
== 404
)
with override_settings(MAILBOX_CREDENTIAL_KEYS=[]):
response = client.post(reverse("sources:mailbox_add"), _payload(password="visible-once"))
assert response.status_code == 200
assert MailboxConnection.objects.filter(user=user).count() == 0
assert b"visible-once" not in response.content
assert b"Credentialopslag is niet geconfigureerd" in response.content
anonymous_response = Client().post(reverse("sources:mailbox_add"), _payload())
assert anonymous_response.status_code == 302
csrf_client = Client(enforce_csrf_checks=True)
csrf_client.force_login(user)
assert csrf_client.post(reverse("sources:mailbox_add"), _payload()).status_code == 403
@pytest.mark.integration
@pytest.mark.django_db
def test_scheduler_queues_only_due_active_mailboxes_and_claim_is_exclusive(user, monkeypatch):
now = timezone.now()
due = MailboxConnection.objects.create(
user=user,
platform=MailboxConnection.Platform.VDAB,
provider=MailboxConnection.Provider.GMAIL,
username="due@example.invalid",
encrypted_password="opaque",
next_poll_at=now - timedelta(minutes=1),
)
MailboxConnection.objects.create(
user=user,
platform=MailboxConnection.Platform.INDEED,
provider=MailboxConnection.Provider.OUTLOOK,
username="later@example.invalid",
encrypted_password="opaque",
next_poll_at=now + timedelta(minutes=20),
)
calls = []
monkeypatch.setattr(
"apps.sources.tasks.poll_mailbox.delay", lambda connection_id: calls.append(connection_id)
)
assert schedule_mailbox_polls() == {"scheduled": 1}
assert calls == [due.pk]
assert claim_mailbox_connection(connection_id=due.pk, worker_token="worker-a") is not None
assert claim_mailbox_connection(connection_id=due.pk, worker_token="worker-b") is None
@pytest.mark.integration
@pytest.mark.django_db
def test_mailbox_task_does_not_persist_or_return_secret_on_imap_error(user, monkeypatch):
connection = MailboxConnection.objects.create(
user=user,
platform=MailboxConnection.Platform.VDAB,
provider=MailboxConnection.Provider.GMAIL,
username="errors@example.invalid",
encrypted_password="opaque",
next_poll_at=timezone.now() - timedelta(minutes=1),
)
monkeypatch.setattr(
"apps.sources.tasks.poll_imap_mailbox",
lambda connection: (_ for _ in ()).throw(RuntimeError("leaked-app-password")),
)
result = poll_mailbox(connection.pk)
connection.refresh_from_db()
assert "leaked-app-password" not in str(result)
assert "leaked-app-password" not in connection.last_error_message
assert result == {"status": "failed", "error_category": "imap", "imported": 0}
+34
View File
@@ -34,6 +34,40 @@ def test_pipeline_is_idempotent_and_scores(source, profile):
assert ScoreRun.objects.filter(profile=profile).count() == 2
@pytest.mark.django_db
def test_pipeline_uses_reviewed_employer_identity_for_public_ats_feed(profile):
source = Source.objects.create(
name="Deliverect",
source_type=Source.Type.ATS,
base_url="https://api.lever.co/v0/postings/deliverect?mode=json",
domain="api.lever.co",
status=Source.Status.ACTIVE,
policy=Source.Policy.ALLOW,
metadata={"employer_name": "Deliverect", "direct_employer": True},
)
content = Path("fixtures/ats/lever-public-postings.json").read_text(encoding="utf-8")
document = RawDocument.objects.create(
source=source,
url=source.base_url,
final_url=source.base_url,
kind=RawDocument.Kind.JSON,
content_type="application/json",
content_hash=hashlib.sha256(content.encode()).hexdigest(),
body_text=content,
byte_length=len(content.encode()),
retain_until=timezone.now() + timedelta(days=7),
)
result = process_raw_document(document)
assert result["created"] == 1
job = JobPosting.objects.get()
assert job.employer_name == "Deliverect"
assert job.direct_employer is True
evidence = job.source_aliases.get().field_evidence.get(field_name="employer_name")
assert evidence.extraction_method == "reviewed-source-config"
@pytest.mark.django_db
def test_pipeline_resolves_recruiter_alias_to_direct_employer():
source = Source.objects.create(
@@ -0,0 +1,106 @@
from decimal import Decimal
import pytest
from django.urls import reverse
from apps.jobs.models import GeocodeLocationLookup
from apps.profiles.forms import SearchProfileForm
def _profile_post_data(**overrides):
data = {
"name": "Testprofiel",
"is_active": "on",
"home_postal_code": "3500",
"max_distance_km": "45",
"desired_titles": ["infrastructure engineer", "cloud engineer"],
"excluded_titles": ["sales"],
"desired_skills": ["azure", "terraform"],
"excluded_skills": ["cold calling"],
"allowed_employment_types": ["full_time", "permanent"],
"preferred_workplace": ["hybrid"],
"preferred_regions": ["Limburg"],
"excluded_regions": ["West-Vlaanderen"],
"recommendation_threshold": "65",
"top_match_threshold": "90",
"digest_time": "07:30",
}
data.update(overrides)
return data
@pytest.mark.django_db
def test_profile_form_uses_guided_choices_and_preserves_existing_custom_values(profile):
form = SearchProfileForm(instance=profile)
assert form.fields["home_postal_code"].widget.attrs["maxlength"] == 4
assert "home_municipality" not in form.fields
assert "home_latitude" not in form.fields
assert "home_longitude" not in form.fields
assert form.fields["desired_titles"].widget.allow_multiple_selected is True
assert ("systeembeheerder", "systeembeheerder") in form.fields["desired_titles"].choices
assert ("VMware", "VMware") in form.fields["desired_skills"].choices
@pytest.mark.django_db
def test_profile_form_rejects_non_belgian_postcode_and_conflicting_choices(profile):
invalid_postcode = SearchProfileForm(
data=_profile_post_data(home_postal_code="3500 Hasselt"), instance=profile
)
assert invalid_postcode.is_valid() is False
assert "home_postal_code" in invalid_postcode.errors
conflicting = SearchProfileForm(
data=_profile_post_data(preferred_regions=["Limburg"], excluded_regions=["Limburg"]),
instance=profile,
)
assert conflicting.is_valid() is False
assert "excluded_regions" in conflicting.errors
@pytest.mark.django_db
def test_profile_update_derives_home_location_from_postcode(client, user, profile):
GeocodeLocationLookup.objects.create(
source_name="test-geodata",
source_version="2026-01",
query_kind=GeocodeLocationLookup.QueryKind.POSTAL,
query_value="3500",
postal_code="3500",
municipality="Hasselt",
region="Limburg",
latitude=Decimal("50.930700"),
longitude=Decimal("5.332500"),
confidence=Decimal("1.00"),
)
client.force_login(user)
response = client.post(
reverse("profiles:edit", kwargs={"pk": profile.pk}), _profile_post_data()
)
assert response.status_code == 302
profile.refresh_from_db()
assert profile.home_postal_code == "3500"
assert profile.home_municipality == "Hasselt"
assert profile.home_latitude == Decimal("50.930700")
assert profile.desired_titles == ["infrastructure engineer", "cloud engineer"]
assert profile.preferred_workplace == ["hybrid"]
@pytest.mark.django_db
def test_profile_update_clears_stale_coordinates_when_postcode_is_unknown(client, user, profile):
client.force_login(user)
response = client.post(
reverse("profiles:edit", kwargs={"pk": profile.pk}),
_profile_post_data(home_postal_code="9999"),
follow=True,
)
assert response.status_code == 200
assert "De postcode is bewaard" in response.content.decode("utf-8")
profile.refresh_from_db()
assert profile.home_postal_code == "9999"
assert profile.home_municipality == ""
assert profile.home_latitude is None
assert profile.home_longitude is None
+91
View File
@@ -0,0 +1,91 @@
from datetime import timedelta
import pytest
import yaml
from django.core.management import call_command
from django.utils import timezone
from apps.sources.models import Source, SourcePolicyReview
from apps.sources.views import _source_list_queryset
@pytest.mark.django_db
def test_seed_sources_registers_reviewed_live_feed_and_hides_demo(tmp_path):
Source.objects.create(
name="Voorbeeldwerkgever (fixture)",
source_type=Source.Type.EMPLOYER,
base_url="https://jobs.example.org/vacatures/",
domain="jobs.example.org",
)
expires_at = (timezone.now() + timedelta(days=30)).isoformat()
retired_source = Source.objects.create(
name="Brusselse werkgever",
source_type=Source.Type.ATS,
base_url="https://brussels-employer.example/jobs",
domain="brussels-employer.example",
status=Source.Status.ACTIVE,
policy=Source.Policy.ALLOW,
next_run_at=timezone.now(),
)
payload = {
"disable_demo_sources": True,
"retire_source_urls": [retired_source.base_url],
"sources": [
{
"name": "VITO",
"url": "https://vito.recruitee.com/api/offers/",
"type": "ats",
"status": "trial",
"policy": "review",
"parser": "ats-recruitee",
"allow_public_endpoint": True,
"metadata": {"country_scope": "BE"},
"review": {
"decision": "trial",
"reason": "Publieke feed",
"evidence_link": "https://docs.recruitee.com/reference/offers",
"expires_at": expires_at,
},
},
{
"name": "Lever werkgever A",
"url": "https://api.lever.co/v0/postings/werkgever-a?mode=json",
"type": "ats",
"status": "trial",
"policy": "allow",
"parser": "ats-lever",
"allow_public_endpoint": True,
},
{
"name": "Lever werkgever B",
"url": "https://api.lever.co/v0/postings/werkgever-b?mode=json",
"type": "ats",
"status": "trial",
"policy": "allow",
"parser": "ats-lever",
"allow_public_endpoint": True,
},
],
}
seed_path = tmp_path / "sources.yaml"
seed_path.write_text(yaml.safe_dump(payload), encoding="utf-8")
call_command("seed_sources", str(seed_path))
call_command("seed_sources", str(seed_path))
demo = Source.objects.get(domain="jobs.example.org")
assert demo.status == Source.Status.DISABLED
assert demo.metadata["demo"] is True
retired_source.refresh_from_db()
assert retired_source.status == Source.Status.DISABLED
assert retired_source.next_run_at is None
assert retired_source.metadata["hidden_from_source_list"] is True
assert retired_source.metadata["retired_reason"] == "Buiten de ingestelde regionale dekking"
source = Source.objects.get(name="VITO")
assert retired_source not in _source_list_queryset()
assert source in _source_list_queryset()
assert source.allow_public_endpoint is True
assert source.parser_key == "ats-recruitee"
assert source.latest_policy_review.decision == SourcePolicyReview.Decision.TRIAL
assert source.policy_reviews.count() == 1
assert Source.objects.filter(domain="api.lever.co", source_type=Source.Type.ATS).count() == 2
+48 -1
View File
@@ -177,7 +177,20 @@ def test_job_profile_source_and_application_views(client, user, profile, job, so
@pytest.mark.integration
@pytest.mark.django_db
def test_sources_candidate_bulk_actions_and_evidence_view(client, user):
def test_sources_candidate_bulk_actions_and_evidence_view(client, user, profile):
profile.home_postal_code = "2400"
profile.max_distance_km = 40
profile.home_latitude = None
profile.home_longitude = None
profile.save(
update_fields=[
"home_postal_code",
"max_distance_km",
"home_latitude",
"home_longitude",
"updated_at",
]
)
candidate = Source.objects.create(
name="Discoverd candidate",
source_type=Source.Type.EMPLOYER,
@@ -206,12 +219,37 @@ def test_sources_candidate_bulk_actions_and_evidence_view(client, user):
status=Source.Status.CANDIDATE,
policy=Source.Policy.REVIEW,
)
watchlist = Source.objects.create(
name="Interessante werkgever zonder stabiele feed",
source_type=Source.Type.EMPLOYER,
base_url="https://watch.example.org/jobs/",
domain="watch.example.org",
status=Source.Status.PAUSED,
policy=Source.Policy.REVIEW,
metadata={
"watchlist": True,
"municipality_scope": "Olen",
"watchlist_reason": "Regionaal interessante werkgever",
"current_observation": "Momenteel geen lokale vacature bevestigd.",
"public_job_page": "https://watch.example.org/jobs/",
},
)
client.force_login(user)
response = client.get(reverse("sources:list"))
assert response.status_code == 200
content = response.content.decode("utf-8")
assert "jobs.example.org/careers" in content
assert "Platformmailboxen" in content
assert "Mailbox koppelen en testen" in content
assert "elke vijf minuten" in content
assert "2400 Mol" in content
assert "maximaal 40 km" in content
assert "Coördinaten ontbreken nog" in content
assert "Interessante werkgeverswaaklijst" in content
assert watchlist.name in content
assert response.context["watchlist_count"] == 1
assert {item.pk for item in response.context["sources"]} == {candidate.pk, other.pk}
response = client.post(
reverse("sources:bulk"),
@@ -229,3 +267,12 @@ def test_sources_candidate_bulk_actions_and_evidence_view(client, user):
other.refresh_from_db()
assert other.status == Source.Status.DISABLED
assert other.policy == Source.Policy.DENY
response = client.post(
reverse("sources:bulk"),
{"action": "dismiss", "source_ids": [str(watchlist.pk)]},
)
assert response.status_code == 302
watchlist.refresh_from_db()
assert watchlist.status == Source.Status.PAUSED
assert watchlist.policy == Source.Policy.REVIEW