diff --git a/.env.example b/.env.example index f1f9df1..8f79895 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,7 @@ DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1 DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost:1226 DJANGO_TIME_ZONE=Europe/Brussels VACATURERADAR_OWNER_NAME=Jens -VACATURERADAR_VERSION=0.3.13 +VACATURERADAR_VERSION=0.3.14 # Gebruik DATABASE_URL niet voor SQLite. Laat leeg voor lokale sqlite-ontwikkeling. POSTGRES_DB=vacatureradar diff --git a/CHANGELOG.md b/CHANGELOG.md index ca510c1..b3f9409 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ Alle betekenisvolle wijzigingen worden hier bijgehouden. Het project volgt voorlopig een pre-1.0 semantische versieaanpak. +## 0.3.14 - 2026-07-29 + +### Gewijzigd + +- De importpipeline hergebruikt werkgevers en actieve profielen binnen één run, synchroniseert provenance gebundeld en vermijdt volledige rescoring van inhoudelijk ongewijzigde vacatures terwijl ScoreRun-historiek behouden blijft. +- `last_changed` wijzigt alleen nog bij een inhoudelijke vacaturewijziging; een nieuwe waarneming actualiseert uitsluitend `last_seen`. +- Een fail-closed PostgreSQL-runner levert één warm-up, drie metingen, p50/p95, throughput, geheugen en queryfamilies als machineleesbaar rapport. + +### Performance + +- De officiële 250-itemmeting is gemiddeld 44,74% sneller, gebruikt minstens 63,73% minder queries en verwerkt 83,42% meer documenten per seconde dan 0.3.13. + ## Niet uitgebracht ### Toegevoegd diff --git a/CURRENT_STATE.md b/CURRENT_STATE.md index 9bbbd8e..f3d958d 100644 --- a/CURRENT_STATE.md +++ b/CURRENT_STATE.md @@ -1,5 +1,5 @@ # Huidige toestand -VacatureRadar 0.3.13 is een single-user-first, server-rendered vacature-intelligencecockpit. De actuele uitvoeringsstatus, laatste gate en externe restpunten staan in [docs/ai/PROJECT_STATE.md](docs/ai/PROJECT_STATE.md). Productgebruik en installatie beginnen in [README.md](README.md). +VacatureRadar 0.3.14 is een single-user-first, server-rendered vacature-intelligencecockpit. De actuele uitvoeringsstatus, laatste gate en externe restpunten staan in [docs/ai/PROJECT_STATE.md](docs/ai/PROJECT_STATE.md). Productgebruik en installatie beginnen in [README.md](README.md). -Versie 0.3.13 draait publiek op `https://vacatureradar.itworx.tech` als Unraid AIO-image `sha256:34aef8853f77a3f311b119bd67d9f1160a97fd4a21e62c9776222be289bf3b88`. TLS, proxy, secure cookies, migrations, containerrestart, kernflows, drie responsive viewports, back-up/restore en rollback zijn op 2026-07-29 live bewezen. Alleen optionele externe identity- en mailboxcredentials staan nog in [USER_INPUT_REQUIRED.md](USER_INPUT_REQUIRED.md). +Release 0.3.14 verlaagt voor de vaste 250-itemimport de gemiddelde duur met 44,74% en het queryvolume met minstens 63,73%, met behoud van parser-, dedupe-, ranking-, provenance- en scorehistoriekcontracten. Het actuele productie-image en livebewijs worden na deployment in [docs/ai/PROJECT_STATE.md](docs/ai/PROJECT_STATE.md) geregistreerd. Alleen optionele externe identity- en mailboxcredentials staan nog in [USER_INPUT_REQUIRED.md](USER_INPUT_REQUIRED.md). diff --git a/apps/jobs/services/pipeline.py b/apps/jobs/services/pipeline.py index 0321681..824092b 100644 --- a/apps/jobs/services/pipeline.py +++ b/apps/jobs/services/pipeline.py @@ -1,12 +1,21 @@ from __future__ import annotations +from dataclasses import dataclass +from dataclasses import field as dataclass_field from decimal import Decimal from urllib.parse import urlsplit from django.db import IntegrityError, transaction from django.utils import timezone -from apps.jobs.models import Employer, FieldProvenance, JobPosting, JobSourceAlias, JobVersion +from apps.jobs.models import ( + Employer, + FieldProvenance, + JobPosting, + JobSourceAlias, + JobVersion, + ScoreRun, +) from apps.profiles.models import SearchProfile from apps.sources.adapters.base import FieldEvidence from apps.sources.adapters.registry import registry @@ -29,17 +38,31 @@ RECRUITER_TERMS = { } +@dataclass +class PersistenceContext: + """Run-scoped reference cache; never shared between workers or imports.""" + + employers: dict[tuple[str, str], Employer | None] = dataclass_field(default_factory=dict) + active_profiles: list[SearchProfile] | None = None + latest_scores: dict[tuple[object, int], ScoreRun] = dataclass_field(default_factory=dict) + + def _confidence(value: float) -> Decimal: return Decimal(str(max(0.0, min(1.0, value)))) -def resolve_employer(draft: CanonicalJobDraft) -> Employer | None: +def resolve_employer( + draft: CanonicalJobDraft, *, context: PersistenceContext | None = None +) -> Employer | None: name = draft.employer_name.strip() domain = draft.employer_domain.strip() if not name and not domain: return None display_name = name or domain normalized = normalize_token(display_name) + cache_key = (normalized, domain) + if context is not None and cache_key in context.employers: + return context.employers[cache_key] recruiter = any(term in normalized for term in RECRUITER_TERMS) employer, _ = Employer.objects.get_or_create( normalized_name=normalized, @@ -61,6 +84,8 @@ def resolve_employer(draft: CanonicalJobDraft) -> Employer | None: changed.extend(["is_recruiter", "is_direct_employer"]) if changed: employer.save(update_fields=[*changed, "updated_at"]) + if context is not None: + context.employers[cache_key] = employer return employer @@ -162,12 +187,81 @@ def _apply_draft( if direct and canonical_url and job.canonical_url != canonical_url: job.canonical_url = canonical_url changed.append("canonical_url") - if changed and "last_changed" not in changed: + substantive_changes = [field for field in changed if field != "last_seen"] + if substantive_changes and "last_changed" not in changed: job.last_changed = timezone.now() changed.append("last_changed") return changed +def _sync_evidence( + *, + job: JobPosting, + alias: JobSourceAlias, + evidence_items: list[FieldEvidence], + parser_version: str, +) -> None: + existing = { + (item.field_name, item.extraction_method): item + for item in FieldProvenance.objects.filter(job=job, source_alias=alias) + } + creates: list[FieldProvenance] = [] + updates: list[FieldProvenance] = [] + for evidence in evidence_items: + key = (evidence.field_name, evidence.method) + values = { + "confidence": _confidence(evidence.confidence), + "evidence_excerpt": evidence.evidence[:1000], + "parser_version": parser_version, + } + current = existing.get(key) + if current is None: + creates.append( + FieldProvenance( + job=job, + source_alias=alias, + field_name=evidence.field_name, + extraction_method=evidence.method, + **values, + ) + ) + continue + changed = False + for field_name, value in values.items(): + if getattr(current, field_name) != value: + setattr(current, field_name, value) + changed = True + if changed: + current.updated_at = timezone.now() + updates.append(current) + if creates: + FieldProvenance.objects.bulk_create(creates, batch_size=250) + if updates: + FieldProvenance.objects.bulk_update( + updates, + ["confidence", "evidence_excerpt", "parser_version", "updated_at"], + batch_size=250, + ) + + +def _copy_score(score: ScoreRun) -> ScoreRun: + return ScoreRun.objects.create( + job=score.job, + profile=score.profile, + profile_version=score.profile_version, + score=score.score, + confidence=score.confidence, + recommendation=score.recommendation, + components=score.components, + positives=score.positives, + concerns=score.concerns, + hard_exclusions=score.hard_exclusions, + evidence=score.evidence, + model_version=score.model_version, + prompt_version=score.prompt_version, + ) + + @transaction.atomic def persist_draft( draft: CanonicalJobDraft, @@ -176,14 +270,16 @@ def persist_draft( parser_key: str, parser_version: str, extraction_confidence: float, + context: PersistenceContext | None = None, ) -> tuple[JobPosting, DedupeDecision, bool]: source = document.source - employer = resolve_employer(draft) + employer = resolve_employer(draft, context=context) decision = find_existing_job(draft, source=source) direct = _source_is_direct(source, draft) if decision.resolved_direct: direct = True created = False + substantive_change = False if decision.job is None: try: @@ -240,6 +336,7 @@ def persist_draft( if extraction_confidence > float(job.extraction_confidence): job.extraction_confidence = _confidence(extraction_confidence) changed.append("extraction_confidence") + substantive_change = any(field not in {"last_seen", "last_changed"} for field in changed) if changed: job.save(update_fields=list(dict.fromkeys([*changed, "updated_at"]))) @@ -284,31 +381,58 @@ def persist_draft( update_fields=["last_seen", "raw_document", "payload", "is_canonical", "updated_at"] ) - for evidence in draft.evidence: - FieldProvenance.objects.update_or_create( - job=job, - source_alias=alias, - field_name=evidence.field_name, - extraction_method=evidence.method, - defaults={ - "confidence": _confidence(evidence.confidence), - "evidence_excerpt": evidence.evidence[:1000], - "parser_version": parser_version, - }, - ) - - JobVersion.objects.get_or_create( + _sync_evidence( job=job, - content_hash=job.content_hash, - defaults={"snapshot": job_snapshot(job), "changed_fields": []}, + alias=alias, + evidence_items=draft.evidence, + parser_version=parser_version, ) - for profile in SearchProfile.objects.filter(is_active=True): - score_and_save(job, profile) + + if created or substantive_change: + JobVersion.objects.get_or_create( + job=job, + content_hash=job.content_hash, + defaults={"snapshot": job_snapshot(job), "changed_fields": []}, + ) + if context is not None: + if context.active_profiles is None: + context.active_profiles = list(SearchProfile.objects.filter(is_active=True)) + profiles = context.active_profiles + else: + profiles = SearchProfile.objects.filter(is_active=True) + for profile in profiles: + score = score_and_save(job, profile) + if context is not None: + context.latest_scores[(job.pk, profile.pk)] = score + else: + if context is not None: + if context.active_profiles is None: + context.active_profiles = list(SearchProfile.objects.filter(is_active=True)) + profiles = context.active_profiles + else: + profiles = SearchProfile.objects.filter(is_active=True) + for profile in profiles: + cache_key = (job.pk, profile.pk) + previous = context.latest_scores.get(cache_key) if context is not None else None + if previous is None: + previous = ( + ScoreRun.objects.filter(job=job, profile=profile) + .order_by("-created_at") + .first() + ) + if previous is None or previous.profile_version != profile.version: + score = score_and_save(job, profile) + else: + score = _copy_score(previous) + if context is not None: + context.latest_scores[cache_key] = score return job, decision, created @transaction.atomic -def process_raw_document(document: RawDocument) -> dict[str, int | str | list[str]]: +def process_raw_document( + document: RawDocument, *, context: PersistenceContext | None = None +) -> dict[str, int | str | list[str]]: result = registry.extract(document) document.parser_key = result.parser_key document.parser_version = result.parser_version @@ -341,6 +465,7 @@ def process_raw_document(document: RawDocument) -> dict[str, int | str | list[st parser_key=result.parser_key, parser_version=result.parser_version, extraction_confidence=result.confidence, + context=context, ) if was_created: created += 1 diff --git a/artifacts/import-performance-report.json b/artifacts/import-performance-report.json new file mode 100644 index 0000000..4d8dcfc --- /dev/null +++ b/artifacts/import-performance-report.json @@ -0,0 +1,745 @@ +{ + "configuration": { + "concurrency": 1, + "dataset": "fixtures/benchmark/quality_benchmark.json", + "iterations": 3, + "jobs": 250, + "warmup_runs": 1 + }, + "database_identity": { + "engine": "django.db.backends.postgresql", + "host": "127.0.0.1", + "name_prefix": "vacatureradar_bench_" + }, + "iterations": [ + { + "dataset_version": "vr116-2026-07-21", + "dedupe": { + "case_reports": [ + { + "decision": "exact_external_id", + "decision_similarity": 1.0, + "expected_match": "seed-infra", + "expected_match_present": true, + "id": "D-001", + "is_expected_match": true, + "matched": true, + "matched_job_id": "267f0fd5-4543-404c-acd1-c6f6ee72e4c0", + "query": "D-001" + }, + { + "decision": "exact_external_id", + "decision_similarity": 1.0, + "expected_match": "seed-network", + "expected_match_present": true, + "id": "D-002", + "is_expected_match": true, + "matched": true, + "matched_job_id": "6a196e1a-bed6-44d0-94df-c2578c367673", + "query": "D-002" + }, + { + "decision": "new", + "decision_similarity": 0.0, + "expected_match": null, + "expected_match_present": false, + "id": "D-003", + "is_expected_match": false, + "matched": false, + "matched_job_id": null, + "query": "D-003" + }, + { + "decision": "new", + "decision_similarity": 0.0, + "expected_match": null, + "expected_match_present": false, + "id": "D-004", + "is_expected_match": false, + "matched": false, + "matched_job_id": null, + "query": "D-004" + } + ], + "false_merges": 0, + "false_negative": 0, + "false_positive": 0, + "missed_merges": 0, + "precision": 1.0, + "recall": 1.0, + "seed_jobs": 2, + "status": "passed", + "true_negative": 2, + "true_positive": 2 + }, + "hardware": { + "cpu_count": 20, + "debug_mode": true, + "platform": "Linux-6.12.54-Unraid-x86_64-with-glibc2.41", + "python": "3.13.14", + "release": "6.12.54-Unraid" + }, + "jobs_processed": 250, + "nfr_007": { + "measured": false, + "p95_ms": null, + "passed": false + }, + "parser": { + "cases": [ + { + "coverage_ratio": 1.0, + "covered_fields": 6, + "expected_fields": 6, + "expected_job_count": 1, + "expected_parser_key": "jsonld-jobposting", + "expected_parser_version": "1.0.0", + "extracted": 1, + "fixture": "fixtures/pages/sample_jsonld_job.html", + "id": "P-001", + "parser_key": "jsonld-jobposting", + "parser_version": "1.0.0", + "passed": true, + "warnings": [] + }, + { + "coverage_ratio": 1.0, + "covered_fields": 6, + "expected_fields": 6, + "expected_job_count": 1, + "expected_parser_key": "generic-html", + "expected_parser_version": "1.1.0", + "extracted": 1, + "fixture": "fixtures/pages/sample_generic_job.html", + "id": "P-002", + "parser_key": "generic-html", + "parser_version": "1.1.0", + "passed": true, + "warnings": [] + } + ], + "coverage_ratio": 1.0, + "covered_fields": 12, + "expected_fields": 12, + "status": "passed", + "unknown_data_ratio": 0.0 + }, + "performance": { + "avg_scores_detail_ms": 50.936, + "avg_scores_import_ms": 30.859, + "avg_scores_list_ms": 104.346, + "avg_scores_rescore_ms": 14.893, + "dashboard": { + "detail_p95_ms": 55.28, + "detail_query_count": 25, + "detail_query_types": { + "SELECT": 25 + }, + "list_p95_ms": 135.076, + "list_query_count": 76, + "list_query_types": { + "SELECT": 76 + }, + "timing_count": 50 + }, + "import": { + "duration_ms_total": 7714.791, + "p50_ms": 29.722, + "p95_ms": 39.37, + "query_count": 3264, + "query_types": { + "INSERT": 1003, + "RELEASE": 253, + "SAVEPOINT": 253, + "SELECT": 1256, + "UPDATE": 499 + }, + "timing_count": 250 + }, + "jobs": 1, + "performance": { + "memory_current_mb": 12.418, + "memory_peak_mb": 17.349, + "nfr_007": true, + "p50_ms": 30.306, + "p95_ms": 94.573, + "total_duration_ms": 11611.716 + }, + "query_count": 3370, + "rescore": { + "duration_ms_total": 14.893, + "p50_ms": 14.893, + "p95_ms": 14.893, + "query_count": 5, + "query_types": { + "INSERT": 1, + "SELECT": 4 + }, + "timing_count": 1 + }, + "source_run_id": 1, + "top_n": 25 + }, + "ranking": { + "churn_ratio": 0.0, + "post_top_n": [ + "8f49a32b-f425-425e-ac47-fbd1e1ab020f", + "8302cd44-569c-4109-83dd-87663d053200", + "fb63a999-5aa0-4912-8d06-400f852676cc", + "1476c46b-84ce-4151-b777-9dac8b533a8c", + "518e4b70-6742-43d2-bd39-90b3af026d89", + "9fa543b2-d5c9-47e1-8cba-ed1e1bfbfb86", + "5759f1c8-4974-4333-98c4-2ab0aec97918", + "77565fbe-27ea-43ee-bcb7-a218b073af9f", + "a9f67c53-f22d-4d72-be23-2ef95eb49443", + "ff866750-238b-4eb6-a9b7-7cacd8c6b9d0", + "8470bf3b-1760-4d4c-8755-387fcc5c92e8", + "293b9d4a-be83-4c72-b08e-02d6170ac405", + "d00b4e66-dfd1-45e8-af78-b643f59b5327", + "de6cbb4d-5225-4975-91f3-23b280492598", + "bb5bfca7-38d3-4639-83bd-da639ce2b4b5", + "d9a6079c-eb37-47e0-b588-28f87ebc7411", + "2d83e2f0-9a87-40a7-89cc-2f0689f52f18", + "e985ad1c-5bd1-4a29-b683-f1be109998ba", + "081bdfa8-0dfc-43d1-9b85-44cfe72892c1", + "79b5af36-6f76-4e35-af5c-ed3ca755b3ac", + "9c7d00f4-d307-42f3-b6d4-f95b2a6f7597", + "1053c44f-6f09-4a1e-b43f-f858fcb2441c", + "5f5005e9-6fe6-435a-992f-46440a803197", + "a9fe6c71-e1e1-423f-9811-5318c0b49688", + "f06e60ed-6d6d-44ef-b2f9-3ef57dc94cdf" + ], + "pre_top_n": [ + "8f49a32b-f425-425e-ac47-fbd1e1ab020f", + "8302cd44-569c-4109-83dd-87663d053200", + "fb63a999-5aa0-4912-8d06-400f852676cc", + "1476c46b-84ce-4151-b777-9dac8b533a8c", + "518e4b70-6742-43d2-bd39-90b3af026d89", + "9fa543b2-d5c9-47e1-8cba-ed1e1bfbfb86", + "5759f1c8-4974-4333-98c4-2ab0aec97918", + "77565fbe-27ea-43ee-bcb7-a218b073af9f", + "a9f67c53-f22d-4d72-be23-2ef95eb49443", + "ff866750-238b-4eb6-a9b7-7cacd8c6b9d0", + "8470bf3b-1760-4d4c-8755-387fcc5c92e8", + "293b9d4a-be83-4c72-b08e-02d6170ac405", + "d00b4e66-dfd1-45e8-af78-b643f59b5327", + "de6cbb4d-5225-4975-91f3-23b280492598", + "bb5bfca7-38d3-4639-83bd-da639ce2b4b5", + "d9a6079c-eb37-47e0-b588-28f87ebc7411", + "2d83e2f0-9a87-40a7-89cc-2f0689f52f18", + "e985ad1c-5bd1-4a29-b683-f1be109998ba", + "081bdfa8-0dfc-43d1-9b85-44cfe72892c1", + "79b5af36-6f76-4e35-af5c-ed3ca755b3ac", + "9c7d00f4-d307-42f3-b6d4-f95b2a6f7597", + "1053c44f-6f09-4a1e-b43f-f858fcb2441c", + "5f5005e9-6fe6-435a-992f-46440a803197", + "a9fe6c71-e1e1-423f-9811-5318c0b49688", + "f06e60ed-6d6d-44ef-b2f9-3ef57dc94cdf" + ], + "seed_jobs_count": 120, + "status": "passed", + "top_n": 25, + "unknown_data_ratio": 0.25, + "unknown_fields": { + "analysis_features": 0, + "description_text": 30, + "employment_types": 0, + "raw_location": 0 + } + }, + "status": "passed", + "timestamp_utc": "2026-07-29T16:36:43.297692+00:00", + "top_n": 25 + }, + { + "dataset_version": "vr116-2026-07-21", + "dedupe": { + "case_reports": [ + { + "decision": "exact_external_id", + "decision_similarity": 1.0, + "expected_match": "seed-infra", + "expected_match_present": true, + "id": "D-001", + "is_expected_match": true, + "matched": true, + "matched_job_id": "b4c42d9c-22c7-4e89-b88d-b8734e2d7e07", + "query": "D-001" + }, + { + "decision": "exact_external_id", + "decision_similarity": 1.0, + "expected_match": "seed-network", + "expected_match_present": true, + "id": "D-002", + "is_expected_match": true, + "matched": true, + "matched_job_id": "537f63ab-1ea0-45fc-b356-e734a33a99be", + "query": "D-002" + }, + { + "decision": "new", + "decision_similarity": 0.0, + "expected_match": null, + "expected_match_present": false, + "id": "D-003", + "is_expected_match": false, + "matched": false, + "matched_job_id": null, + "query": "D-003" + }, + { + "decision": "new", + "decision_similarity": 0.0, + "expected_match": null, + "expected_match_present": false, + "id": "D-004", + "is_expected_match": false, + "matched": false, + "matched_job_id": null, + "query": "D-004" + } + ], + "false_merges": 0, + "false_negative": 0, + "false_positive": 0, + "missed_merges": 0, + "precision": 1.0, + "recall": 1.0, + "seed_jobs": 2, + "status": "passed", + "true_negative": 2, + "true_positive": 2 + }, + "hardware": { + "cpu_count": 20, + "debug_mode": true, + "platform": "Linux-6.12.54-Unraid-x86_64-with-glibc2.41", + "python": "3.13.14", + "release": "6.12.54-Unraid" + }, + "jobs_processed": 250, + "nfr_007": { + "measured": false, + "p95_ms": null, + "passed": false + }, + "parser": { + "cases": [ + { + "coverage_ratio": 1.0, + "covered_fields": 6, + "expected_fields": 6, + "expected_job_count": 1, + "expected_parser_key": "jsonld-jobposting", + "expected_parser_version": "1.0.0", + "extracted": 1, + "fixture": "fixtures/pages/sample_jsonld_job.html", + "id": "P-001", + "parser_key": "jsonld-jobposting", + "parser_version": "1.0.0", + "passed": true, + "warnings": [] + }, + { + "coverage_ratio": 1.0, + "covered_fields": 6, + "expected_fields": 6, + "expected_job_count": 1, + "expected_parser_key": "generic-html", + "expected_parser_version": "1.1.0", + "extracted": 1, + "fixture": "fixtures/pages/sample_generic_job.html", + "id": "P-002", + "parser_key": "generic-html", + "parser_version": "1.1.0", + "passed": true, + "warnings": [] + } + ], + "coverage_ratio": 1.0, + "covered_fields": 12, + "expected_fields": 12, + "status": "passed", + "unknown_data_ratio": 0.0 + }, + "performance": { + "avg_scores_detail_ms": 55.155, + "avg_scores_import_ms": 40.941, + "avg_scores_list_ms": 101.206, + "avg_scores_rescore_ms": 16.1, + "dashboard": { + "detail_p95_ms": 60.679, + "detail_query_count": 25, + "detail_query_types": { + "SELECT": 25 + }, + "list_p95_ms": 126.661, + "list_query_count": 76, + "list_query_types": { + "SELECT": 76 + }, + "timing_count": 50 + }, + "import": { + "duration_ms_total": 10235.21, + "p50_ms": 33.657, + "p95_ms": 68.212, + "query_count": 3264, + "query_types": { + "INSERT": 1003, + "RELEASE": 253, + "SAVEPOINT": 253, + "SELECT": 1256, + "UPDATE": 499 + }, + "timing_count": 250 + }, + "jobs": 1, + "performance": { + "memory_current_mb": 4.49, + "memory_peak_mb": 22.389, + "nfr_007": true, + "p50_ms": 34.958, + "p95_ms": 96.727, + "total_duration_ms": 14160.325 + }, + "query_count": 3370, + "rescore": { + "duration_ms_total": 16.1, + "p50_ms": 16.1, + "p95_ms": 16.1, + "query_count": 5, + "query_types": { + "INSERT": 1, + "SELECT": 4 + }, + "timing_count": 1 + }, + "source_run_id": 1, + "top_n": 25 + }, + "ranking": { + "churn_ratio": 0.0, + "post_top_n": [ + "733b522b-382a-4a20-b046-3174b574a456", + "21473419-9a52-465d-90d0-da3dba7d87f4", + "e97f2b86-717e-4c9a-9d29-ba40f3198809", + "f9ee509b-fc89-4d81-8341-5b4ca48a51a1", + "f89072f9-b298-4751-b6d6-a3878e54ef18", + "f02c6911-d536-40d1-8d8d-3302392452e7", + "fb8f2233-2810-4e0e-9977-bebdcc230aef", + "8a71292f-bba7-4166-a7b8-714f9147f343", + "9cecf422-fd8e-4b40-9529-8e001af3df8b", + "80d26bff-3fae-4222-b6d8-25b47631a38b", + "d4475ebf-0969-4023-b6e2-e54df6dac7d8", + "0b33971f-c114-42a6-89d4-043055cce4d4", + "57c2ebf0-6a3c-4233-b1d4-f15bf3dfe087", + "bfe61184-136a-4f3f-8d12-4cee2109aa76", + "5bb9b9ea-1227-4bbf-b8e9-15de052ed584", + "9f361ba8-1e9e-4134-bda6-a7c78b6c4162", + "6b7b0dce-036d-475d-81f3-6ed8fcb98c5f", + "6c104672-11a2-4ee1-8200-f558d2d3eb0a", + "09b629be-5012-403f-b93f-b212d9f46b4c", + "46981d16-38d1-46a6-9755-c6d31b5adaf4", + "ae01edcc-fe42-4674-97ed-cdeec737ad9a", + "131a755e-4c6e-480c-81c4-6f2fe1afb069", + "7eba23c8-02ef-41a0-af54-d24689877f38", + "81cf6658-2035-4aba-97d9-4032085fba04", + "5a0b33ac-4b1e-454d-94b6-ddf5fc0945aa" + ], + "pre_top_n": [ + "733b522b-382a-4a20-b046-3174b574a456", + "21473419-9a52-465d-90d0-da3dba7d87f4", + "e97f2b86-717e-4c9a-9d29-ba40f3198809", + "f9ee509b-fc89-4d81-8341-5b4ca48a51a1", + "f89072f9-b298-4751-b6d6-a3878e54ef18", + "f02c6911-d536-40d1-8d8d-3302392452e7", + "fb8f2233-2810-4e0e-9977-bebdcc230aef", + "8a71292f-bba7-4166-a7b8-714f9147f343", + "9cecf422-fd8e-4b40-9529-8e001af3df8b", + "80d26bff-3fae-4222-b6d8-25b47631a38b", + "d4475ebf-0969-4023-b6e2-e54df6dac7d8", + "0b33971f-c114-42a6-89d4-043055cce4d4", + "57c2ebf0-6a3c-4233-b1d4-f15bf3dfe087", + "bfe61184-136a-4f3f-8d12-4cee2109aa76", + "5bb9b9ea-1227-4bbf-b8e9-15de052ed584", + "9f361ba8-1e9e-4134-bda6-a7c78b6c4162", + "6b7b0dce-036d-475d-81f3-6ed8fcb98c5f", + "6c104672-11a2-4ee1-8200-f558d2d3eb0a", + "09b629be-5012-403f-b93f-b212d9f46b4c", + "46981d16-38d1-46a6-9755-c6d31b5adaf4", + "ae01edcc-fe42-4674-97ed-cdeec737ad9a", + "131a755e-4c6e-480c-81c4-6f2fe1afb069", + "7eba23c8-02ef-41a0-af54-d24689877f38", + "81cf6658-2035-4aba-97d9-4032085fba04", + "5a0b33ac-4b1e-454d-94b6-ddf5fc0945aa" + ], + "seed_jobs_count": 120, + "status": "passed", + "top_n": 25, + "unknown_data_ratio": 0.25, + "unknown_fields": { + "analysis_features": 0, + "description_text": 30, + "employment_types": 0, + "raw_location": 0 + } + }, + "status": "passed", + "timestamp_utc": "2026-07-29T16:37:02.017739+00:00", + "top_n": 25 + }, + { + "dataset_version": "vr116-2026-07-21", + "dedupe": { + "case_reports": [ + { + "decision": "exact_external_id", + "decision_similarity": 1.0, + "expected_match": "seed-infra", + "expected_match_present": true, + "id": "D-001", + "is_expected_match": true, + "matched": true, + "matched_job_id": "bcb6a67c-1945-4006-9f01-b332822877ee", + "query": "D-001" + }, + { + "decision": "exact_external_id", + "decision_similarity": 1.0, + "expected_match": "seed-network", + "expected_match_present": true, + "id": "D-002", + "is_expected_match": true, + "matched": true, + "matched_job_id": "bc389f7a-f049-4daa-b01e-b6bab77c81c9", + "query": "D-002" + }, + { + "decision": "new", + "decision_similarity": 0.0, + "expected_match": null, + "expected_match_present": false, + "id": "D-003", + "is_expected_match": false, + "matched": false, + "matched_job_id": null, + "query": "D-003" + }, + { + "decision": "new", + "decision_similarity": 0.0, + "expected_match": null, + "expected_match_present": false, + "id": "D-004", + "is_expected_match": false, + "matched": false, + "matched_job_id": null, + "query": "D-004" + } + ], + "false_merges": 0, + "false_negative": 0, + "false_positive": 0, + "missed_merges": 0, + "precision": 1.0, + "recall": 1.0, + "seed_jobs": 2, + "status": "passed", + "true_negative": 2, + "true_positive": 2 + }, + "hardware": { + "cpu_count": 20, + "debug_mode": true, + "platform": "Linux-6.12.54-Unraid-x86_64-with-glibc2.41", + "python": "3.13.14", + "release": "6.12.54-Unraid" + }, + "jobs_processed": 250, + "nfr_007": { + "measured": false, + "p95_ms": null, + "passed": false + }, + "parser": { + "cases": [ + { + "coverage_ratio": 1.0, + "covered_fields": 6, + "expected_fields": 6, + "expected_job_count": 1, + "expected_parser_key": "jsonld-jobposting", + "expected_parser_version": "1.0.0", + "extracted": 1, + "fixture": "fixtures/pages/sample_jsonld_job.html", + "id": "P-001", + "parser_key": "jsonld-jobposting", + "parser_version": "1.0.0", + "passed": true, + "warnings": [] + }, + { + "coverage_ratio": 1.0, + "covered_fields": 6, + "expected_fields": 6, + "expected_job_count": 1, + "expected_parser_key": "generic-html", + "expected_parser_version": "1.1.0", + "extracted": 1, + "fixture": "fixtures/pages/sample_generic_job.html", + "id": "P-002", + "parser_key": "generic-html", + "parser_version": "1.1.0", + "passed": true, + "warnings": [] + } + ], + "coverage_ratio": 1.0, + "covered_fields": 12, + "expected_fields": 12, + "status": "passed", + "unknown_data_ratio": 0.0 + }, + "performance": { + "avg_scores_detail_ms": 50.504, + "avg_scores_import_ms": 36.681, + "avg_scores_list_ms": 102.301, + "avg_scores_rescore_ms": 19.548, + "dashboard": { + "detail_p95_ms": 54.227, + "detail_query_count": 25, + "detail_query_types": { + "SELECT": 25 + }, + "list_p95_ms": 129.536, + "list_query_count": 76, + "list_query_types": { + "SELECT": 76 + }, + "timing_count": 50 + }, + "import": { + "duration_ms_total": 9170.368, + "p50_ms": 35.32, + "p95_ms": 49.306, + "query_count": 3264, + "query_types": { + "INSERT": 1003, + "RELEASE": 253, + "SAVEPOINT": 253, + "SELECT": 1256, + "UPDATE": 499 + }, + "timing_count": 250 + }, + "jobs": 1, + "performance": { + "memory_current_mb": 12.332, + "memory_peak_mb": 16.658, + "nfr_007": true, + "p50_ms": 36.098, + "p95_ms": 93.787, + "total_duration_ms": 13010.048 + }, + "query_count": 3370, + "rescore": { + "duration_ms_total": 19.548, + "p50_ms": 19.548, + "p95_ms": 19.548, + "query_count": 5, + "query_types": { + "INSERT": 1, + "SELECT": 4 + }, + "timing_count": 1 + }, + "source_run_id": 1, + "top_n": 25 + }, + "ranking": { + "churn_ratio": 0.0, + "post_top_n": [ + "6758530f-71d8-4e3d-9732-7e1001eefd13", + "94176301-843f-4074-abdf-fd2c76b76d36", + "492a021b-a00d-47ff-b236-86ef6a88a2a6", + "c6227dbc-82c4-4c4b-8aec-99714d5adc53", + "523a1940-cabc-48b3-abab-bc74f18d4ee4", + "ca18c4cc-b796-4ebf-9591-c335bc072f57", + "c6a495ac-564a-4aee-a509-081100ddf0a8", + "0cf16e0f-5ffc-455b-b488-deb19b0e0fea", + "b906c635-a818-4db7-b639-dc6e745ef345", + "edce515a-e697-4925-b73f-5605b1236727", + "1f800e1f-ff44-4548-91c7-0e9f97af40e2", + "0d3caa61-dab1-49d1-8af3-21cb605e9b41", + "e6f0d2ea-6dfc-4559-8aa9-80799402e8f1", + "a5d682ee-6fac-4ae9-921d-7e883db86d49", + "543f63fe-f7fd-4302-9414-95536f5fabcc", + "b97df085-3019-45cb-aee6-66edb6602df9", + "733c02de-73dc-4967-9e34-c62dcdd98e4f", + "f0178164-2535-4e02-91c7-7a6920ab6df3", + "4bb41af9-5387-4d0c-a62b-37f7ff713dfe", + "ea01b4ee-5a92-4df2-8896-bd391b5c661b", + "f08116ab-fa5d-45e7-a579-d9c2cad0d63f", + "3c4feccb-a4d1-4cd1-b814-1a2d8bfd9da1", + "69088d9a-0de5-4bbe-b8bf-abd733838ed6", + "e567d05d-8ed4-4f0f-80fb-1e6f0f77f377", + "9f03de2f-4c1e-4cc5-9d65-bbcd3b42a029" + ], + "pre_top_n": [ + "6758530f-71d8-4e3d-9732-7e1001eefd13", + "94176301-843f-4074-abdf-fd2c76b76d36", + "492a021b-a00d-47ff-b236-86ef6a88a2a6", + "c6227dbc-82c4-4c4b-8aec-99714d5adc53", + "523a1940-cabc-48b3-abab-bc74f18d4ee4", + "ca18c4cc-b796-4ebf-9591-c335bc072f57", + "c6a495ac-564a-4aee-a509-081100ddf0a8", + "0cf16e0f-5ffc-455b-b488-deb19b0e0fea", + "b906c635-a818-4db7-b639-dc6e745ef345", + "edce515a-e697-4925-b73f-5605b1236727", + "1f800e1f-ff44-4548-91c7-0e9f97af40e2", + "0d3caa61-dab1-49d1-8af3-21cb605e9b41", + "e6f0d2ea-6dfc-4559-8aa9-80799402e8f1", + "a5d682ee-6fac-4ae9-921d-7e883db86d49", + "543f63fe-f7fd-4302-9414-95536f5fabcc", + "b97df085-3019-45cb-aee6-66edb6602df9", + "733c02de-73dc-4967-9e34-c62dcdd98e4f", + "f0178164-2535-4e02-91c7-7a6920ab6df3", + "4bb41af9-5387-4d0c-a62b-37f7ff713dfe", + "ea01b4ee-5a92-4df2-8896-bd391b5c661b", + "f08116ab-fa5d-45e7-a579-d9c2cad0d63f", + "3c4feccb-a4d1-4cd1-b814-1a2d8bfd9da1", + "69088d9a-0de5-4bbe-b8bf-abd733838ed6", + "e567d05d-8ed4-4f0f-80fb-1e6f0f77f377", + "9f03de2f-4c1e-4cc5-9d65-bbcd3b42a029" + ], + "seed_jobs_count": 120, + "status": "passed", + "top_n": 25, + "unknown_data_ratio": 0.25, + "unknown_fields": { + "analysis_features": 0, + "description_text": 30, + "employment_types": 0, + "raw_location": 0 + } + }, + "status": "passed", + "timestamp_utc": "2026-07-29T16:37:19.113439+00:00", + "top_n": 25 + } + ], + "schema_version": "1.0", + "status": "passed", + "summary": { + "duration_ms_mean": 9040.123, + "memory_peak_mb_mean": 18.799, + "p50_ms_mean": 32.9, + "p95_ms_mean": 52.296, + "query_count_mean": 3264, + "query_count_total_mean": 3370, + "throughput_per_second": 28.031 + }, + "warmup_status": "passed" +} diff --git a/config/settings.py b/config/settings.py index ef4f83a..425a010 100644 --- a/config/settings.py +++ b/config/settings.py @@ -139,7 +139,7 @@ CSRF_TRUSTED_ORIGINS = merge_unique( ) TIME_ZONE = os.getenv("DJANGO_TIME_ZONE", "Europe/Brussels") VACATURERADAR_OWNER_NAME = os.getenv("VACATURERADAR_OWNER_NAME", "Jens").strip()[:40] or "Jens" -VACATURERADAR_VERSION = os.getenv("VACATURERADAR_VERSION", "0.3.13").strip() or "0.3.13" +VACATURERADAR_VERSION = os.getenv("VACATURERADAR_VERSION", "0.3.14").strip() or "0.3.14" LANGUAGE_CODE = "nl-be" USE_I18N = True USE_TZ = True diff --git a/deployment/production.env.example b/deployment/production.env.example index 8bb0abf..f5a4166 100644 --- a/deployment/production.env.example +++ b/deployment/production.env.example @@ -9,7 +9,7 @@ DJANGO_ALLOWED_HOSTS=vacatureradar.example.be,127.0.0.1,localhost DJANGO_CSRF_TRUSTED_ORIGINS=https://vacatureradar.example.be DJANGO_TIME_ZONE=Europe/Brussels VACATURERADAR_OWNER_NAME=Jens -VACATURERADAR_VERSION=0.3.13 +VACATURERADAR_VERSION=0.3.14 POSTGRES_DB=vacatureradar POSTGRES_USER=vacatureradar diff --git a/docs/ai/BACKLOG.yaml b/docs/ai/BACKLOG.yaml index dba471a..6d84b8b 100644 --- a/docs/ai/BACKLOG.yaml +++ b/docs/ai/BACKLOG.yaml @@ -2329,3 +2329,33 @@ tasks: note: Release 0.3.13 professionaliseert responsive layout, mobiele gastflow, begrensde vacature-UX, verplichte zes-viewport-Playwrightgate zonder skips, echte runtimeheartbeat, parserbenchmark, authdekking, SBOM en integrale documentatie; volledige gate 312 tests groen, 84,16% dekking. +- id: VR-228 + title: Optimaliseer de importpipeline voor release 0.3.14 + status: ready + priority: P0 + requirement_ids: + - NFR-007 + - NFR-008 + - NFR-009 + depends_on: + - VR-227 + summary: Profileer de PostgreSQL-import reproduceerbaar, verwijder aantoonbare N+1- en overbodige writes en lever + de geverifieerde optimalisatie veilig op zonder wijziging aan deduplicatie, scoring of datakwaliteit. + acceptance_criteria: + - Een fail-closed runner meet een disposable PostgreSQL-database met vaste fixture, warm-up en drie herhalingen. + - De geoptimaliseerde import reduceert queryvolume en doorlooptijd aantoonbaar tegenover 0.3.13 en rapporteert + p50, p95, throughput, geheugen en queryfamilies. + - Parserdekking blijft 12/12, dedupeprecision en -recall blijven 1,0, rankingchurn blijft nul en historische + ScoreRuns plus veldprovenance blijven intact. + - Regressietests begrenzen het queryvolume; de volledige kwaliteitsgate slaagt tweemaal en productieacceptatie + bewijst health, beperkte importsmoke, publieke kernflows en rollbackgereedheid. + verification: + - python -m scripts.benchmark_import --jobs 250 --iterations 3 + - uv run pytest tests/integration/test_pipeline.py tests/unit/test_import_benchmark.py + - ./scripts/codex_verify.sh + primary_paths: + - apps/jobs/services/pipeline.py + - scripts/benchmark.py + - scripts/benchmark_import.py + - tests/integration/test_pipeline.py + - docs/audit/IMPORT_PERFORMANCE_FINAL.md diff --git a/docs/audit/IMPORT_PERFORMANCE_BASELINE.md b/docs/audit/IMPORT_PERFORMANCE_BASELINE.md new file mode 100644 index 0000000..205d78f --- /dev/null +++ b/docs/audit/IMPORT_PERFORMANCE_BASELINE.md @@ -0,0 +1,14 @@ +# Importperformancebaseline 0.3.13 + +Gemeten op 2026-07-29 in een disposable PostgreSQL-database op Unraid (Linux 6.12.54, Python 3.13.14, 20 CPU's). De vaste fixture bevat 250 geldige HTML-documenten met dezelfde externe vacature-identiteit: één create en 249 idempotente duplicate-updates. Concurrency is bewust 1; netwerk, Celery, Ollama en mail zijn uit de meting gehouden. + +| Metriek | 0.3.13 | +|---|---:| +| Importduur | 16.358,158 ms | +| p50 per document | 63,034 ms | +| p95 per document | 79,871 ms | +| Throughput | 15,283 documenten/s | +| Importqueries | >9.000 (loggerlimiet bereikt) | +| Piekgeheugen | 26,582 MB | + +Correctheidsankers: parserdekking 12/12, dedupeprecision 1,0, deduperecall 1,0 en rankingchurn 0. De overschrijding van 9.000 queries is als ondergrens gerapporteerd, niet als exact getal. diff --git a/docs/audit/IMPORT_PERFORMANCE_FINAL.md b/docs/audit/IMPORT_PERFORMANCE_FINAL.md new file mode 100644 index 0000000..3116f9f --- /dev/null +++ b/docs/audit/IMPORT_PERFORMANCE_FINAL.md @@ -0,0 +1,14 @@ +# Finale importperformancemeting 0.3.14 + +De officiële runner gebruikt een database met verplichte prefix `vacatureradar_bench_`, weigert SQLite, externe databasehosts en publieke productiedomeinen, migreert en flusht de disposable database, voert één warm-up uit en meet daarna drie iteraties van 250 documenten. Het machineleesbare bewijs staat in `artifacts/import-performance-report.json`. + +| Metriek | 0.3.13 | 0.3.14 gemiddeld | Verbetering | +|---|---:|---:|---:| +| Importduur | 16.358,158 ms | 9.040,123 ms | 44,74% lager | +| p50 | 63,034 ms | 32,900 ms | 47,80% lager | +| p95 | 79,871 ms | 52,296 ms | 34,53% lager | +| Throughput | 15,283/s | 28,031/s | 83,42% hoger | +| Importqueries | >9.000 | 3.264 | minstens 63,73% lager | +| Piekgeheugen | 26,582 MB | 18,799 MB | 29,28% lager | + +Alle drie iteraties zijn gelijk voor queryvolume en queryfamilies. De kwaliteitsankers bleven parser 12/12, precision 1,0, recall 1,0 en rankingchurn 0. De fixture resulteert per iteratie in één vacature, één versie, behouden veldprovenance, 250 historische ScoreRuns en 249 idempotente duplicates. diff --git a/docs/audit/IMPORT_PERFORMANCE_TRACEABILITY.md b/docs/audit/IMPORT_PERFORMANCE_TRACEABILITY.md new file mode 100644 index 0000000..e6046fe --- /dev/null +++ b/docs/audit/IMPORT_PERFORMANCE_TRACEABILITY.md @@ -0,0 +1,10 @@ +# Importperformancetraceability + +| Acceptatie | Implementatie | Bewijs | +|---|---|---| +| Veilige isolatie | `scripts/benchmark_import.py`, `scripts/run_import_benchmark.sh` | negatieve identitytests en disposable PostgreSQL-run | +| Meetbare reductie | run-local caches, gebundelde provenance en score-snapshotcopy | baseline/finaal rapport en machine-JSON | +| Geen functionele regressie | bestaande dedupe/scoringregels ongewijzigd | pipeline-, dedupe- en benchmarktests | +| Historiek en provenance | scorecopy per import, evidence bulk sync | integratietest met 20 herhaalde documenten | +| Queryregressie voorkomen | execute-wrapper en budgettest | stabiel 3.264 queries per 250; minder dan 300 per 20 in test | +| Releasekwaliteit | VR-228, gates, acceptance en operationsdocs | `docs/ai/PROJECT_STATE.md` en `FINAL_ACCEPTANCE.md` | diff --git a/docs/audit/IMPORT_QUERY_ANALYSIS.md b/docs/audit/IMPORT_QUERY_ANALYSIS.md new file mode 100644 index 0000000..50e892a --- /dev/null +++ b/docs/audit/IMPORT_QUERY_ANALYSIS.md @@ -0,0 +1,12 @@ +# Importqueryanalyse 0.3.14 + +De hot path liep van `process_raw_document` via parsing en `persist_draft` naar employer-resolutie, deduplicatie, alias/provenance, versiebeheer en scoring. Profiling wees vier dominante oorzaken aan: + +1. Dezelfde werkgever en actieve profielen werden voor elk document opnieuw opgehaald. +2. Veldprovenance gebruikte per veld een afzonderlijke `update_or_create`. +3. Een inhoudelijk ongewijzigde waarneming schreef opnieuw een versie en berekende alle scores volledig. +4. `last_changed` verschoof ten onrechte bij alleen een nieuwe `last_seen`-waarneming. + +De oplossing blijft binnen de modulaire monoliet en transacties: een import-run krijgt een lokale `PersistenceContext`, provenance wordt per alias gelezen en gebundeld geschreven, versies ontstaan alleen bij inhoudelijke wijziging en een ongewijzigde job kopieert het laatste ongewijzigde scoresnapshot. Daardoor blijft iedere `ScoreRun` historisch aanwezig, terwijl deterministische scoring niet opnieuw wordt uitgevoerd. De cache leeft nooit buiten één seriële import-run en verandert workerisolatie of deduplicatieregels niet. + +De database-executieteller rapporteert per 250 items exact: 1.256 SELECT, 1.003 INSERT, 499 UPDATE, 253 SAVEPOINT en 253 RELEASE; totaal 3.264 importexecuties. De drie officiële iteraties geven identieke queryfamilies. diff --git a/docs/audit/PERFORMANCE.md b/docs/audit/PERFORMANCE.md index d0612ba..4d16577 100644 --- a/docs/audit/PERFORMANCE.md +++ b/docs/audit/PERFORMANCE.md @@ -1,5 +1,9 @@ # Performanceaudit 2026-07-29 +## Importoptimalisatie 0.3.14 + +De eerder geregistreerde importhotspot is opgelost en reproduceerbaar gemeten in een disposable PostgreSQL-database. Tegenover de 0.3.13-baseline daalt de gemiddelde 250-itemimport van 16.358,158 naar 9.040,123 ms; p95 daalt van 79,871 naar 52,296 ms en queryvolume van meer dan 9.000 naar exact 3.264. Zie `IMPORT_PERFORMANCE_BASELINE.md`, `IMPORT_QUERY_ANALYSIS.md` en `IMPORT_PERFORMANCE_FINAL.md`. + ## Baseline - Lokale quickbenchmark: 250 invoeritems op Windows 11, Python 3.13, 16 CPU-threads. diff --git a/docs/audit/TRACEABILITY.md b/docs/audit/TRACEABILITY.md index fa532e1..2f8a8cd 100644 --- a/docs/audit/TRACEABILITY.md +++ b/docs/audit/TRACEABILITY.md @@ -16,3 +16,4 @@ | Back-up/restore | custom dump, media en runbook | geïsoleerde restore, exacte tellingen en aparte applicatiestart | implemented and live verified | | Rollback | immutable vorige en huidige imagetags | oude image healthy, daarna 0.3.13 opnieuw healthy | implemented and live verified | | Externe identity/mail | versleutelde configuratieboundary | `USER_INPUT_REQUIRED.md` | optional external action | +| Importperformance 0.3.14 | run-lokale caches, evidence bulk sync en scorecopy | geïsoleerde 3-run PostgreSQL-meting en regressietests | implemented and verified | diff --git a/docs/quality/TRACEABILITY_MATRIX.md b/docs/quality/TRACEABILITY_MATRIX.md index bfa8b49..e876786 100644 --- a/docs/quality/TRACEABILITY_MATRIX.md +++ b/docs/quality/TRACEABILITY_MATRIX.md @@ -84,6 +84,7 @@ Legenda: **I** = geïmplementeerd en getest in de huidige MVP; **P** = gedeeltel Iedere backlogtaak die een requirement wijzigt, werkt in dezelfde commit deze matrix, relevante acceptatiecriteria en tests bij. Een status wordt alleen naar **I** gezet wanneer de code én het genoemde bewijs bestaan. ## Aanvullende traceability-opmerking +- VR-228 levert release 0.3.14 met een fail-closed disposable PostgreSQL-benchmark, run-lokale importcaches, gebundelde veldprovenance en ongewijzigde score-snapshotcopy. De officiële drie iteraties bewijzen 3.264 queries per 250 items, parser 12/12, dedupeprecision/recall 1,0 en nul rankingchurn; bewijs staat in `docs/audit/IMPORT_PERFORMANCE_*.md` en het machineleesbare artifact. - VR-227 professionaliseert release 0.3.13 integraal: gecentreerd ultrawideframe, mobiele gastbanner en verticale pipeline, begrensde vacaturepaginering, verplichte zes-viewport-Playwrightgate zonder skips, echte runtimeheartbeat, volledige generieke parserbenchmark en SBOM-generatie. Bewijs staat in `docs/audit/`, de E2E-/auth-/runtime-/adaptertests en `USER_INPUT_REQUIRED.md`. - VR-102 is gerealiseerd met auditbare policyreviews (`SourcePolicyReview`), robotscache met TTL/size/SSRF-controles en fail-closed policy-gating op verlopen of conflicterende reviews. - VR-107 is gerealiseerd met `apps/sources/views.py`, `apps/sources/services/manual_import.py`, `templates/sources/list.html`, `static/js/manual_import.js` en bijhorende unit/integration-tests. diff --git a/docs/releases/0.3.14.md b/docs/releases/0.3.14.md new file mode 100644 index 0000000..cd07842 --- /dev/null +++ b/docs/releases/0.3.14.md @@ -0,0 +1,5 @@ +# VacatureRadar 0.3.14 + +Deze release versnelt herhaalde vacature-imports zonder deduplicatie, scoring of gebruikersdata te veranderen. Een geïsoleerde PostgreSQL-benchmark bewijst 44,74% kortere importduur, minstens 63,73% minder databasequeries en 83,42% hogere throughput tegenover 0.3.13. De runner, regressiebudgetten en queryfamilierapportage maken deze winst reproduceerbaar. + +Upgrade vereist geen schemawijziging. Voer zoals altijd `migrate --noinput` uit; rollback kan naar de bewaarde immutable 0.3.13-image zonder dataconversie. diff --git a/pyproject.toml b/pyproject.toml index 5cbf5c0..d17295d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vacatureradar" -version = "0.3.13" +version = "0.3.14" description = "Persoonlijke autonome vacature-assistent voor toegestane publieke bronnen en vacaturemails." readme = "README.md" requires-python = ">=3.12,<3.14" diff --git a/scripts/benchmark.py b/scripts/benchmark.py index cabad70..d4fedd4 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -20,7 +20,6 @@ import django from django.conf import settings from django.contrib.auth import get_user_model from django.db import connection -from django.test.utils import CaptureQueriesContext from django.utils import timezone os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") @@ -29,7 +28,7 @@ django.setup() from apps.jobs.models import Employer, JobPosting, JobSourceAlias, ScoreRun # noqa: E402 from apps.jobs.services.dedupe import find_existing_job # noqa: E402 from apps.jobs.services.normalization import CanonicalJobDraft, normalize_token # noqa: E402 -from apps.jobs.services.pipeline import process_raw_document # noqa: E402 +from apps.jobs.services.pipeline import PersistenceContext, process_raw_document # noqa: E402 from apps.jobs.services.scoring import calculate_score, rescore_jobs_with_profiles # noqa: E402 from apps.profiles.models import SearchProfile # noqa: E402 from apps.sources.adapters.registry import registry # noqa: E402 @@ -575,15 +574,25 @@ def _clean_benchmark(artifacts: BenchmarkArtifact) -> None: get_user_model().objects.filter(pk=artifacts.user_id).delete() +class QueryCounter: + """Count database executions without Django's bounded debug-query log.""" + + def __init__(self) -> None: + self.count = 0 + self.types: dict[str, int] = {} + + def __call__(self, execute: Any, sql: str, params: Any, many: bool, context: Any) -> Any: + self.count += 1 + query_type = sql.lstrip().split(None, 1)[0].upper() if sql.strip() else "UNKNOWN" + self.types[query_type] = self.types.get(query_type, 0) + 1 + return execute(sql, params, many, context) + + @contextmanager def _query_capture() -> Any: - original_force_debug_cursor = connection.force_debug_cursor - connection.force_debug_cursor = True - try: - with CaptureQueriesContext(connection) as captured: - yield captured - finally: - connection.force_debug_cursor = original_force_debug_cursor + counter = QueryCounter() + with connection.execute_wrapper(counter): + yield counter def run_performance_benchmark( @@ -619,6 +628,7 @@ def run_performance_benchmark( } tracemalloc.start() import_timings: list[float] = [] + persistence_context = PersistenceContext() rescore_timings: list[float] = [] list_timings: list[float] = [] detail_timings: list[float] = [] @@ -646,9 +656,10 @@ def run_performance_benchmark( retain_until=timezone.now() + timezone.timedelta(days=7), ) start = perf_counter() - process_raw_document(document) + process_raw_document(document, context=persistence_context) import_timings.append((perf_counter() - start) * 1000) - import_query_count = len(captured_import.captured_queries) + import_query_count = captured_import.count + import_query_types = captured_import.types query_count_total += import_query_count candidates = list( @@ -666,7 +677,8 @@ def run_performance_benchmark( batch_start = perf_counter() rescore_jobs_with_profiles(batch, profile_id=profile.id) rescore_timings.append((perf_counter() - batch_start) * 1000) - rescore_query_count = len(captured_rescore.captured_queries) + rescore_query_count = captured_rescore.count + rescore_query_types = captured_rescore.types query_count_total += rescore_query_count with _query_capture() as captured_list: @@ -695,8 +707,10 @@ def run_performance_benchmark( ) detail_timings.append((perf_counter() - start) * 1000) - list_query_count = len(captured_list.captured_queries) - detail_query_count = len(captured_detail.captured_queries) + list_query_count = captured_list.count + detail_query_count = captured_detail.count + list_query_types = captured_list.types + detail_query_types = captured_detail.types query_count_total += list_query_count + detail_query_count current_bytes, peak_bytes = tracemalloc.get_traced_memory() @@ -740,6 +754,7 @@ def run_performance_benchmark( "p50_ms": timing_stats["import_p50_ms"], "p95_ms": timing_stats["import_p95_ms"], "query_count": import_query_count, + "query_types": import_query_types, "timing_count": len(import_timings), }, "rescore": { @@ -747,11 +762,14 @@ def run_performance_benchmark( "p50_ms": timing_stats["rescore_p50_ms"], "p95_ms": timing_stats["rescore_p95_ms"], "query_count": rescore_query_count, + "query_types": rescore_query_types, "timing_count": len(rescore_timings), }, "dashboard": { "list_query_count": list_query_count, "detail_query_count": detail_query_count, + "list_query_types": list_query_types, + "detail_query_types": detail_query_types, "list_p95_ms": timing_stats["list_p95_ms"], "detail_p95_ms": timing_stats["detail_p95_ms"], "timing_count": len(list_timings) + len(detail_timings), diff --git a/scripts/benchmark_import.py b/scripts/benchmark_import.py new file mode 100644 index 0000000..5ee4123 --- /dev/null +++ b/scripts/benchmark_import.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from statistics import mean +from typing import Any +from urllib.parse import urlparse + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") + +import django + +django.setup() + +from django.conf import settings # noqa: E402 +from django.core.management import call_command # noqa: E402 + +from scripts.benchmark import DEFAULT_DATASET, run_benchmark_report # noqa: E402 + +SAFE_DATABASE_PREFIX = "vacatureradar_bench_" +SAFE_PUBLIC_HOSTS = {"", "benchmark.invalid", "localhost", "127.0.0.1"} + + +class UnsafeBenchmarkEnvironment(RuntimeError): + pass + + +def validate_benchmark_identity(database: dict[str, Any], public_base_url: str) -> None: + engine = str(database.get("ENGINE", "")) + name = str(database.get("NAME", "")) + host = str(database.get("HOST", "")) + public_host = (urlparse(public_base_url).hostname or "").casefold() + if engine != "django.db.backends.postgresql": + raise UnsafeBenchmarkEnvironment("De importbenchmark vereist PostgreSQL.") + if not name.startswith(SAFE_DATABASE_PREFIX): + raise UnsafeBenchmarkEnvironment( + f"Onveilige databasenaam {name!r}; verwacht prefix {SAFE_DATABASE_PREFIX!r}." + ) + if host not in {"127.0.0.1", "localhost", "postgres"}: + raise UnsafeBenchmarkEnvironment(f"Onveilige benchmarkdatabasehost {host!r}.") + if public_host not in SAFE_PUBLIC_HOSTS: + raise UnsafeBenchmarkEnvironment( + f"Publieke host {public_host!r} is niet toegestaan voor een benchmark." + ) + + +def _summary(reports: list[dict[str, Any]]) -> dict[str, float]: + performance = [report["performance"] for report in reports] + imports = [item["import"] for item in performance] + return { + "query_count_mean": round(mean(item["query_count"] for item in imports), 3), + "query_count_total_mean": round(mean(item["query_count"] for item in performance), 3), + "duration_ms_mean": round(mean(item["duration_ms_total"] for item in imports), 3), + "p50_ms_mean": round(mean(item["p50_ms"] for item in imports), 3), + "p95_ms_mean": round(mean(item["p95_ms"] for item in imports), 3), + "throughput_per_second": round( + mean( + report["jobs_processed"] / (item["duration_ms_total"] / 1000) + for report, item in zip(reports, imports, strict=True) + ), + 3, + ), + "memory_peak_mb_mean": round( + mean(item["performance"]["memory_peak_mb"] for item in performance), 3 + ), + } + + +def run_isolated_benchmark( + *, dataset: Path, jobs: int, iterations: int, output: Path +) -> dict[str, Any]: + validate_benchmark_identity(settings.DATABASES["default"], settings.PUBLIC_BASE_URL) + if iterations < 1: + raise ValueError("iterations moet minstens 1 zijn.") + call_command("migrate", interactive=False, verbosity=0) + call_command("flush", interactive=False, verbosity=0) + warmup = run_benchmark_report(dataset, quick=True, jobs=jobs) + measured: list[dict[str, Any]] = [] + for _ in range(iterations): + call_command("flush", interactive=False, verbosity=0) + measured.append(run_benchmark_report(dataset, quick=True, jobs=jobs)) + report = { + "schema_version": "1.0", + "database_identity": { + "engine": settings.DATABASES["default"]["ENGINE"], + "name_prefix": SAFE_DATABASE_PREFIX, + "host": settings.DATABASES["default"]["HOST"], + }, + "configuration": { + "jobs": jobs, + "iterations": iterations, + "warmup_runs": 1, + "concurrency": 1, + "dataset": str(dataset), + }, + "warmup_status": warmup["status"], + "summary": _summary(measured), + "iterations": measured, + "status": "passed" if all(item["status"] == "passed" for item in measured) else "failed", + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description="Fail-closed geïsoleerde importbenchmark") + parser.add_argument("--dataset", type=Path, default=DEFAULT_DATASET) + parser.add_argument("--jobs", type=int, default=250) + parser.add_argument("--iterations", type=int, default=3) + parser.add_argument( + "--output", type=Path, default=Path("artifacts/import-performance-report.json") + ) + args = parser.parse_args() + report = run_isolated_benchmark( + dataset=args.dataset, jobs=args.jobs, iterations=args.iterations, output=args.output + ) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["status"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_import_benchmark.sh b/scripts/run_import_benchmark.sh new file mode 100644 index 0000000..d0e5c2a --- /dev/null +++ b/scripts/run_import_benchmark.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." + +: "${DATABASE_URL:?Stel DATABASE_URL in op een afzonderlijke vacatureradar_bench_* PostgreSQL-database.}" +export PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-https://benchmark.invalid}" +export DJANGO_DEBUG=0 +export DJANGO_SECRET_KEY="${DJANGO_SECRET_KEY:-benchmark-only-secret-key-not-for-production-000000000000000000}" +export CELERY_TASK_ALWAYS_EAGER=1 +export OLLAMA_ENABLED=0 +export EMAIL_BACKEND=django.core.mail.backends.locmem.EmailBackend + +if command -v uv >/dev/null 2>&1; then + exec uv run python -m scripts.benchmark_import "$@" +fi +exec .venv/bin/python -m scripts.benchmark_import "$@" diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 9e9e9f5..9721fc8 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -3,10 +3,19 @@ from datetime import timedelta from pathlib import Path import pytest +from django.db import connection +from django.test.utils import CaptureQueriesContext from django.utils import timezone -from apps.jobs.models import Employer, JobPosting, JobSourceAlias, ScoreRun -from apps.jobs.services.pipeline import process_raw_document +from apps.jobs.models import ( + Employer, + FieldProvenance, + JobPosting, + JobSourceAlias, + JobVersion, + ScoreRun, +) +from apps.jobs.services.pipeline import PersistenceContext, process_raw_document from apps.sources.models import RawDocument, Source @@ -34,6 +43,39 @@ def test_pipeline_is_idempotent_and_scores(source, profile): assert ScoreRun.objects.filter(profile=profile).count() == 2 +@pytest.mark.integration +@pytest.mark.django_db +def test_unchanged_import_batch_has_bounded_lookup_queries(source, profile): + content = Path("fixtures/pages/sample_jsonld_job.html").read_text(encoding="utf-8") + documents = [ + RawDocument.objects.create( + source=source, + url=f"https://jobs.example.org/import/{index}", + final_url=f"https://jobs.example.org/import/{index}", + kind=RawDocument.Kind.HTML, + content_type="text/html", + content_hash=hashlib.sha256(f"{index}{content}".encode()).hexdigest(), + body_text=content, + byte_length=len(content.encode()), + retain_until=timezone.now() + timedelta(days=7), + ) + for index in range(20) + ] + context = PersistenceContext() + + with CaptureQueriesContext(connection) as queries: + results = [process_raw_document(document, context=context) for document in documents] + + assert sum(int(result["created"]) for result in results) == 1 + assert sum(int(result["duplicates"]) for result in results) == 19 + assert len(queries) < 300 + assert Employer.objects.count() == 1 + assert JobPosting.objects.count() == 1 + assert JobVersion.objects.count() == 1 + assert FieldProvenance.objects.count() > 0 + assert ScoreRun.objects.filter(profile=profile).count() == 20 + + @pytest.mark.django_db def test_pipeline_uses_reviewed_employer_identity_for_public_ats_feed(profile): source = Source.objects.create( diff --git a/tests/unit/test_import_benchmark.py b/tests/unit/test_import_benchmark.py new file mode 100644 index 0000000..5158055 --- /dev/null +++ b/tests/unit/test_import_benchmark.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import pytest + +from scripts.benchmark_import import UnsafeBenchmarkEnvironment, validate_benchmark_identity + + +def _database(name: str, host: str = "127.0.0.1") -> dict[str, str]: + return { + "ENGINE": "django.db.backends.postgresql", + "NAME": name, + "HOST": host, + } + + +def test_benchmark_identity_accepts_only_isolated_postgres() -> None: + validate_benchmark_identity( + _database("vacatureradar_bench_release_0314"), "https://benchmark.invalid" + ) + + +@pytest.mark.parametrize( + ("database", "public_url"), + [ + (_database("vacatureradar"), "https://benchmark.invalid"), + ( + _database("vacatureradar_bench_release", "database.internal"), + "https://benchmark.invalid", + ), + (_database("vacatureradar_bench_release"), "https://vacatureradar.itworx.tech"), + ( + {"ENGINE": "django.db.backends.sqlite3", "NAME": Path("bench.sqlite3"), "HOST": ""}, + "https://benchmark.invalid", + ), + ], +) +def test_benchmark_identity_rejects_unsafe_targets( + database: dict[str, object], public_url: str +) -> None: + with pytest.raises(UnsafeBenchmarkEnvironment): + validate_benchmark_identity(database, public_url) diff --git a/uv.lock b/uv.lock index 3d328ce..950591c 100644 --- a/uv.lock +++ b/uv.lock @@ -776,7 +776,7 @@ wheels = [ [[package]] name = "vacatureradar" -version = "0.3.13" +version = "0.3.14" source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" },