perf: optimize import persistence for 0.3.14

scripts/benchmark.py

@@ -20,7 +20,6 @@ import django
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.db import connection
-from django.test.utils import CaptureQueriesContext
 from django.utils import timezone
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
@@ -29,7 +28,7 @@ django.setup()
 from apps.jobs.models import Employer, JobPosting, JobSourceAlias, ScoreRun  # noqa: E402
 from apps.jobs.services.dedupe import find_existing_job  # noqa: E402
 from apps.jobs.services.normalization import CanonicalJobDraft, normalize_token  # noqa: E402
-from apps.jobs.services.pipeline import process_raw_document  # noqa: E402
+from apps.jobs.services.pipeline import PersistenceContext, process_raw_document  # noqa: E402
 from apps.jobs.services.scoring import calculate_score, rescore_jobs_with_profiles  # noqa: E402
 from apps.profiles.models import SearchProfile  # noqa: E402
 from apps.sources.adapters.registry import registry  # noqa: E402
@@ -575,15 +574,25 @@ def _clean_benchmark(artifacts: BenchmarkArtifact) -> None:
         get_user_model().objects.filter(pk=artifacts.user_id).delete()
 
 
+class QueryCounter:
+    """Count database executions without Django's bounded debug-query log."""
+
+    def __init__(self) -> None:
+        self.count = 0
+        self.types: dict[str, int] = {}
+
+    def __call__(self, execute: Any, sql: str, params: Any, many: bool, context: Any) -> Any:
+        self.count += 1
+        query_type = sql.lstrip().split(None, 1)[0].upper() if sql.strip() else "UNKNOWN"
+        self.types[query_type] = self.types.get(query_type, 0) + 1
+        return execute(sql, params, many, context)
+
+
 @contextmanager
 def _query_capture() -> Any:
-    original_force_debug_cursor = connection.force_debug_cursor
-    connection.force_debug_cursor = True
-    try:
-        with CaptureQueriesContext(connection) as captured:
-            yield captured
-    finally:
-        connection.force_debug_cursor = original_force_debug_cursor
+    counter = QueryCounter()
+    with connection.execute_wrapper(counter):
+        yield counter
 
 
 def run_performance_benchmark(
@@ -619,6 +628,7 @@ def run_performance_benchmark(
     }
     tracemalloc.start()
     import_timings: list[float] = []
+    persistence_context = PersistenceContext()
     rescore_timings: list[float] = []
     list_timings: list[float] = []
     detail_timings: list[float] = []
@@ -646,9 +656,10 @@ def run_performance_benchmark(
                         retain_until=timezone.now() + timezone.timedelta(days=7),
                     )
                     start = perf_counter()
-                    process_raw_document(document)
+                    process_raw_document(document, context=persistence_context)
                     import_timings.append((perf_counter() - start) * 1000)
-        import_query_count = len(captured_import.captured_queries)
+        import_query_count = captured_import.count
+        import_query_types = captured_import.types
         query_count_total += import_query_count
 
         candidates = list(
@@ -666,7 +677,8 @@ def run_performance_benchmark(
                 batch_start = perf_counter()
                 rescore_jobs_with_profiles(batch, profile_id=profile.id)
                 rescore_timings.append((perf_counter() - batch_start) * 1000)
-        rescore_query_count = len(captured_rescore.captured_queries)
+        rescore_query_count = captured_rescore.count
+        rescore_query_types = captured_rescore.types
         query_count_total += rescore_query_count
 
         with _query_capture() as captured_list:
@@ -695,8 +707,10 @@ def run_performance_benchmark(
                 )
                 detail_timings.append((perf_counter() - start) * 1000)
 
-        list_query_count = len(captured_list.captured_queries)
-        detail_query_count = len(captured_detail.captured_queries)
+        list_query_count = captured_list.count
+        detail_query_count = captured_detail.count
+        list_query_types = captured_list.types
+        detail_query_types = captured_detail.types
         query_count_total += list_query_count + detail_query_count
 
         current_bytes, peak_bytes = tracemalloc.get_traced_memory()
@@ -740,6 +754,7 @@ def run_performance_benchmark(
             "p50_ms": timing_stats["import_p50_ms"],
             "p95_ms": timing_stats["import_p95_ms"],
             "query_count": import_query_count,
+            "query_types": import_query_types,
             "timing_count": len(import_timings),
         },
         "rescore": {
@@ -747,11 +762,14 @@ def run_performance_benchmark(
             "p50_ms": timing_stats["rescore_p50_ms"],
             "p95_ms": timing_stats["rescore_p95_ms"],
             "query_count": rescore_query_count,
+            "query_types": rescore_query_types,
             "timing_count": len(rescore_timings),
         },
         "dashboard": {
             "list_query_count": list_query_count,
             "detail_query_count": detail_query_count,
+            "list_query_types": list_query_types,
+            "detail_query_types": detail_query_types,
             "list_p95_ms": timing_stats["list_p95_ms"],
             "detail_p95_ms": timing_stats["detail_p95_ms"],
             "timing_count": len(list_timings) + len(detail_timings),

scripts/benchmark_import.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+from statistics import mean
+from typing import Any
+from urllib.parse import urlparse
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+
+import django
+
+django.setup()
+
+from django.conf import settings  # noqa: E402
+from django.core.management import call_command  # noqa: E402
+
+from scripts.benchmark import DEFAULT_DATASET, run_benchmark_report  # noqa: E402
+
+SAFE_DATABASE_PREFIX = "vacatureradar_bench_"
+SAFE_PUBLIC_HOSTS = {"", "benchmark.invalid", "localhost", "127.0.0.1"}
+
+
+class UnsafeBenchmarkEnvironment(RuntimeError):
+    pass
+
+
+def validate_benchmark_identity(database: dict[str, Any], public_base_url: str) -> None:
+    engine = str(database.get("ENGINE", ""))
+    name = str(database.get("NAME", ""))
+    host = str(database.get("HOST", ""))
+    public_host = (urlparse(public_base_url).hostname or "").casefold()
+    if engine != "django.db.backends.postgresql":
+        raise UnsafeBenchmarkEnvironment("De importbenchmark vereist PostgreSQL.")
+    if not name.startswith(SAFE_DATABASE_PREFIX):
+        raise UnsafeBenchmarkEnvironment(
+            f"Onveilige databasenaam {name!r}; verwacht prefix {SAFE_DATABASE_PREFIX!r}."
+        )
+    if host not in {"127.0.0.1", "localhost", "postgres"}:
+        raise UnsafeBenchmarkEnvironment(f"Onveilige benchmarkdatabasehost {host!r}.")
+    if public_host not in SAFE_PUBLIC_HOSTS:
+        raise UnsafeBenchmarkEnvironment(
+            f"Publieke host {public_host!r} is niet toegestaan voor een benchmark."
+        )
+
+
+def _summary(reports: list[dict[str, Any]]) -> dict[str, float]:
+    performance = [report["performance"] for report in reports]
+    imports = [item["import"] for item in performance]
+    return {
+        "query_count_mean": round(mean(item["query_count"] for item in imports), 3),
+        "query_count_total_mean": round(mean(item["query_count"] for item in performance), 3),
+        "duration_ms_mean": round(mean(item["duration_ms_total"] for item in imports), 3),
+        "p50_ms_mean": round(mean(item["p50_ms"] for item in imports), 3),
+        "p95_ms_mean": round(mean(item["p95_ms"] for item in imports), 3),
+        "throughput_per_second": round(
+            mean(
+                report["jobs_processed"] / (item["duration_ms_total"] / 1000)
+                for report, item in zip(reports, imports, strict=True)
+            ),
+            3,
+        ),
+        "memory_peak_mb_mean": round(
+            mean(item["performance"]["memory_peak_mb"] for item in performance), 3
+        ),
+    }
+
+
+def run_isolated_benchmark(
+    *, dataset: Path, jobs: int, iterations: int, output: Path
+) -> dict[str, Any]:
+    validate_benchmark_identity(settings.DATABASES["default"], settings.PUBLIC_BASE_URL)
+    if iterations < 1:
+        raise ValueError("iterations moet minstens 1 zijn.")
+    call_command("migrate", interactive=False, verbosity=0)
+    call_command("flush", interactive=False, verbosity=0)
+    warmup = run_benchmark_report(dataset, quick=True, jobs=jobs)
+    measured: list[dict[str, Any]] = []
+    for _ in range(iterations):
+        call_command("flush", interactive=False, verbosity=0)
+        measured.append(run_benchmark_report(dataset, quick=True, jobs=jobs))
+    report = {
+        "schema_version": "1.0",
+        "database_identity": {
+            "engine": settings.DATABASES["default"]["ENGINE"],
+            "name_prefix": SAFE_DATABASE_PREFIX,
+            "host": settings.DATABASES["default"]["HOST"],
+        },
+        "configuration": {
+            "jobs": jobs,
+            "iterations": iterations,
+            "warmup_runs": 1,
+            "concurrency": 1,
+            "dataset": str(dataset),
+        },
+        "warmup_status": warmup["status"],
+        "summary": _summary(measured),
+        "iterations": measured,
+        "status": "passed" if all(item["status"] == "passed" for item in measured) else "failed",
+    }
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return report
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Fail-closed geïsoleerde importbenchmark")
+    parser.add_argument("--dataset", type=Path, default=DEFAULT_DATASET)
+    parser.add_argument("--jobs", type=int, default=250)
+    parser.add_argument("--iterations", type=int, default=3)
+    parser.add_argument(
+        "--output", type=Path, default=Path("artifacts/import-performance-report.json")
+    )
+    args = parser.parse_args()
+    report = run_isolated_benchmark(
+        dataset=args.dataset, jobs=args.jobs, iterations=args.iterations, output=args.output
+    )
+    print(json.dumps(report, indent=2, sort_keys=True))
+    return 0 if report["status"] == "passed" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_import_benchmark.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+: "${DATABASE_URL:?Stel DATABASE_URL in op een afzonderlijke vacatureradar_bench_* PostgreSQL-database.}"
+export PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-https://benchmark.invalid}"
+export DJANGO_DEBUG=0
+export DJANGO_SECRET_KEY="${DJANGO_SECRET_KEY:-benchmark-only-secret-key-not-for-production-000000000000000000}"
+export CELERY_TASK_ALWAYS_EAGER=1
+export OLLAMA_ENABLED=0
+export EMAIL_BACKEND=django.core.mail.backends.locmem.EmailBackend
+
+if command -v uv >/dev/null 2>&1; then
+  exec uv run python -m scripts.benchmark_import "$@"
+fi
+exec .venv/bin/python -m scripts.benchmark_import "$@"