perf: optimize import persistence for 0.3.14

tests/integration/test_pipeline.py

@@ -3,10 +3,19 @@ from datetime import timedelta
 from pathlib import Path
 
 import pytest
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
 from django.utils import timezone
 
-from apps.jobs.models import Employer, JobPosting, JobSourceAlias, ScoreRun
-from apps.jobs.services.pipeline import process_raw_document
+from apps.jobs.models import (
+    Employer,
+    FieldProvenance,
+    JobPosting,
+    JobSourceAlias,
+    JobVersion,
+    ScoreRun,
+)
+from apps.jobs.services.pipeline import PersistenceContext, process_raw_document
 from apps.sources.models import RawDocument, Source
 
 
@@ -34,6 +43,39 @@ def test_pipeline_is_idempotent_and_scores(source, profile):
     assert ScoreRun.objects.filter(profile=profile).count() == 2
 
 
+@pytest.mark.integration
+@pytest.mark.django_db
+def test_unchanged_import_batch_has_bounded_lookup_queries(source, profile):
+    content = Path("fixtures/pages/sample_jsonld_job.html").read_text(encoding="utf-8")
+    documents = [
+        RawDocument.objects.create(
+            source=source,
+            url=f"https://jobs.example.org/import/{index}",
+            final_url=f"https://jobs.example.org/import/{index}",
+            kind=RawDocument.Kind.HTML,
+            content_type="text/html",
+            content_hash=hashlib.sha256(f"{index}{content}".encode()).hexdigest(),
+            body_text=content,
+            byte_length=len(content.encode()),
+            retain_until=timezone.now() + timedelta(days=7),
+        )
+        for index in range(20)
+    ]
+    context = PersistenceContext()
+
+    with CaptureQueriesContext(connection) as queries:
+        results = [process_raw_document(document, context=context) for document in documents]
+
+    assert sum(int(result["created"]) for result in results) == 1
+    assert sum(int(result["duplicates"]) for result in results) == 19
+    assert len(queries) < 300
+    assert Employer.objects.count() == 1
+    assert JobPosting.objects.count() == 1
+    assert JobVersion.objects.count() == 1
+    assert FieldProvenance.objects.count() > 0
+    assert ScoreRun.objects.filter(profile=profile).count() == 20
+
+
 @pytest.mark.django_db
 def test_pipeline_uses_reviewed_employer_identity_for_public_ats_feed(profile):
     source = Source.objects.create(

tests/unit/test_import_benchmark.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+
+import pytest
+
+from scripts.benchmark_import import UnsafeBenchmarkEnvironment, validate_benchmark_identity
+
+
+def _database(name: str, host: str = "127.0.0.1") -> dict[str, str]:
+    return {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": name,
+        "HOST": host,
+    }
+
+
+def test_benchmark_identity_accepts_only_isolated_postgres() -> None:
+    validate_benchmark_identity(
+        _database("vacatureradar_bench_release_0314"), "https://benchmark.invalid"
+    )
+
+
+@pytest.mark.parametrize(
+    ("database", "public_url"),
+    [
+        (_database("vacatureradar"), "https://benchmark.invalid"),
+        (
+            _database("vacatureradar_bench_release", "database.internal"),
+            "https://benchmark.invalid",
+        ),
+        (_database("vacatureradar_bench_release"), "https://vacatureradar.itworx.tech"),
+        (
+            {"ENGINE": "django.db.backends.sqlite3", "NAME": Path("bench.sqlite3"), "HOST": ""},
+            "https://benchmark.invalid",
+        ),
+    ],
+)
+def test_benchmark_identity_rejects_unsafe_targets(
+    database: dict[str, object], public_url: str
+) -> None:
+    with pytest.raises(UnsafeBenchmarkEnvironment):
+        validate_benchmark_identity(database, public_url)