feat(provenance): govern source snapshots and data inputs

This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
+47 -6
View File
@@ -1,11 +1,11 @@
from __future__ import annotations
import json
from hashlib import sha256
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
from app.models import Area, Dataset
from app.models import Dataset, SourceRegistry, SourceSnapshot
from app.services.qa_service import QaService
@@ -46,6 +46,47 @@ def _write_features(path: Path, features: list[dict]) -> None:
path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8")
def _authoritative_reference(dataset: Dataset) -> Dataset:
"""Give QA reference fixtures the same durable authority proof as GRB."""
source_id = uuid4()
snapshot_id = uuid4()
checksum = sha256(Path(str(dataset.storage_path)).read_bytes()).hexdigest()
source = SourceRegistry(
id=source_id,
source_key="grb",
display_name="GRB test reference",
classification="authoritative",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders"},
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key=f"qa-grb-{dataset.id}",
checksum_sha256=checksum,
ingest_status="ingested",
freshness_status="current",
)
dataset.source = "grb"
dataset.source_name = "grb"
dataset.dataset_role = "reference"
dataset.status = "ready"
dataset.checksum_sha256 = checksum
dataset.source_registry_id = source_id
dataset.source_snapshot_id = snapshot_id
dataset.data_contract_key = "geointel.vector.geojson"
dataset.data_contract_version = "1.0.0"
dataset.validation_status = "passed"
dataset.provenance_status = "complete"
dataset.lineage_status = "complete"
dataset.quarantine_status = "not_quarantined"
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
project_id = uuid4()
candidate_id = uuid4()
@@ -66,7 +107,7 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
reference = Dataset(
reference = _authoritative_reference(Dataset(
id=reference_id,
project_id=project_id,
name="reference.geojson",
@@ -75,7 +116,7 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
storage_path=str(reference_path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
))
result = QaService.compare_candidate_with_reference(
db=FakeSession([candidate, reference]),
@@ -117,7 +158,7 @@ def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
reference = Dataset(
reference = _authoritative_reference(Dataset(
id=reference_id,
project_id=project_id,
name="reference.geojson",
@@ -126,7 +167,7 @@ def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_
storage_path=str(reference_path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
))
result = QaService.compare_candidate_with_reference(
db=FakeSession([candidate, reference]),