feat(provenance): govern source snapshots and data inputs

This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
+48 -5
View File
@@ -3,6 +3,8 @@ from __future__ import annotations
import argparse
import json
import sys
from datetime import UTC, datetime
from hashlib import sha256
from pathlib import Path
from uuid import uuid4
@@ -12,9 +14,10 @@ BACKEND_ROOT = ROOT / "backend"
if str(BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(BACKEND_ROOT))
from app.models import Area, Dataset, Metric, QualityCheck # noqa: E402
from app.models import Dataset, Metric, QualityCheck, SourceRegistry, SourceSnapshot # noqa: E402
from app.services.qa_service import QaService # noqa: E402
from app.services.quality_service import QualityService # noqa: E402
from app.services.source_registry_service import SourceRegistryService # noqa: E402
class BenchmarkSession:
@@ -55,18 +58,58 @@ def _load_manifest() -> dict:
def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dataset:
# The benchmark is still synthetic evidence, but it intentionally models
# the same complete Phase 2 provenance binding required at the QA service
# boundary. The candidate is a derived fixture; the reference simulates
# a governed GRB building snapshot. No legacy fixture exception is used.
source_key = "grb" if role == "reference" else "derived"
source = SourceRegistry(
id=uuid4(),
**SourceRegistryService.definition_for(source_key).as_model_values(),
)
checksum = sha256(path.read_bytes()).hexdigest()
snapshot = SourceSnapshot(
id=uuid4(),
source_registry_id=source.id,
snapshot_key=f"golden-qa:{source_key}:{checksum}",
checksum_sha256=checksum,
fetched_at=datetime.now(UTC),
crs="EPSG:4326",
units=source.default_units,
spatial_resolution_json={"status": "fixture"},
temporal_coverage_json={"status": "fixture"},
geographic_coverage_json={"scope": "golden-qa-fixture"},
observed_schema_json={"dataset_type": "vector", "fixture_mode": True},
freshness_status="current",
ingest_status="ingested",
known_limitations_json=["Synthetic golden benchmark fixture; not production evidence."],
snapshot_metadata_json={"fixture_mode": True, "benchmark": "golden-qa"},
)
return Dataset(
id=dataset_id,
project_id=project_id,
name=name,
dataset_type="vector",
source="golden_fixture",
dataset_role=role,
source_name="fixture",
source="governed golden benchmark fixture",
dataset_role="reference" if role == "reference" else "derived",
source_name=source.source_key,
reference_layer_name="buildings" if role == "reference" else None,
storage_path=str(path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
checksum_sha256=checksum,
source_registry_id=source.id,
source_snapshot_id=snapshot.id,
source_registry=source,
source_snapshot=snapshot,
data_contract_key="geointel.vector.geojson",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
metadata_json={"crs_assumed": False, "fixture_mode": True},
source_metadata={"fixture_mode": True, "source_registry_key": source.source_key},
provenance_metadata={"fixture_mode": True, "source_snapshot_id": str(snapshot.id)},
status="ready",
)