feat(provenance): govern source snapshots and data inputs
This commit is contained in:
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -12,9 +14,10 @@ BACKEND_ROOT = ROOT / "backend"
|
||||
if str(BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_ROOT))
|
||||
|
||||
from app.models import Area, Dataset, Metric, QualityCheck # noqa: E402
|
||||
from app.models import Dataset, Metric, QualityCheck, SourceRegistry, SourceSnapshot # noqa: E402
|
||||
from app.services.qa_service import QaService # noqa: E402
|
||||
from app.services.quality_service import QualityService # noqa: E402
|
||||
from app.services.source_registry_service import SourceRegistryService # noqa: E402
|
||||
|
||||
|
||||
class BenchmarkSession:
|
||||
@@ -55,18 +58,58 @@ def _load_manifest() -> dict:
|
||||
|
||||
|
||||
def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dataset:
|
||||
# The benchmark is still synthetic evidence, but it intentionally models
|
||||
# the same complete Phase 2 provenance binding required at the QA service
|
||||
# boundary. The candidate is a derived fixture; the reference simulates
|
||||
# a governed GRB building snapshot. No legacy fixture exception is used.
|
||||
source_key = "grb" if role == "reference" else "derived"
|
||||
source = SourceRegistry(
|
||||
id=uuid4(),
|
||||
**SourceRegistryService.definition_for(source_key).as_model_values(),
|
||||
)
|
||||
checksum = sha256(path.read_bytes()).hexdigest()
|
||||
snapshot = SourceSnapshot(
|
||||
id=uuid4(),
|
||||
source_registry_id=source.id,
|
||||
snapshot_key=f"golden-qa:{source_key}:{checksum}",
|
||||
checksum_sha256=checksum,
|
||||
fetched_at=datetime.now(UTC),
|
||||
crs="EPSG:4326",
|
||||
units=source.default_units,
|
||||
spatial_resolution_json={"status": "fixture"},
|
||||
temporal_coverage_json={"status": "fixture"},
|
||||
geographic_coverage_json={"scope": "golden-qa-fixture"},
|
||||
observed_schema_json={"dataset_type": "vector", "fixture_mode": True},
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
known_limitations_json=["Synthetic golden benchmark fixture; not production evidence."],
|
||||
snapshot_metadata_json={"fixture_mode": True, "benchmark": "golden-qa"},
|
||||
)
|
||||
return Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name=name,
|
||||
dataset_type="vector",
|
||||
source="golden_fixture",
|
||||
dataset_role=role,
|
||||
source_name="fixture",
|
||||
source="governed golden benchmark fixture",
|
||||
dataset_role="reference" if role == "reference" else "derived",
|
||||
source_name=source.source_key,
|
||||
reference_layer_name="buildings" if role == "reference" else None,
|
||||
storage_path=str(path),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={"crs_assumed": False},
|
||||
checksum_sha256=checksum,
|
||||
source_registry_id=source.id,
|
||||
source_snapshot_id=snapshot.id,
|
||||
source_registry=source,
|
||||
source_snapshot=snapshot,
|
||||
data_contract_key="geointel.vector.geojson",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
metadata_json={"crs_assumed": False, "fixture_mode": True},
|
||||
source_metadata={"fixture_mode": True, "source_registry_key": source.source_key},
|
||||
provenance_metadata={"fixture_mode": True, "source_snapshot_id": str(snapshot.id)},
|
||||
status="ready",
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user