134 lines
5.4 KiB
Python
134 lines
5.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Emit reproducible local evidence for the Phase-2 data foundation.
|
|
|
|
The collector does not connect to PostgreSQL, source providers, a GPU or any
|
|
training corpus. It inventories the versioned source-policy and contract
|
|
definitions that are present in this checkout. Runtime migration/application
|
|
evidence is recorded separately because it needs an explicitly chosen target
|
|
database and must never be inferred from this static report.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from collections import Counter
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BACKEND = ROOT / "backend"
|
|
if str(BACKEND) not in sys.path:
|
|
sys.path.insert(0, str(BACKEND))
|
|
|
|
from app.services.data_contract_validation import build_default_data_contract_registry # noqa: E402
|
|
from app.services.source_registry_service import SERVER_OWNED_SOURCE_DEFINITIONS # noqa: E402
|
|
|
|
|
|
def _git_value(*args: str) -> str | None:
|
|
try:
|
|
return subprocess.check_output(
|
|
["git", *args],
|
|
cwd=ROOT,
|
|
text=True,
|
|
stderr=subprocess.DEVNULL,
|
|
).strip() or None
|
|
except (OSError, subprocess.CalledProcessError):
|
|
return None
|
|
|
|
|
|
def collect() -> dict[str, Any]:
|
|
definitions = list(SERVER_OWNED_SOURCE_DEFINITIONS.values())
|
|
contracts = build_default_data_contract_registry().registered_contracts()
|
|
classification_counts = Counter(item.classification for item in definitions)
|
|
source_items = [
|
|
{
|
|
"source_key": item.source_key,
|
|
"classification": item.classification,
|
|
"authority_name": item.authority_name,
|
|
"authority_scope": item.authority_scope,
|
|
"provider_adapter_key": item.provider_adapter_key,
|
|
"default_crs": item.default_crs,
|
|
"default_units": item.default_units,
|
|
"freshness_status": item.freshness_status,
|
|
"ingest_status": item.ingest_status,
|
|
"ground_truth_allowed": bool((item.usage_policy or {}).get("ground_truth_allowed")),
|
|
"training_allowed": bool((item.usage_policy or {}).get("training_allowed")),
|
|
}
|
|
for item in sorted(definitions, key=lambda definition: definition.source_key)
|
|
]
|
|
contract_items = [
|
|
{
|
|
"key": item.key,
|
|
"version": item.version,
|
|
"kind": item.kind.value,
|
|
"fingerprint_sha256": item.fingerprint(),
|
|
"canonical_storage_crs": item.canonical_storage_crs,
|
|
"accepted_source_crs": sorted(item.accepted_source_crs),
|
|
"required_metadata_fields": list(item.required_metadata_fields),
|
|
"requires_source_registry": item.lineage_rules.require_source_registry,
|
|
"requires_source_snapshot": item.lineage_rules.require_source_snapshot,
|
|
"requires_upstream_assets": item.lineage_rules.require_upstream_assets,
|
|
}
|
|
for item in sorted(contracts, key=lambda contract: (contract.kind.value, contract.key, contract.version))
|
|
]
|
|
return {
|
|
"schema_version": 1,
|
|
"program": "GeoIntel Accuracy Improvement Program",
|
|
"phase": "P2",
|
|
"collected_at": datetime.now(UTC).isoformat(),
|
|
"repository": {
|
|
"branch": _git_value("branch", "--show-current"),
|
|
"head": _git_value("rev-parse", "HEAD"),
|
|
"dirty": bool(_git_value("status", "--porcelain")),
|
|
},
|
|
"scope": "Belgium and the Belgian North Sea",
|
|
"migration_revision": "202608010001",
|
|
"source_registry": {
|
|
"definition_count": len(source_items),
|
|
"classification_counts": dict(sorted(classification_counts.items())),
|
|
"required_building_policy": {
|
|
"grb_primary_building_validation": SERVER_OWNED_SOURCE_DEFINITIONS["grb"].usage_policy[
|
|
"validation_authority"
|
|
].get("building_validation"),
|
|
"buildings_register_classification": SERVER_OWNED_SOURCE_DEFINITIONS[
|
|
"digitaal_vlaanderen_buildings_addresses_register"
|
|
].classification,
|
|
"sentinel_2_classification": SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"].classification,
|
|
"dhmv_classification": SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"].classification,
|
|
"osm_ground_truth_allowed": SERVER_OWNED_SOURCE_DEFINITIONS["osm"].usage_policy[
|
|
"ground_truth_allowed"
|
|
],
|
|
},
|
|
"definitions": source_items,
|
|
},
|
|
"data_contracts": contract_items,
|
|
"claim_boundary": (
|
|
"Static policy/contract inventory only. It does not attest that a database migration ran, "
|
|
"that legacy rows are complete, or that a model/corpus is release-ready."
|
|
),
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--output",
|
|
type=Path,
|
|
default=ROOT / "artifacts" / "evidence" / "accuracy" / "P2" / "source-contract-inventory.json",
|
|
)
|
|
args = parser.parse_args()
|
|
result = collect()
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(json.dumps({"output": str(args.output), "source_count": result["source_registry"]["definition_count"]}))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|