geointel/backend/app/services/derived_dataset_governance_service.py

"""Governance for persisted derived and fixture datasets.

Dataset importers own raw-source ingestion.  This small service owns the
other persistence boundary: artifacts produced inside the workbench (vector
and raster operations) and the explicitly local demo fixtures.  It is kept
separate from :mod:`dataset_service` so an operation can never create a ready
dataset without a source registry binding, immutable snapshot, validation
report and, for derived results, a durable lineage edge.

The ``Session.query`` capability check deliberately preserves lightweight
unit-test doubles used by pre-Phase-2 tests.  Real SQLAlchemy sessions always
take the governed branch; the compatibility branch is not reachable in the
application runtime.
"""

from __future__ import annotations

from datetime import datetime, timezone
from hashlib import sha256
import json
import re
from typing import Any, Mapping

from sqlalchemy.orm import Session

from app.models import Dataset, DatasetVersion
from app.services.data_contract_validation import (
    IssueSeverity,
    LineageStatus,
    LineageEvidence,
    ProvenanceStatus,
    QuarantineStatus,
    TransformationEvidence,
    ValidationIssue,
    ValidationReport,
    ValidationStatus,
    build_raster_ingest_input,
    build_vector_ingest_input,
    validate_registered_asset,
)
from app.services.data_quarantine_service import DataQuarantineService
from app.services.dataset_consumption_gate_service import DatasetConsumptionDecision, DatasetConsumptionGate
from app.services.source_registry_service import SourceRegistryService


_SHA256 = re.compile(r"^[0-9a-f]{64}$")
_CANONICAL_VECTOR_CRS = "EPSG:4326"


class DerivedDatasetGovernanceService:
    """Apply Phase-2 provenance rules to non-importer Dataset creation.

    Callers add the dataset and first immutable version, then call one of the
    ``govern_*`` methods before committing.  A failed contract deliberately
    leaves the artifact and its durable quarantine record in the transaction;
    it is never silently promoted to ``ready``.
    """

    @staticmethod
    def persistence_available(db: Session) -> bool:
        """Return whether this is a real ORM persistence session.

        Historical unit tests use minimal fakes with ``add``/``commit`` only.
        Keeping that explicitly isolated avoids pretending a fake test store
        has source-registry guarantees while production remains fail-closed.
        """

        return callable(getattr(db, "query", None)) and callable(getattr(db, "flush", None))

    @classmethod
    def govern_vector(
        cls,
        db: Session,
        *,
        dataset: Dataset,
        dataset_version: DatasetVersion,
        feature_collection: Mapping[str, Any],
        source_key: str,
        operation: str,
        parent_dataset: Dataset | None = None,
        operation_parameters: Mapping[str, Any] | None = None,
    ) -> bool:
        """Validate and bind a vector result, returning ``True`` when ready."""

        if not cls.persistence_available(db):
            # Explicit compatibility for historical minimal test fixtures.
            # Production sessions always have query/flush and never take this
            # branch.
            dataset.status = "ready"
            return True

        parent_gate = cls._parent_derived_processing_gate(parent_dataset)
        metadata = dict(dataset.metadata_json or {})
        output_crs = str(metadata.get("crs") or dataset.crs or _CANONICAL_VECTOR_CRS)
        source = SourceRegistryService.ensure_server_owned_source(db, source_key)
        snapshot = cls._record_snapshot(
            db,
            source_key=source_key,
            dataset=dataset,
            operation=operation,
            source_crs=output_crs,
            spatial_resolution=metadata.get("resolution_json") or {"status": "not_applicable"},
            geographic_coverage={"bounds": metadata.get("bounds_json") or dataset.bounds_json},
            observed_schema={
                "dataset_type": "vector",
                "geometry_types": metadata.get("geometry_types") or [],
                "feature_count": metadata.get("feature_count"),
            },
        )
        lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
        report = validate_registered_asset(
            build_vector_ingest_input(
                asset_id=str(dataset.id),
                source_crs=output_crs,
                storage_crs=output_crs,
                feature_collection=feature_collection,
                checksum_sha256=dataset.checksum_sha256,
                computed_checksum_sha256=dataset.checksum_sha256,
                source_registry_id=str(source.id),
                source_snapshot_id=str(snapshot.id),
                imported_at=dataset.imported_at or datetime.now(timezone.utc),
                metadata=cls._contract_metadata(metadata, source.license_name),
                observed_at=dataset.observed_at,
                valid_from=dataset.valid_from,
                valid_to=dataset.valid_to,
                temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
                source_version=dataset.source_version,
                source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
                lineage=lineage,
            )
        )
        if parent_gate is not None and not parent_gate.eligible:
            report = cls._with_parent_gate_failure(report, parent_gate)
        return cls._apply(
            db,
            dataset=dataset,
            dataset_version=dataset_version,
            source=source,
            snapshot=snapshot,
            report=report,
            stage="derived_vector_validation",
            parent_dataset=parent_dataset,
            operation=operation,
            operation_parameters=operation_parameters,
        )

    @classmethod
    def govern_raster(
        cls,
        db: Session,
        *,
        dataset: Dataset,
        dataset_version: DatasetVersion,
        raster_metadata: Mapping[str, Any],
        source_key: str,
        operation: str,
        parent_dataset: Dataset | None = None,
        operation_parameters: Mapping[str, Any] | None = None,
    ) -> bool:
        """Validate and bind a raster result, returning ``True`` when ready."""

        if not cls.persistence_available(db):
            dataset.status = "ready"
            return True

        parent_gate = cls._parent_derived_processing_gate(parent_dataset)
        metadata = dict(raster_metadata or {})
        output_crs = str(metadata.get("crs") or dataset.crs or "") or None
        source = SourceRegistryService.ensure_server_owned_source(db, source_key)
        snapshot = cls._record_snapshot(
            db,
            source_key=source_key,
            dataset=dataset,
            operation=operation,
            source_crs=output_crs,
            spatial_resolution=cls._raster_resolution(metadata, output_crs),
            geographic_coverage={"bounds": metadata.get("bounds") or dataset.bounds_json},
            observed_schema={
                "dataset_type": "raster",
                "width": metadata.get("width"),
                "height": metadata.get("height"),
                "band_count": metadata.get("band_count"),
                "dtype": metadata.get("dtype"),
            },
        )
        lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
        report = validate_registered_asset(
            build_raster_ingest_input(
                asset_id=str(dataset.id),
                source_crs=output_crs,
                storage_crs=output_crs,
                raster_profile=metadata,
                bounds=metadata.get("bounds") or dataset.bounds_json,
                resolution=cls._raster_resolution(metadata, output_crs),
                checksum_sha256=dataset.checksum_sha256,
                computed_checksum_sha256=dataset.checksum_sha256,
                source_registry_id=str(source.id),
                source_snapshot_id=str(snapshot.id),
                imported_at=dataset.imported_at or datetime.now(timezone.utc),
                metadata=cls._contract_metadata(metadata, source.license_name),
                observed_at=dataset.observed_at,
                valid_from=dataset.valid_from,
                valid_to=dataset.valid_to,
                temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
                source_version=dataset.source_version,
                source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
                lineage=lineage,
            )
        )
        if parent_gate is not None and not parent_gate.eligible:
            report = cls._with_parent_gate_failure(report, parent_gate)
        return cls._apply(
            db,
            dataset=dataset,
            dataset_version=dataset_version,
            source=source,
            snapshot=snapshot,
            report=report,
            stage="derived_raster_validation",
            parent_dataset=parent_dataset,
            operation=operation,
            operation_parameters=operation_parameters,
        )

    @classmethod
    def _record_snapshot(
        cls,
        db: Session,
        *,
        source_key: str,
        dataset: Dataset,
        operation: str,
        source_crs: str | None,
        spatial_resolution: Mapping[str, Any] | None,
        geographic_coverage: Mapping[str, Any] | None,
        observed_schema: Mapping[str, Any] | None,
    ):
        checksum = cls._checksum_or_placeholder(dataset.checksum_sha256)
        # A derived artifact has a distinct creation event even when its bytes
        # equal a prior output.  Include the immutable dataset id so source
        # snapshots never collide on a different fetched_at timestamp.
        snapshot_key = f"{source_key}:{operation}:{dataset.id}:{checksum}"
        return SourceRegistryService.record_snapshot(
            db,
            source_key=source_key,
            snapshot_key=snapshot_key,
            checksum_sha256=checksum,
            source_version=dataset.source_version or f"{operation}:1.0.0",
            snapshot_at=dataset.observed_at,
            fetched_at=dataset.imported_at or datetime.now(timezone.utc),
            crs=source_crs,
            units=cls._units_for_crs(source_crs),
            spatial_resolution=dict(spatial_resolution or {"status": "unknown"}),
            temporal_coverage={
                "observed_at": cls._datetime_value(dataset.observed_at),
                "valid_from": cls._datetime_value(dataset.valid_from),
                "valid_to": cls._datetime_value(dataset.valid_to),
            },
            geographic_coverage=dict(geographic_coverage or {"status": "unknown"}),
            observed_schema=dict(observed_schema or {"status": "unknown"}),
            # A transform cannot establish source freshness.  With no source
            # observation it is not applicable; with one it still needs an
            # explicit policy review rather than a fabricated "current" flag.
            freshness_status="not_applicable" if dataset.observed_at is None else "review_required",
            ingest_status="ingested",
            known_limitations=[
                "Derived and fixture artifacts inherit no automatic source authority beyond their explicit registry entry and lineage.",
            ],
            snapshot_metadata={
                "operation": operation,
                "dataset_id": str(dataset.id),
                "storage_path": dataset.storage_path,
                "artifact_checksum_sha256": dataset.checksum_sha256,
            },
        )

    @classmethod
    def _apply(
        cls,
        db: Session,
        *,
        dataset: Dataset,
        dataset_version: DatasetVersion,
        source: Any,
        snapshot: Any,
        report: ValidationReport,
        stage: str,
        parent_dataset: Dataset | None,
        operation: str,
        operation_parameters: Mapping[str, Any] | None,
    ) -> bool:
        fields = report.persistence_fields()
        dataset.validation_report_json = fields["validation_report_json"]
        dataset_version.validation_report_json = fields["validation_report_json"]
        SourceRegistryService.bind_dataset_provenance(
            dataset,
            source=source,
            snapshot=snapshot,
            data_contract_key=fields["data_contract_key"],
            data_contract_version=fields["data_contract_version"],
            validation_status=fields["validation_status"],
            provenance_status=fields["provenance_status"],
            lineage_status=fields["lineage_status"],
        )
        SourceRegistryService.bind_dataset_version_provenance(
            dataset_version,
            source=source,
            snapshot=snapshot,
            data_contract_key=fields["data_contract_key"],
            data_contract_version=fields["data_contract_version"],
            validation_status=fields["validation_status"],
            provenance_status=fields["provenance_status"],
            lineage_status=fields["lineage_status"],
        )

        # IDs for DatasetVersion defaults exist only after the caller adds and
        # flushes both rows.  The operation services call this before commit.
        db.flush()
        if parent_dataset is not None:
            SourceRegistryService.record_lineage_edge(
                db,
                parent_dataset_id=parent_dataset.id,
                child_dataset_id=dataset.id,
                parent_dataset_version_id=cls._latest_parent_version_id(db, parent_dataset),
                child_dataset_version_id=dataset_version.id,
                relation_type="derived_from",
                transformation_name=operation,
                transformation_version="1.0.0",
                parameters=dict(operation_parameters or {}),
                input_checksum_sha256=cls._valid_checksum(parent_dataset.checksum_sha256),
                output_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
            )

        decision = DataQuarantineService.decide(report)
        if decision.eligible_for_use:
            dataset.status = "ready"
            dataset.quarantine_status = "not_quarantined"
            return True

        SourceRegistryService.quarantine_dataset(
            db,
            dataset=dataset,
            dataset_version=dataset_version,
            source_snapshot=snapshot,
            stage=stage,
            reason_code=(decision.reason_codes[0] if decision.reason_codes else "DATA_CONTRACT_FAILED"),
            details={"validation_report": report.to_dict(), "quarantine_decision": decision.to_dict()},
            artifact_path=dataset.storage_path,
            artifact_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
        )
        return False

    @staticmethod
    def _contract_metadata(metadata: Mapping[str, Any], license_name: str) -> dict[str, Any]:
        values = dict(metadata)
        values.setdefault("license", license_name)
        return values

    @classmethod
    def _lineage_evidence(
        cls,
        parent_dataset: Dataset | None,
        operation: str,
        operation_parameters: Mapping[str, Any] | None,
    ) -> LineageEvidence:
        upstream_ids: tuple[str, ...] = ()
        upstream_checksums: tuple[str, ...] = ()
        if parent_dataset is not None:
            upstream_ids = (str(parent_dataset.id),)
            # An invalid/missing parent checksum intentionally fails the
            # derived contract rather than inventing traceability.  The same
            # applies to a legacy/unvalidated parent: it may remain visible
            # as evidence, but cannot create a new ready derived asset.
            parent_is_governed = (
                parent_dataset.status == "ready"
                and parent_dataset.quarantine_status == "not_quarantined"
                and parent_dataset.validation_status == "passed"
                and parent_dataset.provenance_status == "complete"
                and parent_dataset.source_registry_id is not None
                and parent_dataset.source_snapshot_id is not None
            )
            upstream_checksums = (
                parent_dataset.checksum_sha256 if parent_is_governed else "parent_dataset_not_governed",
            )
        transform_checksum = cls._stable_hash(
            {"operation": operation, "version": "1.0.0", "parameters": dict(operation_parameters or {})}
        )
        return LineageEvidence(
            upstream_asset_ids=upstream_ids,
            upstream_checksums_sha256=upstream_checksums,
            transformations=(
                TransformationEvidence(
                    name=operation,
                    version="1.0.0",
                    checksum_sha256=transform_checksum,
                ),
            ),
        )

    @staticmethod
    def _parent_derived_processing_gate(parent_dataset: Dataset | None) -> DatasetConsumptionDecision | None:
        """Evaluate the durable parent boundary before creating a ready child.

        We deliberately turn a rejected parent into a child validation failure
        (instead of simply raising): the output is then persisted with its
        source snapshot, validation evidence and durable quarantine record.
        This makes an attempted derivation from a manual or experimental
        dataset observable and prevents a caller from bypassing the boundary
        by invoking the governance service directly.
        """

        if parent_dataset is None:
            return None
        return DatasetConsumptionGate.evaluate(parent_dataset, purpose="derived_processing")

    @staticmethod
    def _with_parent_gate_failure(
        report: ValidationReport,
        decision: DatasetConsumptionDecision,
    ) -> ValidationReport:
        """Attach an auditable, fail-closed lineage failure to a report."""

        issue = ValidationIssue(
            code="PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING",
            category="lineage",
            field="lineage.parent_dataset",
            message="Parent dataset failed the governed derived-processing consumption gate.",
            severity=IssueSeverity.ERROR,
            expected="eligible governed parent dataset",
            observed={
                "dataset_id": decision.evidence.get("dataset_id"),
                "reasons": list(decision.reasons),
                "source_key": decision.evidence.get("source_key"),
                "source_classification": decision.evidence.get("source_classification"),
            },
        )
        return ValidationReport(
            asset_id=report.asset_id,
            data_contract_key=report.data_contract_key,
            data_contract_version=report.data_contract_version,
            contract_fingerprint_sha256=report.contract_fingerprint_sha256,
            validation_status=ValidationStatus.FAILED,
            provenance_status=(
                ProvenanceStatus.INCOMPLETE
                if report.provenance_status == ProvenanceStatus.COMPLETE
                else report.provenance_status
            ),
            lineage_status=LineageStatus.INCOMPLETE,
            quarantine_status=QuarantineStatus.QUARANTINED,
            validation_scope=report.validation_scope,
            checked_at=report.checked_at,
            issues=(*report.issues, issue),
        )

    @staticmethod
    def _latest_parent_version_id(db: Session, dataset: Dataset):
        version = (
            db.query(DatasetVersion)
            .filter(DatasetVersion.dataset_id == dataset.id)
            .order_by(DatasetVersion.version.desc())
            .first()
        )
        return version.id if version is not None else None

    @staticmethod
    def _raster_resolution(metadata: Mapping[str, Any], crs: str | None) -> dict[str, Any] | None:
        values = metadata.get("resolution")
        if not isinstance(values, (list, tuple)) or len(values) < 2:
            return None
        try:
            return {"x": abs(float(values[0])), "y": abs(float(values[1])), "unit": DerivedDatasetGovernanceService._resolution_unit(crs)}
        except (TypeError, ValueError):
            return None

    @staticmethod
    def _resolution_unit(crs: str | None) -> str:
        return "degree" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "m"

    @staticmethod
    def _units_for_crs(crs: str | None) -> str:
        return "degrees" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "metres"

    @staticmethod
    def _temporal_unknown_reason(dataset: Dataset) -> str | None:
        if dataset.observed_at is not None:
            return None
        return "Derived or fixture artifact inherits no precise observation timestamp from its input."

    @staticmethod
    def _source_version_unknown_reason(dataset: Dataset) -> str | None:
        if dataset.source_version:
            return None
        return "Derived or fixture artifact has no source edition; the transform version is recorded separately."

    @staticmethod
    def _datetime_value(value: datetime | None) -> str | None:
        return value.astimezone(timezone.utc).isoformat() if value is not None else None

    @staticmethod
    def _stable_hash(value: Mapping[str, Any]) -> str:
        return sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()

    @staticmethod
    def _valid_checksum(value: str | None) -> str | None:
        normalized = str(value or "").strip().lower()
        return normalized if _SHA256.fullmatch(normalized) else None

    @classmethod
    def _checksum_or_placeholder(cls, value: str | None) -> str:
        checksum = cls._valid_checksum(value)
        if checksum is not None:
            return checksum
        # The contract receives the original invalid/missing checksum and
        # quarantines it.  A deterministic placeholder only permits storing
        # the rejected snapshot without fabricating a valid artifact hash.
        return cls._stable_hash({"invalid_dataset_checksum": value or "missing"})