Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
211 lines
7.4 KiB
Python
211 lines
7.4 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
from geoalchemy2.shape import from_shape
|
|
import pytest
|
|
from shapely.geometry import box
|
|
|
|
from app.core.config import Settings
|
|
from app.core.errors import AppError
|
|
from app.models import Area, Dataset, DatasetVersion, SourceRegistry, SourceSnapshot
|
|
from app.services.tile_manifest_service import TileManifestService, canonical_manifest_json
|
|
|
|
|
|
class FakeSession:
|
|
def __init__(self, objects=None) -> None:
|
|
self.objects = objects or {}
|
|
|
|
def get(self, model, item_id):
|
|
return self.objects.get((model, item_id))
|
|
|
|
|
|
def _dataset_and_session(*, with_area: bool = True):
|
|
project_id = uuid4()
|
|
dataset_id = uuid4()
|
|
registry_id = uuid4()
|
|
snapshot_id = uuid4()
|
|
checksum = "a" * 64
|
|
area = None
|
|
area_id = uuid4() if with_area else None
|
|
if area_id is not None:
|
|
area = Area(
|
|
id=area_id,
|
|
project_id=project_id,
|
|
name="Inference AOI",
|
|
geometry=from_shape(box(4.0, 51.0, 5.0, 52.0), srid=4326),
|
|
)
|
|
registry = SourceRegistry(
|
|
id=registry_id,
|
|
source_key="governed-test-raster",
|
|
display_name="Governed test raster",
|
|
classification="derived",
|
|
authority_name="GeoIntel",
|
|
)
|
|
snapshot = SourceSnapshot(
|
|
id=snapshot_id,
|
|
source_registry_id=registry_id,
|
|
snapshot_key="snapshot-1",
|
|
checksum_sha256=checksum,
|
|
ingest_status="ingested",
|
|
freshness_status="current",
|
|
)
|
|
dataset = Dataset(
|
|
id=dataset_id,
|
|
project_id=project_id,
|
|
area_id=area_id,
|
|
name="orthophoto.tif",
|
|
dataset_type="raster",
|
|
source="governed-test-raster",
|
|
source_name="governed-test-raster",
|
|
storage_path="storage/uploads/orthophoto.tif",
|
|
size_bytes=123,
|
|
checksum_sha256=checksum,
|
|
crs="EPSG:4326",
|
|
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
|
|
source_registry_id=registry_id,
|
|
source_snapshot_id=snapshot_id,
|
|
data_contract_key="geointel.raster.geotiff",
|
|
data_contract_version="1.0.0",
|
|
validation_status="passed",
|
|
provenance_status="complete",
|
|
lineage_status="not_applicable",
|
|
quarantine_status="not_quarantined",
|
|
status="ready",
|
|
)
|
|
dataset.source_registry = registry
|
|
dataset.source_snapshot = snapshot
|
|
version = DatasetVersion(
|
|
id=uuid4(),
|
|
dataset_id=dataset_id,
|
|
version=3,
|
|
checksum_sha256=checksum,
|
|
)
|
|
dataset.versions.append(version)
|
|
objects = {(Dataset, dataset_id): dataset}
|
|
if area is not None:
|
|
objects[(Area, area.id)] = area
|
|
return FakeSession(objects), dataset, area
|
|
|
|
|
|
def _manifest(tmp_path: Path, db: FakeSession, dataset: Dataset) -> Path:
|
|
tile_path = tmp_path / "tile_0000.tif"
|
|
tile_path.write_bytes(b"immutable tile")
|
|
payload = {
|
|
**TileManifestService.dataset_binding(db, dataset),
|
|
"tile_set_id": "tile-set-1",
|
|
"crs": "EPSG:4326",
|
|
"source_crs": "EPSG:4326",
|
|
"dataset_crs": "EPSG:4326",
|
|
"bounds": [4.0, 51.0, 5.0, 52.0],
|
|
"count": 1,
|
|
"tiles": [
|
|
{
|
|
"path": str(tile_path),
|
|
"bounds": [4.0, 51.0, 5.0, 52.0],
|
|
"crs": "EPSG:4326",
|
|
"index": 0,
|
|
**TileManifestService.tile_integrity(tile_path),
|
|
}
|
|
],
|
|
}
|
|
manifest_path = tmp_path / "manifest.json"
|
|
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
|
|
return manifest_path
|
|
|
|
|
|
def _validate(tmp_path: Path, db: FakeSession, dataset: Dataset, *, prefix: str = "DETECTION"):
|
|
manifest_path = tmp_path / "manifest.json"
|
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
return TileManifestService.validate_for_inference(
|
|
db,
|
|
dataset,
|
|
payload,
|
|
manifest_path=manifest_path,
|
|
settings=Settings(_env_file=None, storage_root=str(tmp_path)),
|
|
error_prefix=prefix,
|
|
)
|
|
|
|
|
|
def test_versioned_tile_manifest_binds_dataset_version_snapshot_area_and_tile_bytes(tmp_path: Path) -> None:
|
|
db, dataset, _area = _dataset_and_session()
|
|
manifest_path = _manifest(tmp_path, db, dataset)
|
|
|
|
evidence = _validate(tmp_path, db, dataset)
|
|
|
|
assert evidence["manifest_path"] == str(manifest_path.resolve())
|
|
assert evidence["source_dataset_id"] == str(dataset.id)
|
|
assert evidence["source_snapshot_id"] == str(dataset.source_snapshot_id)
|
|
assert evidence["dataset_version_id"] == str(dataset.versions[0].id)
|
|
assert evidence["source_area_id"] == str(dataset.area_id)
|
|
assert evidence["tile_count"] == 1
|
|
assert evidence["tile_union_bounds_epsg4326"] == pytest.approx([4.0, 51.0, 5.0, 52.0])
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("field", "value", "expected_code"),
|
|
[
|
|
("source_dataset_id", lambda: str(uuid4()), "DETECTION_TILE_MANIFEST_DATASET_MISMATCH"),
|
|
("source_dataset_checksum_sha256", lambda: "b" * 64, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
|
("source_snapshot_id", lambda: str(uuid4()), "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
|
("dataset_version", lambda: 99, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
|
("source_area_geometry_sha256", lambda: "c" * 64, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
|
],
|
|
)
|
|
def test_tile_manifest_rejects_dataset_or_provenance_mismatch(
|
|
tmp_path: Path,
|
|
field: str,
|
|
value,
|
|
expected_code: str,
|
|
) -> None:
|
|
db, dataset, _area = _dataset_and_session()
|
|
manifest_path = _manifest(tmp_path, db, dataset)
|
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
payload[field] = value()
|
|
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
_validate(tmp_path, db, dataset)
|
|
|
|
assert exc_info.value.code == expected_code
|
|
|
|
|
|
def test_tile_manifest_rejects_tile_bytes_changed_after_manifest_creation(tmp_path: Path) -> None:
|
|
db, dataset, _area = _dataset_and_session()
|
|
manifest_path = _manifest(tmp_path, db, dataset)
|
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
Path(payload["tiles"][0]["path"]).write_bytes(b"tampered tile")
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
_validate(tmp_path, db, dataset)
|
|
|
|
assert exc_info.value.code == "DETECTION_TILE_MANIFEST_TILE_INTEGRITY_MISMATCH"
|
|
|
|
|
|
def test_tile_manifest_rejects_dataset_without_version_binding(tmp_path: Path) -> None:
|
|
db, dataset, _area = _dataset_and_session()
|
|
_manifest(tmp_path, db, dataset)
|
|
dataset.versions.clear()
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
_validate(tmp_path, db, dataset)
|
|
|
|
assert exc_info.value.code == "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"
|
|
assert "dataset_version_id" in exc_info.value.details["missing_fields"]
|
|
|
|
|
|
def test_segmentation_tile_manifest_rejects_union_outside_dataset_scope(tmp_path: Path) -> None:
|
|
db, dataset, _area = _dataset_and_session(with_area=False)
|
|
manifest_path = _manifest(tmp_path, db, dataset)
|
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
payload["bounds"] = [4.0, 51.0, 6.0, 52.0]
|
|
payload["tiles"][0]["bounds"] = [4.0, 51.0, 6.0, 52.0]
|
|
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
_validate(tmp_path, db, dataset, prefix="SEGMENTATION")
|
|
|
|
assert exc_info.value.code == "SEGMENTATION_TILE_MANIFEST_SCOPE_MISMATCH"
|