287 lines
10 KiB
Python
287 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import gzip
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
from shapely.geometry import box, mapping, shape
|
|
from shapely.ops import transform as transform_geometry
|
|
|
|
from app.models import Dataset
|
|
from app.services.vector_feature_service import VectorFeatureService
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPTS = ROOT / "scripts"
|
|
if str(SCRIPTS) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
|
|
|
|
def load_script():
|
|
path = SCRIPTS / "provision_regional_soil_map.py"
|
|
spec = importlib.util.spec_from_file_location("test_provision_regional_soil_map", path)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
class FakeResponse:
|
|
status_code = 200
|
|
ok = True
|
|
text = ""
|
|
|
|
def __init__(self, payload):
|
|
self.payload = payload
|
|
|
|
def json(self):
|
|
return self.payload
|
|
|
|
|
|
class FakeApiSession:
|
|
def __init__(self, payload):
|
|
self.payload = payload
|
|
self.calls = []
|
|
|
|
def post(self, url, **kwargs):
|
|
self.calls.append((url, kwargs))
|
|
return FakeResponse({"data": self.payload})
|
|
|
|
|
|
def source_feature(feature_id: str, geometry) -> dict:
|
|
return {
|
|
"type": "Feature",
|
|
"id": feature_id,
|
|
"geometry": mapping(geometry),
|
|
"properties": {
|
|
"gid": 1,
|
|
"id_kaartvlak": 10,
|
|
"Bodemtype": "Zeg",
|
|
"Unibodemtype": "Zeg",
|
|
"Bodemserie": "Zeg",
|
|
"Beknopte_omschrijving_bodemserie": "Natte zandbodem",
|
|
"Gegeneraliseerde_legende": "Nat zand",
|
|
"Textuurklasse_code": "Z",
|
|
"Textuurklasse": "zand",
|
|
"Drainageklasse_code": "e",
|
|
"Drainageklasse": "nat",
|
|
"Profielontwikkelingsgroep_code": "g",
|
|
"Profielontwikkelingsgroep": "humus B horizont",
|
|
"Eenduidige_legende_titel": "bodemserie Zeg",
|
|
},
|
|
}
|
|
|
|
|
|
def test_normalized_regional_features_keep_member_identity_and_unique_ids() -> None:
|
|
module = load_script()
|
|
boundary = box(5.0, 51.0, 5.1, 51.1)
|
|
boundary_lambert72 = transform_geometry(module.soil.TO_LAMBERT72.transform, boundary)
|
|
feature = source_feature("bodemtypes.1", box(4.98, 51.02, 5.05, 51.08))
|
|
|
|
left, _ = module.soil.normalize_feature(
|
|
feature,
|
|
boundary_lambert72,
|
|
municipality="Mol",
|
|
nis_code="13025",
|
|
coverage_scope="test-region",
|
|
feature_id_suffix="13025",
|
|
)
|
|
right, _ = module.soil.normalize_feature(
|
|
feature,
|
|
boundary_lambert72,
|
|
municipality="Balen",
|
|
nis_code="13003",
|
|
coverage_scope="test-region",
|
|
feature_id_suffix="13003",
|
|
)
|
|
|
|
assert left is not None and right is not None
|
|
assert left["id"] == "bodemtypes.1:13025"
|
|
assert right["id"] == "bodemtypes.1:13003"
|
|
assert left["properties"]["municipality"] == "Mol"
|
|
assert left["properties"]["coverage_scope"] == "test-region"
|
|
assert shape(left["geometry"]).within(boundary.buffer(1e-7))
|
|
|
|
|
|
def test_partition_retains_gzipped_source_and_is_checksum_reusable(tmp_path: Path, monkeypatch) -> None:
|
|
module = load_script()
|
|
scope = module.GeographicScope(
|
|
key="test-region",
|
|
display_name="Test region",
|
|
project_name="Test",
|
|
project_region="Test",
|
|
area_name="Test area",
|
|
authority_name="Test",
|
|
authority_url="https://example.test",
|
|
scope_type="test",
|
|
limitation_message="Test",
|
|
members=(module.ScopeMember("Mol", "13025"),),
|
|
)
|
|
boundary = box(5.0, 51.0, 5.1, 51.1)
|
|
payload = {
|
|
"type": "FeatureCollection",
|
|
"features": [source_feature("bodemtypes.1", box(4.98, 51.02, 5.05, 51.08))],
|
|
}
|
|
raw_bytes = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
monkeypatch.setattr(
|
|
module.soil,
|
|
"iter_wfs_pages",
|
|
lambda *_args, **_kwargs: iter([(payload, "https://example.test/page", raw_bytes)]),
|
|
)
|
|
|
|
manifest = module.prepare_partition(
|
|
object(),
|
|
output_root=tmp_path,
|
|
scope=scope,
|
|
member=scope.members[0],
|
|
boundary_wgs84=boundary,
|
|
page_limit=100,
|
|
max_features=1000,
|
|
timeout=30,
|
|
force=False,
|
|
)
|
|
output = json.loads(Path(manifest["output_path"]).read_text(encoding="utf-8"))
|
|
raw_path = Path(manifest["manifest_path"]).parent / manifest["raw_pages"][0]["artifact_path"]
|
|
|
|
assert manifest["feature_count"] == 1
|
|
assert output["features"][0]["id"] == "bodemtypes.1:13025"
|
|
assert gzip.decompress(raw_path.read_bytes()) == raw_bytes
|
|
|
|
cached = module.prepare_partition(
|
|
object(),
|
|
output_root=tmp_path,
|
|
scope=scope,
|
|
member=scope.members[0],
|
|
boundary_wgs84=boundary,
|
|
page_limit=100,
|
|
max_features=1000,
|
|
timeout=30,
|
|
force=False,
|
|
)
|
|
assert cached["output_sha256"] == manifest["output_sha256"]
|
|
|
|
|
|
def test_snapshot_assembles_all_partitions_with_governed_area_metrics(tmp_path: Path) -> None:
|
|
module = load_script()
|
|
scope = module.GeographicScope(
|
|
key="test-region",
|
|
display_name="Test region",
|
|
project_name="Test",
|
|
project_region="Test",
|
|
area_name="Test area",
|
|
authority_name="Test",
|
|
authority_url="https://example.test",
|
|
scope_type="test",
|
|
limitation_message="Test",
|
|
members=(module.ScopeMember("Left", "10001"), module.ScopeMember("Right", "10002")),
|
|
)
|
|
partitions = []
|
|
for index, member in enumerate(scope.members):
|
|
output_path, manifest_path, _raw_dir = module.partition_paths(tmp_path / scope.key, member.nis_code)
|
|
feature = source_feature(f"bodemtypes.{index}", box(5.0 + index * 0.1, 51.0, 5.05 + index * 0.1, 51.05))
|
|
feature["id"] = f"bodemtypes.{index}:{member.nis_code}"
|
|
feature["properties"].update(
|
|
{
|
|
"clipped_area_ha": 1.0 + index,
|
|
"soil_generalized_legend": "Nat zand",
|
|
"soil_texture_class": "zand",
|
|
"soil_drainage_class": "nat",
|
|
}
|
|
)
|
|
module.soil.write_json_atomic(output_path, {"type": "FeatureCollection", "features": [feature]})
|
|
partitions.append(
|
|
{
|
|
"municipality": member.name,
|
|
"nis_code": member.nis_code,
|
|
"feature_count": 1,
|
|
"raw_source_feature_count": 1,
|
|
"page_count": 1,
|
|
"output_path": str(output_path),
|
|
"output_sha256": module.soil.sha256_file(output_path),
|
|
"manifest_path": str(manifest_path),
|
|
}
|
|
)
|
|
|
|
output_path, _manifest_path, manifest = module.assemble_snapshot(
|
|
output_root=tmp_path,
|
|
scope=scope,
|
|
partitions=partitions,
|
|
member_boundaries_sha256="boundaries-hash",
|
|
max_total_features=10,
|
|
)
|
|
output = json.loads(output_path.read_text(encoding="utf-8"))
|
|
|
|
assert manifest["coverage_complete"] is True
|
|
assert manifest["feature_count"] == 2
|
|
assert manifest["area_by_generalized_legend_ha"]["Nat zand"] == 3.0
|
|
assert manifest["area_by_texture_ha"]["zand"] == 3.0
|
|
assert len({feature["id"] for feature in output["features"]}) == 2
|
|
|
|
|
|
def test_upload_contract_is_regional_historical_and_canonical(tmp_path: Path) -> None:
|
|
module = load_script()
|
|
scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]
|
|
path = tmp_path / "soil.geojson"
|
|
path.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
|
|
manifest_path = tmp_path / "manifest.json"
|
|
manifest = {
|
|
"coverage_complete": True,
|
|
"feature_count": 42,
|
|
"output_sha256": "output-hash",
|
|
"partition_identity_sha256": "partition-hash",
|
|
"partitions": [{} for _ in scope.members],
|
|
"generated_at": "2026-07-16T00:00:00+00:00",
|
|
"limitations": ["historical"],
|
|
}
|
|
session = FakeApiSession({"id": "dataset-id", "feature_count": 42})
|
|
|
|
result = module.upload_snapshot(
|
|
session,
|
|
base_url="http://backend:8000",
|
|
project_id="project-id",
|
|
area_id="area-id",
|
|
scope=scope,
|
|
path=path,
|
|
manifest_path=manifest_path,
|
|
manifest=manifest,
|
|
timeout=30,
|
|
)
|
|
data = session.calls[0][1]["data"]
|
|
source_metadata = json.loads(data["source_metadata_json"])
|
|
provenance = json.loads(data["provenance_metadata_json"])
|
|
|
|
assert result["id"] == "dataset-id"
|
|
assert data["area_id"] == "area-id"
|
|
assert data["temporal_series_key"] == "dov:digital-soil-map:kempen-transport-region"
|
|
assert data["valid_from"] == module.soil.VALID_FROM
|
|
assert data["valid_to"] == module.soil.VALID_TO
|
|
assert source_metadata["coverage_scope"] == "kempen-transport-region"
|
|
assert source_metadata["member_count"] == 28
|
|
assert source_metadata["authority_level"] == "authoritative_historical_baseline"
|
|
assert provenance["operator_tool"] == "provision_regional_soil_map.py"
|
|
assert provenance["raw_source_responses_retained"] is True
|
|
|
|
|
|
def test_regional_operator_is_packaged_release_checked_and_keeps_exact_intersections() -> None:
|
|
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
|
readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8")
|
|
service = (ROOT / "backend/app/services/vector_feature_service.py").read_text(encoding="utf-8")
|
|
|
|
assert "COPY scripts/provision_regional_soil_map.py" in dockerfile
|
|
assert "py_compile scripts/provision_regional_soil_map.py" in readiness
|
|
assert '"provision_regional_soil_map.py"' not in service
|
|
|
|
dataset = Dataset(
|
|
name="regional-soil.geojson",
|
|
dataset_type="vector",
|
|
status="ready",
|
|
source_metadata={"partitioned_source_audit": True, "geometry_clipped_to_area": True},
|
|
provenance_metadata={"operator_tool": "provision_regional_soil_map.py"},
|
|
)
|
|
assert VectorFeatureService.can_use_full_area_fast_path(dataset, None) is False
|
|
assert VectorFeatureService.preclipped_partition_filter(dataset, "Gemeente Mol - officiele grens") is None
|
|
assert VectorFeatureService.preclipped_partition_filter(dataset, "Vervoerregio Kempen") is None
|