Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
262 lines
10 KiB
Python
262 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import rasterio
|
|
from rasterio.transform import from_origin
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts" / "normalize_belgium_building_labels.py"
|
|
SPEC = importlib.util.spec_from_file_location("normalize_buildings", SCRIPT)
|
|
assert SPEC and SPEC.loader
|
|
module = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(module)
|
|
|
|
ASSEMBLER_SPEC = importlib.util.spec_from_file_location(
|
|
"assemble_building_corpus", ROOT / "scripts" / "assemble_belgium_building_corpus.py"
|
|
)
|
|
assert ASSEMBLER_SPEC and ASSEMBLER_SPEC.loader
|
|
assembler = importlib.util.module_from_spec(ASSEMBLER_SPEC)
|
|
ASSEMBLER_SPEC.loader.exec_module(assembler)
|
|
|
|
|
|
def test_normalizer_retains_native_identity_and_records_rejections(tmp_path: Path) -> None:
|
|
raster_path = tmp_path / "image.tif"
|
|
with rasterio.open(
|
|
raster_path,
|
|
"w",
|
|
driver="GTiff",
|
|
width=100,
|
|
height=100,
|
|
count=3,
|
|
dtype="uint8",
|
|
crs="EPSG:4326",
|
|
transform=from_origin(4.0, 51.0, 0.001, 0.001),
|
|
) as dataset:
|
|
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
|
|
valid = {
|
|
"type": "Feature",
|
|
"id": "native-1",
|
|
"properties": {"TYPE": "main building"},
|
|
"geometry": {"type": "Polygon", "coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]]},
|
|
}
|
|
duplicate = json.loads(json.dumps(valid))
|
|
duplicate["id"] = "native-2"
|
|
canopy = json.loads(json.dumps(valid))
|
|
canopy["id"] = "native-3"
|
|
canopy["properties"]["TYPE"] = "canopy"
|
|
tiny = json.loads(json.dumps(valid))
|
|
tiny["id"] = "native-4"
|
|
tiny["geometry"] = {"type": "Polygon", "coordinates": [[[4.03, 50.97], [4.031, 50.97], [4.031, 50.969], [4.03, 50.969], [4.03, 50.97]]]}
|
|
reference_path = tmp_path / "reference.geojson"
|
|
reference_path.write_text(json.dumps({"type": "FeatureCollection", "features": [valid, duplicate, canopy, tiny]}), encoding="utf-8")
|
|
|
|
normalized, audit = module.normalize(
|
|
reference_path=reference_path,
|
|
raster_path=raster_path,
|
|
source_name="urbis",
|
|
min_label_px=3,
|
|
imagery_observed_at="2026-01-01T00:00:00Z",
|
|
reference_observed_at="2025-12-01T00:00:00Z",
|
|
)
|
|
|
|
assert len(normalized["features"]) == 1
|
|
properties = normalized["features"][0]["properties"]
|
|
assert properties["canonical_class"] == "building"
|
|
assert properties["source_name"] == "urbis"
|
|
assert properties["source_feature_id"] == "native-1"
|
|
assert properties["source_class"] == "main building"
|
|
assert audit["decision_counts"] == {
|
|
"accepted": 1,
|
|
"below_resolvable_pixel_size": 1,
|
|
"duplicate_geometry": 1,
|
|
"excluded_canopy": 1,
|
|
}
|
|
assert audit["temporal_mismatch_days"] == 31
|
|
|
|
|
|
def test_spatial_leakage_audit_fails_cross_split_neighbors() -> None:
|
|
samples = [
|
|
{"sample_slug": "train-a", "split": "train", "bbox_epsg4326": [4.0, 50.0, 4.01, 50.01]},
|
|
{"sample_slug": "val-a", "split": "val", "bbox_epsg4326": [4.005, 50.005, 4.02, 50.02]},
|
|
{"sample_slug": "test-far", "split": "test", "bbox_epsg4326": [5.0, 51.0, 5.01, 51.01]},
|
|
]
|
|
audit = assembler.audit_spatial_leakage(samples)
|
|
assert audit["status"] == "failed"
|
|
assert audit["findings"][0]["left"] == "train-a"
|
|
assert audit["findings"][0]["right"] == "val-a"
|
|
|
|
|
|
def test_normalizer_rejects_features_created_after_dated_imagery(tmp_path: Path) -> None:
|
|
raster_path = tmp_path / "image.tif"
|
|
with rasterio.open(
|
|
raster_path,
|
|
"w",
|
|
driver="GTiff",
|
|
width=100,
|
|
height=100,
|
|
count=3,
|
|
dtype="uint8",
|
|
crs="EPSG:4326",
|
|
transform=from_origin(4.0, 51.0, 0.001, 0.001),
|
|
) as dataset:
|
|
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
|
|
geometry = {
|
|
"type": "Polygon",
|
|
"coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]],
|
|
}
|
|
reference_path = tmp_path / "reference.geojson"
|
|
reference_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{"type": "Feature", "id": "old", "properties": {"BEGINDATUM": "2024-01-01"}, "geometry": geometry},
|
|
{"type": "Feature", "id": "new", "properties": {"BEGINDATUM": "2026-01-01"}, "geometry": geometry},
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
normalized, audit = module.normalize(
|
|
reference_path=reference_path,
|
|
raster_path=raster_path,
|
|
source_name="grb",
|
|
min_label_px=3,
|
|
imagery_observed_at="2025-01-01T00:00:00Z",
|
|
imagery_valid_to="2025-12-31T23:59:59Z",
|
|
reference_observed_at="2026-07-01T00:00:00Z",
|
|
)
|
|
assert len(normalized["features"]) == 1
|
|
assert audit["decision_counts"] == {"accepted": 1, "created_after_imagery_period": 1}
|
|
|
|
|
|
def test_normalizer_rejects_same_year_feature_after_annual_mosaic_start(tmp_path: Path) -> None:
|
|
raster_path = tmp_path / "image.tif"
|
|
with rasterio.open(
|
|
raster_path, "w", driver="GTiff", width=100, height=100, count=3,
|
|
dtype="uint8", crs="EPSG:4326", transform=from_origin(4.0, 51.0, 0.001, 0.001),
|
|
) as dataset:
|
|
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
|
|
geometry = {
|
|
"type": "Polygon",
|
|
"coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]],
|
|
}
|
|
reference_path = tmp_path / "reference.geojson"
|
|
reference_path.write_text(json.dumps({
|
|
"type": "FeatureCollection",
|
|
"features": [{"type": "Feature", "properties": {"BEGINDATUM": "2025-08-18"}, "geometry": geometry}],
|
|
}), encoding="utf-8")
|
|
|
|
normalized, audit = module.normalize(
|
|
reference_path=reference_path,
|
|
raster_path=raster_path,
|
|
source_name="grb",
|
|
min_label_px=3,
|
|
imagery_observed_at="2025-01-01T00:00:00Z",
|
|
imagery_valid_to="2025-12-31T23:59:59Z",
|
|
reference_observed_at="2026-07-01T00:00:00Z",
|
|
)
|
|
|
|
assert normalized["features"] == []
|
|
assert audit["decision_counts"] == {"created_after_imagery_period": 1}
|
|
assert audit["imagery_feature_creation_cutoff"] == "2025-01-01T00:00:00+00:00"
|
|
|
|
|
|
def test_normalizer_applies_provider_native_source_class_allowlist(tmp_path: Path) -> None:
|
|
raster_path = tmp_path / "image.tif"
|
|
with rasterio.open(
|
|
raster_path, "w", driver="GTiff", width=100, height=100, count=3,
|
|
dtype="uint8", crs="EPSG:4326", transform=from_origin(4.0, 51.0, 0.001, 0.001),
|
|
) as dataset:
|
|
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
|
|
def feature(feature_id: str, source_type: int, left: float) -> dict:
|
|
return {
|
|
"type": "Feature", "id": feature_id, "properties": {"TYPE": source_type},
|
|
"geometry": {"type": "Polygon", "coordinates": [[[left, 50.99], [left + .01, 50.99], [left + .01, 50.98], [left, 50.98], [left, 50.99]]]},
|
|
}
|
|
reference_path = tmp_path / "reference.geojson"
|
|
reference_path.write_text(json.dumps({"type": "FeatureCollection", "features": [feature("main", 1, 4.01), feature("annex", 2, 4.03)]}), encoding="utf-8")
|
|
|
|
normalized, audit = module.normalize(
|
|
reference_path=reference_path, raster_path=raster_path, source_name="grb",
|
|
min_label_px=3, imagery_observed_at=None, reference_observed_at=None,
|
|
allowed_source_classes={"1"},
|
|
)
|
|
|
|
assert [item["id"] for item in normalized["features"]] == ["grb:main"]
|
|
assert audit["allowed_source_classes"] == ["1"]
|
|
assert audit["decision_counts"] == {"accepted": 1, "source_class_not_allowed": 1}
|
|
|
|
|
|
def test_normalizer_merges_only_touching_visible_roof_instances(tmp_path: Path) -> None:
|
|
raster_path = tmp_path / "image.tif"
|
|
with rasterio.open(
|
|
raster_path,
|
|
"w",
|
|
driver="GTiff",
|
|
width=100,
|
|
height=100,
|
|
count=3,
|
|
dtype="uint8",
|
|
crs="EPSG:4326",
|
|
transform=from_origin(4.0, 51.0, 0.001, 0.001),
|
|
) as dataset:
|
|
dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
|
|
polygons = [
|
|
[[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]],
|
|
[[[4.02, 50.99], [4.03, 50.99], [4.03, 50.98], [4.02, 50.98], [4.02, 50.99]]],
|
|
[[[4.04, 50.99], [4.05, 50.99], [4.05, 50.98], [4.04, 50.98], [4.04, 50.99]]],
|
|
]
|
|
reference_path = tmp_path / "reference.geojson"
|
|
reference_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{"type": "Feature", "id": str(index), "properties": {}, "geometry": {"type": "Polygon", "coordinates": coordinates}}
|
|
for index, coordinates in enumerate(polygons)
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
normalized, audit = module.normalize(
|
|
reference_path=reference_path,
|
|
raster_path=raster_path,
|
|
source_name="urbis",
|
|
min_label_px=3,
|
|
imagery_observed_at=None,
|
|
reference_observed_at=None,
|
|
merge_touching_roofs=True,
|
|
)
|
|
assert len(normalized["features"]) == 2
|
|
assert sorted(item["properties"]["source_feature_count"] for item in normalized["features"]) == [1, 2]
|
|
assert audit["accepted_source_feature_count"] == 3
|
|
assert audit["accepted_feature_count"] == 2
|
|
|
|
|
|
def test_visible_roof_merge_retains_large_touching_chains() -> None:
|
|
features = []
|
|
for index in range(13):
|
|
left = float(index)
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"id": str(index),
|
|
"properties": {"source_feature_id": str(index)},
|
|
"geometry": {
|
|
"type": "Polygon",
|
|
"coordinates": [[[left, 0], [left + 1, 0], [left + 1, 1], [left, 1], [left, 0]]],
|
|
},
|
|
}
|
|
)
|
|
merged = module.merge_touching_roof_instances(features, "grb")
|
|
assert len(merged) == 13
|
|
assert {item["properties"]["label_semantics"] for item in merged} == {
|
|
"native_instance_complex_touch_group"
|
|
}
|