fix: read signed WALOUS rasters safely
This commit is contained in:
@@ -247,7 +247,10 @@ class WalousLandCoverService:
|
|||||||
band = source.read(1, window=window, out_shape=(height, width), masked=True, resampling=Resampling.nearest)
|
band = source.read(1, window=window, out_shape=(height, width), masked=True, resampling=Resampling.nearest)
|
||||||
output_transform = from_bounds(*bounds, width, height)
|
output_transform = from_bounds(*bounds, width, height)
|
||||||
outside_scope = geometry_mask([mapping(clipped_geometry)], out_shape=(height, width), transform=output_transform, invert=False)
|
outside_scope = geometry_mask([mapping(clipped_geometry)], out_shape=(height, width), transform=output_transform, invert=False)
|
||||||
raw = np.asarray(band.filled(WalousLandCoverService.NODATA), dtype="uint8")
|
# The official 2023 GeoTIFF is signed int8 while GDAL exposes
|
||||||
|
# its nodata sentinel as 255. Filling before widening would
|
||||||
|
# therefore reject the sentinel as out of range for int8.
|
||||||
|
raw = np.asarray(np.ma.getdata(band), dtype="uint8")
|
||||||
invalid = np.ma.getmaskarray(band) | outside_scope
|
invalid = np.ma.getmaskarray(band) | outside_scope
|
||||||
if source.nodata is not None:
|
if source.nodata is not None:
|
||||||
invalid |= np.isclose(raw.astype("float64"), float(source.nodata))
|
invalid |= np.isclose(raw.astype("float64"), float(source.nodata))
|
||||||
@@ -476,7 +479,7 @@ class WalousLandCoverService:
|
|||||||
raise AppError(code="WALOUS_SELECTION_OUTSIDE_DATASET", message="Selection does not overlap the persisted WALOUS raster", status_code=422)
|
raise AppError(code="WALOUS_SELECTION_OUTSIDE_DATASET", message="Selection does not overlap the persisted WALOUS raster", status_code=422)
|
||||||
clipped, transform = mask(source, [mapping(geometry)], crop=True, filled=False, indexes=[1])
|
clipped, transform = mask(source, [mapping(geometry)], crop=True, filled=False, indexes=[1])
|
||||||
band = np.ma.asarray(clipped[0])
|
band = np.ma.asarray(clipped[0])
|
||||||
raw = np.asarray(band.filled(WalousLandCoverService.NODATA), dtype="uint8")
|
raw = np.asarray(np.ma.getdata(band), dtype="uint8")
|
||||||
selected = geometry_mask([mapping(geometry)], out_shape=raw.shape, transform=transform, invert=True)
|
selected = geometry_mask([mapping(geometry)], out_shape=raw.shape, transform=transform, invert=True)
|
||||||
valid = selected & ~np.ma.getmaskarray(band) & (raw != WalousLandCoverService.NODATA)
|
valid = selected & ~np.ma.getmaskarray(band) & (raw != WalousLandCoverService.NODATA)
|
||||||
values = raw[valid]
|
values = raw[valid]
|
||||||
@@ -553,7 +556,7 @@ class WalousLandCoverService:
|
|||||||
scale = min(1.0, max_dimension / max(source.width, source.height))
|
scale = min(1.0, max_dimension / max(source.width, source.height))
|
||||||
width, height = max(1, round(source.width * scale)), max(1, round(source.height * scale))
|
width, height = max(1, round(source.width * scale)), max(1, round(source.height * scale))
|
||||||
values = source.read(1, out_shape=(height, width), masked=True, resampling=Resampling.nearest)
|
values = source.read(1, out_shape=(height, width), masked=True, resampling=Resampling.nearest)
|
||||||
raw = np.asarray(values.filled(WalousLandCoverService.NODATA), dtype="uint8")
|
raw = np.asarray(np.ma.getdata(values), dtype="uint8")
|
||||||
rgba = np.zeros((height, width, 4), dtype="uint8")
|
rgba = np.zeros((height, width, 4), dtype="uint8")
|
||||||
for value, color in WalousLandCoverService.CLASS_COLORS.items():
|
for value, color in WalousLandCoverService.CLASS_COLORS.items():
|
||||||
selected = raw == value
|
selected = raw == value
|
||||||
|
|||||||
@@ -75,13 +75,18 @@ class FakeSession:
|
|||||||
return row
|
return row
|
||||||
|
|
||||||
|
|
||||||
def make_source(path: Path) -> tuple[list[float], np.ndarray]:
|
def make_source(
|
||||||
|
path: Path,
|
||||||
|
*,
|
||||||
|
dtype: str = "uint8",
|
||||||
|
nodata: int = 255,
|
||||||
|
) -> tuple[list[float], np.ndarray]:
|
||||||
to_3812 = Transformer.from_crs("EPSG:4326", "EPSG:3812", always_xy=True)
|
to_3812 = Transformer.from_crs("EPSG:4326", "EPSG:3812", always_xy=True)
|
||||||
to_4326 = Transformer.from_crs("EPSG:3812", "EPSG:4326", always_xy=True)
|
to_4326 = Transformer.from_crs("EPSG:3812", "EPSG:4326", always_xy=True)
|
||||||
x, y = to_3812.transform(4.85, 50.45)
|
x, y = to_3812.transform(4.85, 50.45)
|
||||||
transform = from_origin(x, y + 100, 1, 1)
|
transform = from_origin(x, y + 100, 1, 1)
|
||||||
class_codes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90]
|
class_codes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90]
|
||||||
values = np.empty((100, len(class_codes) * 20), dtype="uint8")
|
values = np.empty((100, len(class_codes) * 20), dtype=dtype)
|
||||||
for index, class_code in enumerate(class_codes):
|
for index, class_code in enumerate(class_codes):
|
||||||
values[:, index * 20 : (index + 1) * 20] = class_code
|
values[:, index * 20 : (index + 1) * 20] = class_code
|
||||||
with rasterio.open(
|
with rasterio.open(
|
||||||
@@ -91,10 +96,10 @@ def make_source(path: Path) -> tuple[list[float], np.ndarray]:
|
|||||||
width=values.shape[1],
|
width=values.shape[1],
|
||||||
height=100,
|
height=100,
|
||||||
count=1,
|
count=1,
|
||||||
dtype="uint8",
|
dtype=dtype,
|
||||||
crs="EPSG:3812",
|
crs="EPSG:3812",
|
||||||
transform=transform,
|
transform=transform,
|
||||||
nodata=255,
|
nodata=nodata,
|
||||||
) as target:
|
) as target:
|
||||||
target.write(values, 1)
|
target.write(values, 1)
|
||||||
min_lon, min_lat = to_4326.transform(x, y)
|
min_lon, min_lat = to_4326.transform(x, y)
|
||||||
@@ -169,6 +174,40 @@ def test_walous_acquisition_reads_real_classes_and_persists_provenance(tmp_path:
|
|||||||
assert captured["valid_to"] == captured["observed_at"]
|
assert captured["valid_to"] == captured["observed_at"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_walous_acquisition_accepts_official_signed_int8_nodata(tmp_path: Path, monkeypatch) -> None:
|
||||||
|
bbox, _values = make_source(
|
||||||
|
tmp_path / "walous_land_cover_2023_3812.tif",
|
||||||
|
dtype="int8",
|
||||||
|
nodata=-128,
|
||||||
|
)
|
||||||
|
project = Project(id=uuid4(), name="Belgium")
|
||||||
|
output_id = uuid4()
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
def persist(_db, **kwargs):
|
||||||
|
captured.update(kwargs)
|
||||||
|
return SimpleNamespace(id=output_id)
|
||||||
|
|
||||||
|
monkeypatch.setattr(DatasetService, "import_raster_bytes", persist)
|
||||||
|
result = WalousLandCoverService.acquire(
|
||||||
|
FakeSession(project),
|
||||||
|
project.id,
|
||||||
|
ThematicRasterAcquireRequest(
|
||||||
|
bbox={"min_x": bbox[0], "min_y": bbox[1], "max_x": bbox[2], "max_y": bbox[3], "crs": "EPSG:4326"},
|
||||||
|
product_key="walous_land_cover_2023",
|
||||||
|
force_refresh=True,
|
||||||
|
),
|
||||||
|
settings=settings(tmp_path),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["output_dataset_id"] == str(output_id)
|
||||||
|
assert captured["source_metadata"]["classes_present"] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90]
|
||||||
|
with rasterio.MemoryFile(captured["content"]) as memory:
|
||||||
|
with memory.open() as derived:
|
||||||
|
assert derived.dtypes == ("uint8",)
|
||||||
|
assert derived.nodata == 255
|
||||||
|
|
||||||
|
|
||||||
def test_walous_analysis_returns_semantic_area_metrics(tmp_path: Path, monkeypatch) -> None:
|
def test_walous_analysis_returns_semantic_area_metrics(tmp_path: Path, monkeypatch) -> None:
|
||||||
bbox, _values = make_source(tmp_path / "walous_land_cover_2023_3812.tif")
|
bbox, _values = make_source(tmp_path / "walous_land_cover_2023_3812.tif")
|
||||||
project = Project(id=uuid4(), name="Belgium")
|
project = Project(id=uuid4(), name="Belgium")
|
||||||
|
|||||||
Reference in New Issue
Block a user