diff --git a/backend/tests/test_building_label_normalization.py b/backend/tests/test_building_label_normalization.py index f8e9c5b8..2e4de13c 100644 --- a/backend/tests/test_building_label_normalization.py +++ b/backend/tests/test_building_label_normalization.py @@ -15,6 +15,13 @@ assert SPEC and SPEC.loader module = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(module) +ASSEMBLER_SPEC = importlib.util.spec_from_file_location( + "assemble_building_corpus", ROOT / "scripts" / "assemble_belgium_building_corpus.py" +) +assert ASSEMBLER_SPEC and ASSEMBLER_SPEC.loader +assembler = importlib.util.module_from_spec(ASSEMBLER_SPEC) +ASSEMBLER_SPEC.loader.exec_module(assembler) + def test_normalizer_retains_native_identity_and_records_rejections(tmp_path: Path) -> None: raster_path = tmp_path / "image.tif" @@ -69,3 +76,15 @@ def test_normalizer_retains_native_identity_and_records_rejections(tmp_path: Pat "excluded_canopy": 1, } assert audit["temporal_mismatch_days"] == 31 + + +def test_spatial_leakage_audit_fails_cross_split_neighbors() -> None: + samples = [ + {"sample_slug": "train-a", "split": "train", "bbox_epsg4326": [4.0, 50.0, 4.01, 50.01]}, + {"sample_slug": "val-a", "split": "val", "bbox_epsg4326": [4.005, 50.005, 4.02, 50.02]}, + {"sample_slug": "test-far", "split": "test", "bbox_epsg4326": [5.0, 51.0, 5.01, 51.01]}, + ] + audit = assembler.audit_spatial_leakage(samples) + assert audit["status"] == "failed" + assert audit["findings"][0]["left"] == "train-a" + assert audit["findings"][0]["right"] == "val-a" diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index c06569ab..60b1ba34 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11467,3 +11467,34 @@ Next gate: - Verification: frontend build and 78 focused contracts passed. The full backend run reported 1124 passed plus five unrelated local Windows/WSL `bash` path failures. +## 2026-07-26 - Belgian building corpus candidate and CUDA matrix + +- Added provider-aware canonical label normalization for GRB, PICC and UrbIS. + Every source feature retains its native ID/class and receives an explicit + accept/reject reason covering geometry repair, duplicates, semantic + exclusions and resolvable pixel size. +- Added an immutable corpus assembler that resolves only persisted governed + Dataset IDs, validates regional provider pairing, copies checksum-bound + artifacts and refuses a non-empty output directory. +- Frozen experimental corpus `building-be-v1-candidate-20260726` contains 19 + geographically separated AOIs spanning all three land regions and explicit + train, validation, calibration, test and background-test roles. Manifest + SHA-256 is `0450ce782c35c5955e519fae489ffdbef6075d871adb0d196cc6a244571788fa`. +- The training export contains 36 tiles and 7,219 tile-level labels; its + automated label-size, variance and split audit passed. A 12-tile regional + pilot contact sheet was rendered for human review. +- CUDA training on the RTX 4080 SUPER completed a generic YOLOv8s candidate + (`mAP50=0.0883`, `mAP50-95=0.0248`) and an incumbent fine-tune + (`mAP50=0.187`, `mAP50-95=0.0617`) on the held-out regional validation + samples. Both are immutable candidates and neither is promoted. +- A 60-epoch incumbent fine-tune completed with artifact SHA-256 + `594f9fef356940e7f7839da36561387a9a2f436c1e498e5733f99e1170c22fa6`. + On the unopened regional test AOIs (Leuven, Mons and Brussels rail) it + achieved precision `0.315`, recall `0.229`, mAP50 `0.120` and mAP50-95 + `0.0340`. The incumbent scored `0.227`, `0.233`, `0.0930` and `0.0262` + respectively. The challenger improves precision/AP but slightly lowers + recall and remains far below a credible national acceptance floor. +- The scores prove that the current small candidate corpus is insufficient for + a national production claim. Human review, broader negative coverage, + leakage audit and independent calibration/test evaluation remain blocking + gates; the active production asset was left unchanged. diff --git a/docs/PYTORCH_TRAINING_ROADMAP_BELGIUM.md b/docs/PYTORCH_TRAINING_ROADMAP_BELGIUM.md index d8b15a86..3fff0eb5 100644 --- a/docs/PYTORCH_TRAINING_ROADMAP_BELGIUM.md +++ b/docs/PYTORCH_TRAINING_ROADMAP_BELGIUM.md @@ -121,21 +121,21 @@ manual URL editing and without crossing provider coverage zones. ### WP2 — Reference-label normalization -- [ ] normalize GRB, PICC and UrbIS building semantics to canonical `building`; -- [ ] retain original feature IDs and source classes; -- [ ] quantify invalid geometry repairs, duplicates and temporal mismatches; -- [ ] define exclusion rules for ruins, canopies, underground structures and +- [x] normalize GRB, PICC and UrbIS building semantics to canonical `building`; +- [x] retain original feature IDs and source classes; +- [x] quantify invalid geometry repairs, duplicates and temporal mismatches; +- [x] define exclusion rules for ruins, canopies, underground structures and objects below the resolvable pixel size; -- [ ] generate overlay/contact-sheet review packs. +- [x] generate overlay/contact-sheet review packs. Exit: every accepted/rejected label has a machine-readable reason and provenance. ### WP3 — Frozen Belgium building corpus -- [ ] select and persist the geographic strata and split assignments; -- [ ] acquire imagery and references through governed services; -- [ ] export tiles at candidate resolutions with deterministic negative sampling; -- [ ] run leakage, checksum, class, label-size and visual-variance audits; +- [x] select and persist the initial geographic strata and split assignments; +- [x] acquire the candidate imagery and references through governed services; +- [x] export tiles at candidate resolutions with deterministic negative sampling; +- [x] run checksum, class, label-size and visual-variance audits; - [ ] complete representative human label review; - [ ] freeze dataset version `building-be-v1` and make it immutable. diff --git a/docs/TODO.md b/docs/TODO.md index a3aa800e..c9be84d9 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -940,3 +940,13 @@ This file now starts with the current implementation status. Older preparation/b - [ ] Resolve governed North Sea bathymetry access without bypassing TLS. - [x] Enforce the active building-only model classes, CUDA readiness and persisted Mol/Kempen Area scope. - [ ] Add national/regional holdout evidence before expanding model classes or validated Areas. +## Belgian PyTorch training programme (2026-07-26) + +- [x] Add governed SPW and UrbIS orthophoto acquisition. +- [x] Normalize GRB/PICC/UrbIS building labels with decision provenance. +- [x] Assemble and checksum an initial 19-AOI Belgian candidate corpus. +- [x] Run generic and incumbent-based CUDA candidate training without promotion. +- [ ] Complete representative human label/contact-sheet review. +- [ ] Expand each region/context/split until the national minimum-composition gate passes. +- [ ] Run explicit spatial leakage and independent calibration/test evaluation. +- [ ] Promote only if every regional and pure-background gate passes. diff --git a/scripts/assemble_belgium_building_corpus.py b/scripts/assemble_belgium_building_corpus.py index fff3e01e..aedfa754 100644 --- a/scripts/assemble_belgium_building_corpus.py +++ b/scripts/assemble_belgium_building_corpus.py @@ -7,14 +7,25 @@ import argparse import hashlib import json import shutil +import sys from pathlib import Path from typing import Any from uuid import UUID -from app.db.session import SessionLocal -from app.models import Dataset +from pyproj import Transformer +from shapely.geometry import box +from shapely.ops import transform as shapely_transform -from normalize_belgium_building_labels import normalize +REPO_ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = REPO_ROOT if (REPO_ROOT / "app").is_dir() else REPO_ROOT / "backend" +for import_root in (Path(__file__).resolve().parent, APP_ROOT): + if str(import_root) not in sys.path: + sys.path.insert(0, str(import_root)) + +from app.db.session import SessionLocal # noqa: E402 +from app.models import Dataset # noqa: E402 + +from normalize_belgium_building_labels import normalize # noqa: E402 REGION_SOURCES = { "flanders": ("digitaal_vlaanderen_orthophoto", "grb"), @@ -58,6 +69,33 @@ def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset) return region, expected_reference +def audit_spatial_leakage(samples: list[dict[str, Any]], buffer_m: float = 64.0) -> dict[str, Any]: + transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) + findings: list[dict[str, Any]] = [] + metric_boxes: list[tuple[dict[str, Any], Any]] = [] + for sample in samples: + bounds = sample.get("bbox_epsg4326") + if not isinstance(bounds, list) or len(bounds) != 4: + raise SystemExit(f"Missing governed bbox for leakage audit: {sample['sample_slug']}") + metric_boxes.append((sample, shapely_transform(transformer.transform, box(*map(float, bounds))))) + for index, (left, left_geometry) in enumerate(metric_boxes): + for right, right_geometry in metric_boxes[index + 1 :]: + if left["split"] == right["split"]: + continue + distance_m = left_geometry.distance(right_geometry) + if distance_m < buffer_m: + findings.append( + { + "left": left["sample_slug"], + "left_split": left["split"], + "right": right["sample_slug"], + "right_split": right["split"], + "distance_m": distance_m, + } + ) + return {"status": "ok" if not findings else "failed", "buffer_m": buffer_m, "findings": findings} + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--spec", type=Path, required=True) @@ -137,6 +175,12 @@ def main() -> int: } manifest_path = output_dir / "operator_samples_manifest.json" manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + leakage_audit = audit_spatial_leakage(manifest_samples) + (output_dir / "spatial-leakage-audit.json").write_text( + json.dumps(leakage_audit, ensure_ascii=False, indent=2), encoding="utf-8" + ) + if leakage_audit["status"] != "ok": + raise SystemExit("Spatial split leakage audit failed") freeze = { "dataset_version": args.version, "manifest_sha256": sha256(manifest_path),