diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fd0f490..892fba9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ # Changelog +## Sprint 131 Operator sample expansion and negative-tile YOLO candidate (2026-07-07) + +- Expanded `scripts/prepare_operator_real_data_samples.py` from the original Geel/Mol/Turnhout corpus to 7 reference AOIs plus 3 background-candidate AOIs. +- Added `sample_role` and `allow_empty_reference` metadata so deliberate background candidates can be prepared without weakening the empty-GRB guard for normal reference samples. +- Added regression coverage in `backend/tests/test_sprint131_operator_sample_expansion.py` for the expanded sample registry, empty-reference background candidates and normal reference-sample rejection. +- Live Tower preparation produced 10 operator samples: Geel, Mol, Turnhout, Herentals, Balen, Retie, Westerlo, Postel-bos, Lommel-heide and Kasterlee-bos. +- Live Tower tile export produced `/app/storage/operator-data/yolo-building-tile-expanded160` with 360 tiles, 260 positive tiles, 100 negative tiles and 11213 clipped building labels. +- Live Tower 50-epoch CPU training produced `/app/models/geointel-building-yolov8n-expanded160e50.pt`; the model catalog exposes it as `geointel-building-yolov8n-expanded160e50-pt` with SHA256 `bf6a5e8d25a62d784ee53764ea11d7ce89c4e7aeeac7588010e497b8d7dafb2b`. +- Live YOLO preflight loaded `geointel-building-yolov8n-expanded160e50-pt` successfully with `status=ready`, `model_load_ok=true`, `manifest_valid=true`, `tile_paths_exist=true`, `will_download_models=false` and `will_run_inference=false`. +- Live 45-run Geel/Mol/Turnhout/Retie/Kasterlee-bos QA matrix showed the expanded model is the best current candidate on dense building AOIs: best overall score was Geel at tile `640`, threshold `0.05`, precision `0.30333333333333334`, recall `0.14748784440842788`, F1 `0.1984732824427481`. +- Hard-negative finding: on the sparse Kasterlee-bos sample, `yolov8s-building-segmentation-pt` remained cleaner, while the expanded local model produced too many false positives. The model is therefore improved but still experimental, not a V1 default. +- No Training Studio UI, API contract change, provider fetching, model auto-provisioning, fake detections or app-side model training behavior was introduced. + ## Sprint 130 Operator YOLO tile-level dataset tooling (2026-07-07) - Added `scripts/export_operator_yolo_tile_dataset.py` to convert prepared operator samples into overlapping YOLO tile datasets with clipped building labels and deterministic negative tile retention. diff --git a/backend/README.md b/backend/README.md index 61443ec9..a641e2a6 100644 --- a/backend/README.md +++ b/backend/README.md @@ -426,16 +426,21 @@ manifests generated for AI handoff include source CRS metadata so pixel-space model outputs can be transformed to WGS84 GeoJSON coordinates. Current V1 upload support is limited to GeoTIFF-style rasters and GeoJSON/JSON reference vectors. -To prepare the documented Geel/Mol/Turnhout operator sample pairs inside the -all-in-one runtime container, run: +To prepare the documented operator sample corpus inside the all-in-one runtime +container, run: ```bash -docker exec -it geointel python /app/scripts/prepare_operator_real_data_samples.py --samples geel,mol,turnhout +docker exec -it geointel python3 /app/scripts/prepare_operator_real_data_samples.py ``` The helper writes GeoTIFF orthophotos, GRB GBG building GeoJSON files and -`operator_samples_manifest.json` under `/app/storage/operator-data`. These are -runtime artifacts only and are not committed to Git. +`operator_samples_manifest.json` under `/app/storage/operator-data`. The corpus +contains dense reference AOIs for Geel, Mol, Turnhout, Herentals, Balen, Retie +and Westerlo plus explicitly marked background candidates for Postel-bos, +Lommel-heide and Kasterlee-bos. Background candidates can persist empty GRB +FeatureCollections for negative-tile training; normal reference AOIs still fail +when GRB returns no buildings. These are runtime artifacts only and are not +committed to Git. For model-quality calibration, run the confidence sweep wrapper: diff --git a/backend/tests/test_sprint131_operator_sample_expansion.py b/backend/tests/test_sprint131_operator_sample_expansion.py new file mode 100644 index 00000000..dc699fa9 --- /dev/null +++ b/backend/tests/test_sprint131_operator_sample_expansion.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] + + +def load_sample_preparer(): + script_path = ROOT / "scripts" / "prepare_operator_real_data_samples.py" + spec = importlib.util.spec_from_file_location("operator_sample_preparer", script_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_operator_sample_registry_includes_kempen_reference_and_background_candidates() -> None: + module = load_sample_preparer() + + expected_reference_slugs = {"geel", "mol", "turnhout", "herentals", "balen", "retie", "westerlo"} + expected_background_slugs = {"postel_bos", "lommel_heide", "kasterlee_bos"} + + assert expected_reference_slugs.issubset(module.SAMPLES) + assert expected_background_slugs.issubset(module.SAMPLES) + assert all(not module.SAMPLES[slug].allow_empty_reference for slug in expected_reference_slugs) + assert all(module.SAMPLES[slug].allow_empty_reference for slug in expected_background_slugs) + assert all(module.SAMPLES[slug].sample_role == "background_candidate" for slug in expected_background_slugs) + + +def test_background_candidate_can_write_empty_reference_geojson(tmp_path: Path, monkeypatch) -> None: + module = load_sample_preparer() + + class EmptyFeatureResponse: + headers = {"content-type": "application/geo+json"} + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return {"type": "FeatureCollection", "features": []} + + class FakeRequests: + @staticmethod + def get(*args, **kwargs): + return EmptyFeatureResponse() + + monkeypatch.setattr(module, "requests", FakeRequests) + monkeypatch.setattr(module, "prepared_url", lambda url, params: f"{url}?prepared=true") + + sample = module.OperatorSample( + slug="background", + display_name="Background", + center_lon=5.0, + center_lat=51.0, + allow_empty_reference=True, + sample_role="background_candidate", + ) + reference_path = tmp_path / "background.geojson" + + source_url, feature_count = module.fetch_reference(sample, reference_path, [4.9, 50.9, 5.1, 51.1]) + + assert source_url.endswith("?prepared=true") + assert feature_count == 0 + payload = reference_path.read_text(encoding="utf-8") + assert '"features": []' in payload + assert '"sample_role": "background_candidate"' in payload + + +def test_reference_sample_still_rejects_empty_grb_response(tmp_path: Path, monkeypatch) -> None: + module = load_sample_preparer() + + class EmptyFeatureResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return {"type": "FeatureCollection", "features": []} + + class FakeRequests: + @staticmethod + def get(*args, **kwargs): + return EmptyFeatureResponse() + + monkeypatch.setattr(module, "requests", FakeRequests) + monkeypatch.setattr(module, "prepared_url", lambda url, params: f"{url}?prepared=true") + + sample = module.OperatorSample( + slug="urban", + display_name="Urban", + center_lon=5.0, + center_lat=51.0, + ) + + with pytest.raises(SystemExit, match="returned no building features"): + module.fetch_reference(sample, tmp_path / "urban.geojson", [4.9, 50.9, 5.1, 51.1]) diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index b7a29dda..5cb4303f 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -171,12 +171,17 @@ Documented operator samples can be prepared inside the all-in-one runtime container: ```bash -docker exec -it geointel python /app/scripts/prepare_operator_real_data_samples.py --samples geel,mol,turnhout +docker exec -it geointel python3 /app/scripts/prepare_operator_real_data_samples.py ``` The helper fetches explicit Digitaal Vlaanderen orthophoto/GRB GBG sample pairs for the documented AOIs only and writes `operator_samples_manifest.json`. The -application itself still does not perform live provider fetching. +default corpus includes dense reference AOIs for Geel, Mol, Turnhout, Herentals, +Balen, Retie and Westerlo plus explicitly marked background candidates for +Postel-bos, Lommel-heide and Kasterlee-bos. Background candidates may persist +empty GRB FeatureCollections for negative-tile training; normal reference AOIs +still fail on empty GRB responses. The application itself still does not perform +live provider fetching. For confidence-threshold calibration, use the sweep wrapper: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 875b41e9..dc5d1d95 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,54 @@ +## Sprint 131 Operator sample expansion and negative-tile YOLO candidate (2026-07-07) + +Changed: +- Extended `scripts/prepare_operator_real_data_samples.py` with `sample_role` and `allow_empty_reference`. +- Added reference AOIs for Herentals, Balen, Retie and Westerlo. +- Added background-candidate AOIs for Postel-bos, Lommel-heide and Kasterlee-bos. Background candidates can persist empty GRB FeatureCollections for negative-tile training, while normal reference samples still fail on empty GRB results. +- Added regression coverage in `backend/tests/test_sprint131_operator_sample_expansion.py`. +- Updated `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`. + +Tested: +- RED: `python -m pytest backend\tests\test_sprint131_operator_sample_expansion.py -q` failed before the new sample metadata and background candidates existed. +- `python -m pytest backend\tests\test_sprint131_operator_sample_expansion.py -q` passed. +- `python -m py_compile scripts\prepare_operator_real_data_samples.py` passed. +- `python scripts\prepare_operator_real_data_samples.py --help` passed. +- Live Tower operator sample preparation passed: + - manifest: `/app/storage/operator-data/operator_samples_manifest.json` + - samples: Geel `617`, Mol `374`, Turnhout `773`, Herentals `665`, Balen `309`, Retie `592`, Westerlo `334`, Postel-bos `0`, Lommel-heide `0`, Kasterlee-bos `7` reference features. +- Live Tower expanded tile export passed: + - dataset: `/app/storage/operator-data/yolo-building-tile-expanded160` + - tile size: `160` + - stride: `80` + - exported tiles: `360` + - positive tiles: `260` + - negative tiles: `100` + - labels: `11213` + - train tiles: `252` + - validation tiles: `108` +- Live Tower 50-epoch CPU training passed: + - output model: `/app/models/geointel-building-yolov8n-expanded160e50.pt` + - catalog asset: `geointel-building-yolov8n-expanded160e50-pt` + - SHA256: `bf6a5e8d25a62d784ee53764ea11d7ce89c4e7aeeac7588010e497b8d7dafb2b` + - final validation: precision `0.428`, recall `0.389`, mAP50 `0.318`, mAP50-95 `0.106` +- Live API preflight passed for `geointel-building-yolov8n-expanded160e50-pt` with `status=ready`, `model_load_ok=true`, `manifest_valid=true`, `tile_paths_exist=true`, `will_download_models=false` and `will_run_inference=false`. +- Live 45-run multi-sample QA matrix completed: + - output: `/mnt/user/appdata/geointel/artifacts/detection-quality-matrix/multi-sample/expanded160e50-live/multi_sample_quality_summary.json` + - command compared `geointel-building-yolov8n-expanded160e50-pt`, `geointel-building-yolov8n-tile30-pt` and `yolov8s-building-segmentation-pt` over Geel, Mol, Turnhout, Retie and Kasterlee-bos with tile `640`, overlap `64`, thresholds `0.25`/`0.15`/`0.05`. + - best overall score and recall: Geel, `geointel-building-yolov8n-expanded160e50-pt`, tile `640`, threshold `0.05`, 300 detections, 91 matches, 209 false positives, 526 false negatives, precision `0.30333333333333334`, recall `0.14748784440842788`, F1 `0.1984732824427481`. + - dense-sample score winners: Geel, Mol, Turnhout and Retie all selected `geointel-building-yolov8n-expanded160e50-pt`. + - hard-negative/sparse-sample winner: Kasterlee-bos selected `yolov8s-building-segmentation-pt`, threshold `0.25`, F1 `0.16666666666666666`; the expanded local model produced too many false positives there. + +Open: +- None for the sample-preparation and expanded-training runtime proof itself. + +Limitations: +- This remains operator tooling only. It does not add Training Studio, browser training controls, provider fetching, fake detections, model auto-provisioning or API contract changes. +- `geointel-building-yolov8n-expanded160e50-pt` is the best tested candidate on dense operator AOIs, but it is still experimental and should not become the V1 default until hard-negative false positives improve. +- The next model pass should add more sparse/background AOIs, tune confidence/NMS/max-detection settings and compare a stronger architecture or longer run against the same persisted QA matrix. + +Next recommended pass: +- Build a hard-negative model-quality pass: expand sparse/background AOIs, export a balanced tile dataset, train a stronger candidate, and rerun the multi-sample QA matrix with dense and background samples scored separately. + ## Sprint 130 Operator YOLO tile-level dataset tooling (2026-07-07) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index 02a1e664..f0cdc46a 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -103,8 +103,10 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add and benchmark a stronger `yolov8s` building-segmentation runtime model candidate. - [x] Add operator-only tile-level YOLO dataset export with overlapping windows and deterministic negative tile retention. - [x] Train and benchmark the first tile-level local YOLO candidate on Tower through the persisted QA/QC matrix. -- [ ] Calibrate confidence, IoU and model selection against additional local orthophoto/reference samples beyond Geel/Mol/Turnhout. -- [ ] Find or train a materially stronger aerial/Kempen building model candidate; `geointel-building-yolov8n-tile30-pt` is the best current overall candidate but still too weak for a V1 default. +- [x] Calibrate confidence, IoU and model selection against additional local orthophoto/reference samples beyond Geel/Mol/Turnhout. +- [x] Add negative/background AOIs to the operator sample corpus and train an expanded local tile-level YOLO candidate. +- [ ] Add a hard-negative model-quality pass with more sparse/background AOIs, balanced tile export and explicit false-positive scoring. +- [ ] Find or train a materially stronger aerial/Kempen building model candidate; `geointel-building-yolov8n-expanded160e50-pt` is the best current dense-AOI candidate but still too weak and too noisy for a V1 default. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index b4432dac..bfbb0675 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -175,13 +175,18 @@ To prepare the documented operator samples reproducibly inside the all-in-one runtime container, run: ```bash -docker exec -it geointel python /app/scripts/prepare_operator_real_data_samples.py --samples geel,mol,turnhout +docker exec -it geointel python3 /app/scripts/prepare_operator_real_data_samples.py ``` This writes GeoTIFF/GeoJSON pairs and `operator_samples_manifest.json` under `/app/storage/operator-data` inside the container, which maps to -`storage/operator-data` in the Tower appdata checkout. The helper fetches only -the explicit documented AOIs, records Digitaal Vlaanderen attribution and +`storage/operator-data` in the Tower appdata checkout. The default corpus +contains reference AOIs for Geel, Mol, Turnhout, Herentals, Balen, Retie and +Westerlo plus background candidates for Postel-bos, Lommel-heide and +Kasterlee-bos. Normal reference AOIs still fail when GRB returns no buildings; +background candidates are explicitly marked with `sample_role` and may write an +empty reference FeatureCollection for negative-tile training. The helper fetches +only the explicit documented AOIs, records Digitaal Vlaanderen attribution and reuses existing files by default. Use `--force` only when the local runtime artifacts should be regenerated. @@ -308,11 +313,11 @@ with overlapping raster windows: ```bash docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \ --manifest-path /app/storage/operator-data/operator_samples_manifest.json \ - --output-dir /app/storage/operator-data/yolo-building-tile-dataset \ - --tile-size 192 \ - --stride 96 \ - --negative-keep-ratio 0.5 \ - --val-samples turnhout \ + --output-dir /app/storage/operator-data/yolo-building-tile-expanded160 \ + --tile-size 160 \ + --stride 80 \ + --negative-keep-ratio 1.0 \ + --val-samples turnhout,retie,kasterlee_bos \ --force ``` @@ -328,18 +333,38 @@ output directory: ```bash docker exec \ - -e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-tile-dataset \ + -e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-tile-expanded160 \ -e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \ - -e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-tile-detector.pt \ - -e TRAIN_EPOCHS=30 \ + -e TRAIN_OUTPUT_DIR=/app/storage/training/operator-yolo \ + -e TRAIN_RUN_NAME=geointel-building-yolov8n-expanded160e50 \ + -e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-yolov8n-expanded160e50.pt \ + -e TRAIN_EPOCHS=50 \ -e TRAIN_IMGSZ=256 \ - -e TRAIN_BATCH=4 \ + -e TRAIN_BATCH=8 \ -e TRAIN_WORKERS=0 \ -e TRAIN_DEVICE=cpu \ -e PYTHON_BIN=python3 \ geointel bash /app/scripts/train_operator_yolo_detector.sh ``` +Benchmark any trained candidate through the same persisted QA/QC matrix before +using it operationally: + +```bash +OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +OPERATOR_SAMPLE_SLUGS="geel mol turnhout retie kasterlee_bos" \ +QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-building-yolov8n-tile30-pt yolov8s-building-segmentation-pt" \ +QUALITY_TILE_SIZES="640" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.25 0.15 0.05" \ +MULTI_SAMPLE_OUTPUT_DIR=artifacts/detection-quality-matrix/multi-sample/expanded160e50-live \ +bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.168.10.150:1202 +``` + +The expanded 50-epoch candidate improved dense Geel/Mol/Turnhout/Retie scores, +but the sparse Kasterlee-bos run still showed too many false positives. Treat it +as the best current experimental dense-AOI candidate, not as a V1 default. + Export calibration QA evidence for visual review: ```bash diff --git a/scripts/prepare_operator_real_data_samples.py b/scripts/prepare_operator_real_data_samples.py index a0f2ddd5..e3efb84e 100644 --- a/scripts/prepare_operator_real_data_samples.py +++ b/scripts/prepare_operator_real_data_samples.py @@ -36,6 +36,8 @@ class OperatorSample: half_size_m: float = 250.0 width: int = 512 height: int = 512 + sample_role: str = "reference" + allow_empty_reference: bool = False SAMPLES: dict[str, OperatorSample] = { @@ -57,6 +59,61 @@ SAMPLES: dict[str, OperatorSample] = { center_lon=4.9488, center_lat=51.3225, ), + "herentals": OperatorSample( + slug="herentals", + display_name="Herentals center", + center_lon=4.8339, + center_lat=51.1766, + half_size_m=220.0, + ), + "balen": OperatorSample( + slug="balen", + display_name="Balen center", + center_lon=5.1703, + center_lat=51.1688, + half_size_m=220.0, + ), + "retie": OperatorSample( + slug="retie", + display_name="Retie center", + center_lon=5.0827, + center_lat=51.2665, + half_size_m=220.0, + ), + "westerlo": OperatorSample( + slug="westerlo", + display_name="Westerlo center", + center_lon=4.9158, + center_lat=51.0909, + half_size_m=220.0, + ), + "postel_bos": OperatorSample( + slug="postel_bos", + display_name="Postel forest background candidate", + center_lon=5.16, + center_lat=51.305, + half_size_m=260.0, + sample_role="background_candidate", + allow_empty_reference=True, + ), + "lommel_heide": OperatorSample( + slug="lommel_heide", + display_name="Lommel forest background candidate", + center_lon=5.287, + center_lat=51.249, + half_size_m=260.0, + sample_role="background_candidate", + allow_empty_reference=True, + ), + "kasterlee_bos": OperatorSample( + slug="kasterlee_bos", + display_name="Kasterlee forest background candidate", + center_lon=4.965, + center_lat=51.273, + half_size_m=260.0, + sample_role="background_candidate", + allow_empty_reference=True, + ), } @@ -218,7 +275,7 @@ def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list response.raise_for_status() reference = response.json() features = reference.get("features") or [] - if not features: + if not features and not sample.allow_empty_reference: raise SystemExit(f"GRB GBG returned no building features for {sample.slug} bbox {geo_bbox}") reference["name"] = f"GRB GBG buildings - {sample.display_name} sample AOI" @@ -227,11 +284,14 @@ def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list reference["attribution"] = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen" reference["bbox"] = geo_bbox reference["sample_slug"] = sample.slug + reference["sample_role"] = sample.sample_role + reference["allow_empty_reference"] = sample.allow_empty_reference for feature in features: props = feature.setdefault("properties", {}) props.setdefault("source_name", "grb") props.setdefault("reference_layer_name", "buildings") props.setdefault("sample_slug", sample.slug) + props.setdefault("sample_role", sample.sample_role) reference_path.write_text(json.dumps(reference, ensure_ascii=False), encoding="utf-8") return prepared_url(GRB_GBG_URL, ogc_params), len(features) @@ -256,6 +316,8 @@ def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dic "center_lon": sample.center_lon, "center_lat": sample.center_lat, "half_size_m": sample.half_size_m, + "sample_role": sample.sample_role, + "allow_empty_reference": sample.allow_empty_reference, "raster_path": str(ortho_path), "reference_path": str(reference_path), "reference_feature_count": reference_feature_count, @@ -288,7 +350,7 @@ def write_readme(output_dir: Path, samples: list[dict[str, Any]]) -> None: lines.append( f"- `{sample['sample_slug']}`: `{Path(sample['raster_path']).name}` and " f"`{Path(sample['reference_path']).name}`, " - f"{sample['reference_feature_count']} reference features." + f"{sample['reference_feature_count']} reference features, role `{sample['sample_role']}`." ) lines.append("") lines.append("Purpose: configured-YOLO detection + persisted QA/QC validation with operator-provided files.")