diff --git a/CHANGELOG.md b/CHANGELOG.md index b8f75e01..6292c899 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 127 Multi-sample detection quality calibration tooling (2026-07-07) + +- Added `scripts/prepare_operator_real_data_samples.py` to prepare documented Geel, Mol and Turnhout orthophoto/GRB GBG building sample pairs as explicit runtime artifacts. +- Added `scripts/run_multi_sample_detection_quality_matrix.sh` to run the existing real-data quality matrix for every prepared sample and combine the results. +- The combined summary writes `multi_sample_quality_summary.json` with overall score/recall/precision rankings and per-sample best configurations. +- Added readiness coverage and regression tests for the sample-preparation and multi-sample matrix contracts. + ## Sprint 126 Detection quality matrix tooling (2026-07-07) - Added `scripts/run_detection_quality_matrix.sh` to compare local model assets, raster tile sizes, tile overlaps and confidence thresholds through the existing real-data detection + QA workflow. diff --git a/backend/README.md b/backend/README.md index 8ba9b968..8b578ddd 100644 --- a/backend/README.md +++ b/backend/README.md @@ -374,6 +374,17 @@ manifests generated for AI handoff include source CRS metadata so pixel-space model outputs can be transformed to WGS84 GeoJSON coordinates. Current V1 upload support is limited to GeoTIFF-style rasters and GeoJSON/JSON reference vectors. +To prepare the documented Geel/Mol/Turnhout operator sample pairs inside the +all-in-one runtime container, run: + +```bash +docker exec -it geointel python /app/scripts/prepare_operator_real_data_samples.py --samples geel,mol,turnhout +``` + +The helper writes GeoTIFF orthophotos, GRB GBG building GeoJSON files and +`operator_samples_manifest.json` under `/app/storage/operator-data`. These are +runtime artifacts only and are not committed to Git. + For model-quality calibration, run the confidence sweep wrapper: ```bash @@ -409,6 +420,21 @@ and false-positive/false-negative counts. It ranks `best_by_score`, `best_by_recall` and `best_by_precision`. It does not download weights, create fake detections, fetch live providers or change backend API behavior. +To aggregate the same matrix over every prepared operator sample, run: + +```bash +OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \ +QUALITY_TILE_SIZES="512 640" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.50 0.15" \ +bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.168.10.150:1202 +``` + +The combined `multi_sample_quality_summary.json` reports per-sample and overall +best configurations. It is an operator benchmarking command, not a backend API +or provider import path. + To inspect the evidence behind a calibration run, export the persisted QA evidence bundle: diff --git a/backend/tests/test_sprint127_operator_sample_quality_matrix.py b/backend/tests/test_sprint127_operator_sample_quality_matrix.py new file mode 100644 index 00000000..13e909a3 --- /dev/null +++ b/backend/tests/test_sprint127_operator_sample_quality_matrix.py @@ -0,0 +1,71 @@ +from pathlib import Path +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_prepare_operator_real_data_samples_fetches_documented_ortho_and_grb_pairs() -> None: + script_path = ROOT / "scripts" / "prepare_operator_real_data_samples.py" + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert script_path.exists() + script = script_path.read_text(encoding="utf-8") + + assert "py_compile scripts/prepare_operator_real_data_samples.py" in readiness + assert "SAMPLES" in script + assert '"geel"' in script + assert '"mol"' in script + assert '"turnhout"' in script + assert "https://geo.api.vlaanderen.be/omwrgbmrvl/wms" in script + assert "LAYERS" in script + assert "Ortho" in script + assert "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items" in script + assert "source_name" in script + assert "reference_layer_name" in script + assert "operator_samples_manifest.json" in script + assert "skip_existing" in script + assert "demo/workflow" not in script + assert "fixture_mode" not in script + + +def test_prepare_operator_real_data_samples_help_does_not_require_gis_dependencies() -> None: + script_path = ROOT / "scripts" / "prepare_operator_real_data_samples.py" + + result = subprocess.run( + [sys.executable, str(script_path), "--help"], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0 + assert "Prepare real Digitaal Vlaanderen" in result.stdout + assert "--samples" in result.stdout + + +def test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sample() -> None: + script_path = ROOT / "scripts" / "run_multi_sample_detection_quality_matrix.sh" + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert script_path.exists() + script = script_path.read_text(encoding="utf-8") + + assert "bash -n scripts/run_multi_sample_detection_quality_matrix.sh" in readiness + assert "OPERATOR_SAMPLE_MANIFEST_PATH" in script + assert "operator_samples_manifest.json" in script + assert "run_detection_quality_matrix.sh" in script + assert "QUALITY_MODEL_ASSET_IDS" in script + assert "QUALITY_TILE_SIZES" in script + assert "QUALITY_TILE_OVERLAPS" in script + assert "QUALITY_THRESHOLDS" in script + assert "REAL_RASTER_PATH" in script + assert "REAL_REFERENCE_VECTOR_PATH" in script + assert "multi_sample_quality_summary.json" in script + assert "best_overall_by_score" in script + assert "best_by_sample" in script + assert "sample_slug" in script + assert "demo/workflow" not in script + assert "fixture_mode" not in script + assert "will_download_models" not in script diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index ce26f95d..2af325c1 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -167,6 +167,17 @@ download model weights. A zero detection count is valid as runtime evidence only when the selected model genuinely returns no usable detections after canonical class filtering; it does not prove the model is useful for the target imagery. +Documented operator samples can be prepared inside the all-in-one runtime +container: + +```bash +docker exec -it geointel python /app/scripts/prepare_operator_real_data_samples.py --samples geel,mol,turnhout +``` + +The helper fetches explicit Digitaal Vlaanderen orthophoto/GRB GBG sample pairs +for the documented AOIs only and writes `operator_samples_manifest.json`. The +application itself still does not perform live provider fetching. + For confidence-threshold calibration, use the sweep wrapper: ```bash @@ -201,6 +212,23 @@ rankings `best_by_score`, `best_by_recall` and `best_by_precision` are operator decision aids only; GeoIntel still does not download models, seed fixture detections or treat AI detections as ground truth without QA/QC. +To compare the same model/tile/threshold grid across all prepared operator +samples, use: + +```bash +OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \ +QUALITY_TILE_SIZES="512 640" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.50 0.15" \ +bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.168.10.150:1202 +``` + +The multi-sample summary exposes `best_overall_by_score`, +`best_overall_by_recall`, `best_overall_by_precision` and `best_by_sample` so +model-quality decisions are based on repeated persisted QA/QC evidence rather +than one AOI. + For visual error inspection, export the persisted QA evidence from a calibration summary: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index d89b3a60..c10a9488 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,37 @@ +## Sprint 127 Multi-sample detection quality calibration tooling (2026-07-07) + +Changed: +- Added `scripts/prepare_operator_real_data_samples.py` as an explicit operator/runtime helper for documented Geel, Mol and Turnhout real-data samples. +- The helper downloads small Digitaal Vlaanderen OMWRGBMRVL WMS `Ortho` GeoTIFFs and GRB OGC API Features `GBG` building GeoJSON references for the documented AOIs only. +- The helper writes `operator_samples_manifest.json`, sample metadata, source URLs and attribution under the runtime operator-data directory and reuses existing files by default. +- Added `scripts/run_multi_sample_detection_quality_matrix.sh` to run `scripts/run_detection_quality_matrix.sh` once per manifest sample. +- The multi-sample wrapper combines per-sample `quality_matrix_summary.json` files into `multi_sample_quality_summary.json` with `best_overall_by_score`, `best_overall_by_recall`, `best_overall_by_precision` and `best_by_sample`. +- Added readiness checks for Python compile and shell syntax. +- Added regression coverage in `backend/tests/test_sprint127_operator_sample_quality_matrix.py`. +- Updated `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`. + +Tested: +- RED: `python -m pytest backend\tests\test_sprint127_operator_sample_quality_matrix.py -q` failed because the sample-preparation and multi-sample scripts did not exist. +- RED: `python -m pytest backend\tests\test_sprint127_operator_sample_quality_matrix.py::test_prepare_operator_real_data_samples_help_does_not_require_gis_dependencies -q` failed because `--help` required missing GIS dependencies. +- `python -m pytest backend\tests\test_sprint127_operator_sample_quality_matrix.py -q` passed. +- `python scripts\prepare_operator_real_data_samples.py --help` passed without requiring local GIS dependencies. +- `python -m py_compile scripts\prepare_operator_real_data_samples.py` passed. +- `bash -n scripts/run_multi_sample_detection_quality_matrix.sh` passed. +- `python -m pytest backend\tests\test_sprint127_operator_sample_quality_matrix.py backend\tests\test_sprint126_detection_quality_matrix.py backend\tests\test_sprint125_detection_calibration_evidence_bundle.py backend\tests\test_sprint124_detection_calibration_sweep.py -q` passed. +- `python scripts\smoke_docs.py` passed. +- `git diff --check` passed. +- `bash scripts/run_readiness_check.sh` passed: 393 backend tests, frontend typecheck/build, Alembic head `202606120900`, live smoke syntax checks and the new sample/multi-sample checks. + +Open: +- Live Tower sample preparation and multi-sample matrix run still needed. + +Limitations: +- This is operator tooling only. It does not add a live GRB provider, live orthophoto provider, application endpoint, migration, frontend feature, model download or fixture inference path. +- The prepared sample files are runtime artifacts under appdata/storage and remain excluded from Git. + +Next recommended pass: +- Run the sample-preparation helper in the Tower all-in-one container, then run the multi-sample matrix from the Tower checkout and document the combined quality baseline. + ## Sprint 126 Detection quality matrix tooling (2026-07-07) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index e4d0ee3f..43e1f458 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -98,7 +98,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add real-data detection calibration sweep tooling for confidence-threshold and QA/QC metric comparison. - [x] Add calibration QA evidence export tooling for false-positive/false-negative inspection artifacts. - [x] Add real-data detection quality matrix tooling for model/tile/threshold comparison. -- [ ] Calibrate confidence, IoU and model selection against persisted Geel detections and additional local orthophoto/reference samples. +- [x] Add reproducible Geel/Mol/Turnhout operator sample preparation and multi-sample quality matrix tooling. +- [ ] Calibrate confidence, IoU and model selection against persisted Geel/Mol/Turnhout detections and any additional local orthophoto/reference samples. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index d099220c..2f6beddc 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -171,6 +171,20 @@ Those files are runtime artifacts generated from Digitaal Vlaanderen's OMWRGBMRVL WMS `Ortho` layer and GRB OGC API Features `GBG` building collection for a small Geel AOI. They are intentionally not repository fixtures. +To prepare the documented operator samples reproducibly inside the all-in-one +runtime container, run: + +```bash +docker exec -it geointel python /app/scripts/prepare_operator_real_data_samples.py --samples geel,mol,turnhout +``` + +This writes GeoTIFF/GeoJSON pairs and `operator_samples_manifest.json` under +`/app/storage/operator-data` inside the container, which maps to +`storage/operator-data` in the Tower appdata checkout. The helper fetches only +the explicit documented AOIs, records Digitaal Vlaanderen attribution and +reuses existing files by default. Use `--force` only when the local runtime +artifacts should be regenerated. + The real-data smoke is intentionally mutating and refuses to run without operator-supplied files. Current V1 upload support expects a georeferenced `.tif`, `.tiff` or `.geotiff` raster and a `.geojson` or `.json` reference @@ -229,6 +243,24 @@ set. The summary ranks `best_by_score`, `best_by_recall` and `QualityCheck`/`Metric` evidence rather than visual guesses. It does not create provider data, use fixtures or download model weights. +Run the same matrix across every prepared operator sample: + +```bash +OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \ +QUALITY_TILE_SIZES="512 640" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.50 0.15" \ +bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.168.10.150:1202 +``` + +The multi-sample wrapper writes one per-sample `quality_matrix_summary.json` +plus a combined `multi_sample_quality_summary.json` with +`best_overall_by_score`, `best_overall_by_recall`, +`best_overall_by_precision` and `best_by_sample` rankings. It resolves +container-style `/app/storage/...` manifest paths to repo-relative +`storage/...` paths when run from the Tower host checkout. + Export calibration QA evidence for visual review: ```bash diff --git a/scripts/prepare_operator_real_data_samples.py b/scripts/prepare_operator_real_data_samples.py new file mode 100644 index 00000000..a0f2ddd5 --- /dev/null +++ b/scripts/prepare_operator_real_data_samples.py @@ -0,0 +1,319 @@ +"""Prepare explicit real operator samples for GeoIntel detection QA. + +This script downloads small orthophoto and GRB building reference pairs from +Digitaal Vlaanderen for documented Kempen AOIs. It is an operator/runtime +helper, not an application provider integration: no GeoIntel API route calls it +and no production data is fetched silently by the app. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms" +GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items" +DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data") +requests: Any = None +rasterio: Any = None +Transformer: Any = None +MemoryFile: Any = None +from_bounds: Any = None + + +@dataclass(frozen=True) +class OperatorSample: + slug: str + display_name: str + center_lon: float + center_lat: float + half_size_m: float = 250.0 + width: int = 512 + height: int = 512 + + +SAMPLES: dict[str, OperatorSample] = { + "geel": OperatorSample( + slug="geel", + display_name="Geel center", + center_lon=4.991, + center_lat=51.162, + ), + "mol": OperatorSample( + slug="mol", + display_name="Mol center", + center_lon=5.1167, + center_lat=51.1919, + ), + "turnhout": OperatorSample( + slug="turnhout", + display_name="Turnhout center", + center_lon=4.9488, + center_lat=51.3225, + ), +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Prepare real Digitaal Vlaanderen orthophoto/GRB building samples for GeoIntel operator QA.", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path(os.environ.get("OPERATOR_DATA_DIR", DEFAULT_OUTPUT_DIR)), + help="Directory for generated GeoTIFF, GeoJSON and manifest files.", + ) + parser.add_argument( + "--samples", + default=",".join(SAMPLES), + help="Comma/space separated sample slugs to prepare. Defaults to all documented samples.", + ) + parser.add_argument( + "--force", + action="store_true", + help="Refetch and overwrite sample files. By default existing raster/reference pairs are reused.", + ) + parser.add_argument( + "--manifest-name", + default="operator_samples_manifest.json", + help="Manifest filename written inside output-dir.", + ) + return parser.parse_args() + + +def ensure_gis_dependencies() -> None: + global MemoryFile, Transformer, from_bounds, rasterio, requests + try: + import requests as requests_module + import rasterio as rasterio_module + from pyproj import Transformer as transformer_class + from rasterio.io import MemoryFile as memory_file_class + from rasterio.transform import from_bounds as from_bounds_function + except Exception as exc: # pragma: no cover - exercised only in runtime envs. + raise SystemExit( + "prepare_operator_real_data_samples.py requires requests, rasterio and pyproj. " + "Run it inside the GeoIntel all-in-one container or an equivalent GIS Python environment." + ) from exc + requests = requests_module + rasterio = rasterio_module + Transformer = transformer_class + MemoryFile = memory_file_class + from_bounds = from_bounds_function + + +def selected_samples(raw: str) -> list[OperatorSample]: + slugs = [value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()] + if not slugs: + raise SystemExit("--samples must include at least one sample slug") + unknown = [slug for slug in slugs if slug not in SAMPLES] + if unknown: + raise SystemExit(f"Unknown sample slug(s): {', '.join(unknown)}. Known: {', '.join(SAMPLES)}") + return [SAMPLES[slug] for slug in slugs] + + +def sample_bounds(sample: OperatorSample) -> tuple[tuple[float, float, float, float], list[float]]: + lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) + wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True) + center_x, center_y = lambert.transform(sample.center_lon, sample.center_lat) + minx = center_x - sample.half_size_m + miny = center_y - sample.half_size_m + maxx = center_x + sample.half_size_m + maxy = center_y + sample.half_size_m + corners = [ + wgs84.transform(x, y) + for x, y in ((minx, miny), (minx, maxy), (maxx, miny), (maxx, maxy)) + ] + lon_values = [point[0] for point in corners] + lat_values = [point[1] for point in corners] + return (minx, miny, maxx, maxy), [ + min(lon_values), + min(lat_values), + max(lon_values), + max(lat_values), + ] + + +def prepared_url(url: str, params: dict[str, str]) -> str: + return requests.Request("GET", url, params=params).prepare().url + + +def raster_summary(path: Path) -> dict[str, Any]: + with rasterio.open(path) as ds: + return { + "path": str(path), + "crs": str(ds.crs), + "bounds": list(ds.bounds), + "width": ds.width, + "height": ds.height, + "count": ds.count, + "dtypes": list(ds.dtypes), + } + + +def geojson_feature_count(path: Path) -> int: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + return len(payload.get("features") or []) + + +def fetch_orthophoto(sample: OperatorSample, ortho_path: Path, lambert_bbox: tuple[float, float, float, float]) -> str: + minx, miny, maxx, maxy = lambert_bbox + wms_params = { + "SERVICE": "WMS", + "VERSION": "1.3.0", + "REQUEST": "GetMap", + "LAYERS": "Ortho", + "STYLES": "", + "FORMAT": "image/tiff", + "CRS": "EPSG:31370", + "BBOX": f"{minx},{miny},{maxx},{maxy}", + "WIDTH": str(sample.width), + "HEIGHT": str(sample.height), + } + response = requests.get(WMS_URL, params=wms_params, timeout=120) + response.raise_for_status() + content_type = response.headers.get("content-type", "") + if "image" not in content_type.lower() and "tiff" not in content_type.lower(): + raise SystemExit(f"Orthophoto WMS did not return an image for {sample.slug}: {content_type}") + + with MemoryFile(response.content) as memfile: + with memfile.open() as src: + image = src.read() + profile = src.profile.copy() + profile.update( + driver="GTiff", + width=src.width, + height=src.height, + count=src.count, + dtype=src.dtypes[0], + crs="EPSG:31370", + transform=from_bounds(minx, miny, maxx, maxy, src.width, src.height), + compress="deflate", + tiled=False, + ) + with rasterio.open(ortho_path, "w", **profile) as dst: + dst.write(image) + dst.update_tags( + source="Digitaal Vlaanderen OMWRGBMRVL WMS Ortho layer", + source_url=prepared_url(WMS_URL, wms_params), + attribution="Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen", + aoi=f"{sample.display_name} sample AOI for GeoIntel operator validation", + ) + return prepared_url(WMS_URL, wms_params) + + +def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list[float]) -> tuple[str, int]: + ogc_params = { + "f": "application/geo+json", + "limit": "1000", + "bbox": ",".join(f"{value:.8f}" for value in geo_bbox), + } + response = requests.get(GRB_GBG_URL, params=ogc_params, timeout=120) + response.raise_for_status() + reference = response.json() + features = reference.get("features") or [] + if not features: + raise SystemExit(f"GRB GBG returned no building features for {sample.slug} bbox {geo_bbox}") + + reference["name"] = f"GRB GBG buildings - {sample.display_name} sample AOI" + reference["source"] = "Digitaal Vlaanderen GRB OGC API Features collection GBG" + reference["source_url"] = prepared_url(GRB_GBG_URL, ogc_params) + reference["attribution"] = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen" + reference["bbox"] = geo_bbox + reference["sample_slug"] = sample.slug + for feature in features: + props = feature.setdefault("properties", {}) + props.setdefault("source_name", "grb") + props.setdefault("reference_layer_name", "buildings") + props.setdefault("sample_slug", sample.slug) + + reference_path.write_text(json.dumps(reference, ensure_ascii=False), encoding="utf-8") + return prepared_url(GRB_GBG_URL, ogc_params), len(features) + + +def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dict[str, Any]: + ortho_path = output_dir / f"{sample.slug}_orthophoto_wms_512.tif" + reference_path = output_dir / f"{sample.slug}_grb_gbg_buildings.geojson" + lambert_bbox, geo_bbox = sample_bounds(sample) + skip_existing = ortho_path.exists() and reference_path.exists() and not force + + source_urls: dict[str, str | None] = {"orthophoto": None, "reference": None} + if not skip_existing: + source_urls["orthophoto"] = fetch_orthophoto(sample, ortho_path, lambert_bbox) + source_urls["reference"], reference_feature_count = fetch_reference(sample, reference_path, geo_bbox) + else: + reference_feature_count = geojson_feature_count(reference_path) + + return { + "sample_slug": sample.slug, + "display_name": sample.display_name, + "center_lon": sample.center_lon, + "center_lat": sample.center_lat, + "half_size_m": sample.half_size_m, + "raster_path": str(ortho_path), + "reference_path": str(reference_path), + "reference_feature_count": reference_feature_count, + "raster": raster_summary(ortho_path), + "wgs84_bbox": geo_bbox, + "epsg31370_bbox": list(lambert_bbox), + "skip_existing": skip_existing, + "source_urls": source_urls, + "attribution": { + "orthophoto": "Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen", + "reference": "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen", + }, + } + + +def write_readme(output_dir: Path, samples: list[dict[str, Any]]) -> None: + lines = [ + "# GeoIntel operator real-data samples", + "", + "Generated for runtime validation, not committed to the GeoIntel repository.", + "", + "Sources:", + "- Orthophoto rasters: Digitaal Vlaanderen OMWRGBMRVL WMS `Ortho` layer.", + "- Reference buildings: Digitaal Vlaanderen GRB OGC API Features `GBG` collection.", + "- Attribution: Bron: Orthofotomozaiek Vlaanderen / Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen.", + "", + "Samples:", + ] + for sample in samples: + lines.append( + f"- `{sample['sample_slug']}`: `{Path(sample['raster_path']).name}` and " + f"`{Path(sample['reference_path']).name}`, " + f"{sample['reference_feature_count']} reference features." + ) + lines.append("") + lines.append("Purpose: configured-YOLO detection + persisted QA/QC validation with operator-provided files.") + (output_dir / "README.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> int: + args = parse_args() + ensure_gis_dependencies() + output_dir: Path = args.output_dir + output_dir.mkdir(parents=True, exist_ok=True) + samples = [prepare_sample(sample, output_dir, force=args.force) for sample in selected_samples(args.samples)] + write_readme(output_dir, samples) + + manifest = { + "schema_version": 1, + "description": "GeoIntel operator real-data samples for configured-YOLO QA validation.", + "output_dir": str(output_dir), + "samples": samples, + } + manifest_path = output_dir / args.manifest_name + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps({"status": "ok", "manifest_path": str(manifest_path), "samples": samples}, indent=2, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_multi_sample_detection_quality_matrix.sh b/scripts/run_multi_sample_detection_quality_matrix.sh new file mode 100644 index 00000000..0db4ca12 --- /dev/null +++ b/scripts/run_multi_sample_detection_quality_matrix.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: + OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ + QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \ + QUALITY_TILE_SIZES="512 640" \ + QUALITY_TILE_OVERLAPS="64" \ + QUALITY_THRESHOLDS="0.50 0.15" \ + bash scripts/run_multi_sample_detection_quality_matrix.sh [base_url] + +Optional environment: + OPERATOR_SAMPLE_MANIFEST_PATH Manifest from prepare_operator_real_data_samples.py. + OPERATOR_SAMPLE_SLUGS Optional comma/space separated sample slug filter. + MULTI_SAMPLE_OUTPUT_DIR Output directory, default: artifacts/detection-quality-matrix/multi-sample/. + QUALITY_MODEL_ASSET_IDS Forwarded to run_detection_quality_matrix.sh. + QUALITY_TILE_SIZES Forwarded to run_detection_quality_matrix.sh. + QUALITY_TILE_OVERLAPS Forwarded to run_detection_quality_matrix.sh. + QUALITY_THRESHOLDS Forwarded to run_detection_quality_matrix.sh. + REAL_IOU_THRESHOLD Forwarded to run_detection_quality_matrix.sh. + +This script does not run inference itself. It repeats the existing real-data +quality matrix once per documented operator sample and combines the summaries. +EOF +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" +OPERATOR_SAMPLE_MANIFEST_PATH="${OPERATOR_SAMPLE_MANIFEST_PATH:-storage/operator-data/operator_samples_manifest.json}" +OPERATOR_SAMPLE_SLUGS="${OPERATOR_SAMPLE_SLUGS:-}" +MULTI_SAMPLE_OUTPUT_DIR="${MULTI_SAMPLE_OUTPUT_DIR:-artifacts/detection-quality-matrix/multi-sample/$(date -u +%Y%m%dT%H%M%SZ)}" + +if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then + usage + exit 0 +fi + +if [ ! -f "${OPERATOR_SAMPLE_MANIFEST_PATH}" ]; then + echo "OPERATOR_SAMPLE_MANIFEST_PATH does not point to a readable manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}" >&2 + exit 2 +fi + +if [ -n "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="${PYTHON_BIN}" +else + PYTHON_BIN="" + for candidate in python3 python.exe python; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then + PYTHON_BIN="${candidate}" + break + fi + done +fi + +if [ -z "${PYTHON_BIN}" ]; then + echo "A Python interpreter is required for JSON parsing" >&2 + exit 1 +fi + +mkdir -p "${MULTI_SAMPLE_OUTPUT_DIR}" +sample_manifest_tsv="${MULTI_SAMPLE_OUTPUT_DIR}/multi_sample_requests.tsv" + +"${PYTHON_BIN}" - \ + "${ROOT}" \ + "${OPERATOR_SAMPLE_MANIFEST_PATH}" \ + "${OPERATOR_SAMPLE_SLUGS}" \ + "${sample_manifest_tsv}" <<'PY' +import json +import os +import sys +from pathlib import Path + +root = Path(sys.argv[1]).resolve() +manifest_path = Path(sys.argv[2]) +slug_filter_raw = sys.argv[3] +output_path = Path(sys.argv[4]) + +payload = json.loads(manifest_path.read_text(encoding="utf-8-sig")) +samples = payload.get("samples") or [] +if not samples: + raise SystemExit("Operator sample manifest contains no samples") + +requested_slugs = { + value.strip().lower() + for value in slug_filter_raw.replace(",", " ").split() + if value.strip() +} + + +def resolve_path(raw: str) -> str: + path = Path(raw) + if path.exists(): + return str(path) + if raw.startswith("/app/"): + candidate = root / raw.removeprefix("/app/") + if candidate.exists(): + return str(candidate) + candidate = root / raw + if candidate.exists(): + return str(candidate) + raise SystemExit(f"Sample file is not readable from this host: {raw}") + + +with output_path.open("w", encoding="utf-8") as handle: + selected = 0 + for sample in samples: + sample_slug = str(sample.get("sample_slug") or "").lower() + if not sample_slug: + raise SystemExit("Operator sample is missing sample_slug") + if requested_slugs and sample_slug not in requested_slugs: + continue + raster_path = resolve_path(str(sample.get("raster_path") or "")) + reference_path = resolve_path(str(sample.get("reference_path") or "")) + reference_count = int(sample.get("reference_feature_count") or 0) + if reference_count < 1: + raise SystemExit(f"Operator sample has no reference features: {sample_slug}") + handle.write(f"{sample_slug}\t{raster_path}\t{reference_path}\t{reference_count}\n") + selected += 1 + +if selected == 0: + raise SystemExit("No operator samples matched OPERATOR_SAMPLE_SLUGS") +PY + +echo "== GeoIntel multi-sample detection quality matrix ==" +echo "Base URL: ${BASE_URL}" +echo "Manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}" +echo "Sample filter: ${OPERATOR_SAMPLE_SLUGS:-all}" +echo "Output: ${MULTI_SAMPLE_OUTPUT_DIR}" + +while IFS=$'\t' read -r sample_slug raster_path reference_path reference_feature_count; do + sample_output_dir="${MULTI_SAMPLE_OUTPUT_DIR}/${sample_slug}" + mkdir -p "${sample_output_dir}" + echo "-- Sample ${sample_slug}: reference_features=${reference_feature_count} --" + REAL_RASTER_PATH="${raster_path}" \ + REAL_REFERENCE_VECTOR_PATH="${reference_path}" \ + QUALITY_OUTPUT_DIR="${sample_output_dir}" \ + bash scripts/run_detection_quality_matrix.sh "${BASE_URL}" +done < "${sample_manifest_tsv}" + +"${PYTHON_BIN}" - "${MULTI_SAMPLE_OUTPUT_DIR}" "${BASE_URL}" "${OPERATOR_SAMPLE_MANIFEST_PATH}" <<'PY' +import glob +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + +output_dir = Path(sys.argv[1]) +base_url = sys.argv[2] +manifest_path = sys.argv[3] + +sample_summaries = [] +flat_items = [] +for summary_path in sorted(glob.glob(str(output_dir / "*" / "quality_matrix_summary.json"))): + sample_slug = Path(summary_path).parent.name + summary = json.loads(Path(summary_path).read_text(encoding="utf-8")) + items = summary.get("items") or [] + for item in items: + enriched = dict(item) + enriched["sample_slug"] = sample_slug + flat_items.append(enriched) + sample_summaries.append( + { + "sample_slug": sample_slug, + "summary_path": summary_path, + "run_count": len(items), + "best_by_score": summary.get("best_by_score"), + "best_by_recall": summary.get("best_by_recall"), + "best_by_precision": summary.get("best_by_precision"), + } + ) + +if not flat_items: + raise SystemExit("No sample quality matrix summaries were produced") + + +def best(metric: str): + ranked = [item for item in flat_items if item.get(metric) is not None] + return max(ranked, key=lambda item: item[metric], default=None) + + +best_by_sample = { + sample["sample_slug"]: sample.get("best_by_score") + for sample in sample_summaries +} +summary = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "base_url": base_url, + "operator_sample_manifest_path": manifest_path, + "sample_count": len(sample_summaries), + "run_count": len(flat_items), + "best_overall_by_score": best("quality_score"), + "best_overall_by_recall": best("recall"), + "best_overall_by_precision": best("precision"), + "best_by_sample": best_by_sample, + "sample_summaries": sample_summaries, + "items": flat_items, +} +summary_path = output_dir / "multi_sample_quality_summary.json" +summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8") + +print("") +print("Multi-sample detection quality summary") +print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn") +for item in flat_items: + print( + "{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format( + **item + ) + ) +print("") +print(f"Summary: {summary_path}") +for key in ("best_overall_by_score", "best_overall_by_recall", "best_overall_by_precision"): + item = summary.get(key) + if item: + print( + f"{key} sample={item['sample_slug']} model={item['model_asset_id']} " + f"tile={item['tile_size']} overlap={item['tile_overlap']} " + f"threshold={item['threshold']:.2f} score={item.get('quality_score')} " + f"precision={item.get('precision')} recall={item.get('recall')} f1={item.get('f1_score')}" + ) +PY diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 2e4227e2..4749a16a 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -41,6 +41,7 @@ ${PYTHON_BIN} -m py_compile scripts/gis_import_smoke.py ${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py ${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py ${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py +${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m compileall backend/app @@ -58,6 +59,7 @@ bash -n scripts/verify_real_data_detection_qa_workflow.sh bash -n scripts/run_detection_calibration_sweep.sh bash -n scripts/export_detection_calibration_evidence.sh bash -n scripts/run_detection_quality_matrix.sh +bash -n scripts/run_multi_sample_detection_quality_matrix.sh bash -n scripts/verify_workbench_default_state.sh bash -n scripts/verify_workbench_interactions.sh bash -n scripts/verify_gis_runtime.sh