From 7bf0757470c5a385bc49b5f3d49ade11a3eec25f Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 9 Jul 2026 14:03:38 +0200 Subject: [PATCH] Fix runtime GIS uploads for operator QA --- CHANGELOG.md | 10 +++++++ backend/tests/test_docker_runtime_config.py | 8 +++++ ...print129_operator_yolo_training_dataset.py | 2 ++ deploy/unraid/nginx-all-in-one.conf | 1 + docs/CODEX_EXECUTION_LOG.md | 30 +++++++++++++++++++ docs/TODO.md | 6 +++- frontend/nginx.conf | 1 + scripts/README.md | 11 +++++-- scripts/train_operator_yolo_detector.sh | 11 +++++-- 9 files changed, 75 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8abcae8..dc250fbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ # Changelog +## Sprint 151 runtime GIS upload and AOI1024 YOLO candidate (2026-07-09) + +- Fixed the operator YOLO training wrapper so the all-in-one runtime defaults to `/opt/geointel/venv/bin/python` when present, while still falling back to `python3` for local shells. +- Raised the Nginx upload limit to `250m` in both the compose frontend proxy and Unraid all-in-one proxy after a live 1024px GeoTIFF QA upload hit `413 Request Entity Too Large`. +- Prepared a larger Tower operator sample manifest at `/app/storage/operator-data/operator-samples-1024` using explicit `1024x1024` rasters and doubled AOI half-size. +- Exported and audited `/app/storage/operator-data/yolo-building-aoi1024-visible025`: 144 tiles, 117 positive tiles, 27 negative tiles, 15,079 labels and `min_label_visible_ratio=0.25`; audit remains `needs_attention` because median label area is still below gate. +- Trained inactive local model asset `geointel-building-yolov8s-aoi1024visible025e50-pt` from the AOI1024 dataset. Ultralytics validation ended at approximately precision `0.275`, recall `0.331`, mAP50 `0.188` and mAP50-95 `0.0716`. +- The candidate remains inactive and must pass persisted detection QA/QC plus background/hard-negative promotion gates before default activation. +- No API contract, migration, product feature, provider fetching, fake detection data or active model default changed. + ## Sprint 150 YOLO label visible-ratio gate (2026-07-09) - Added `--min-label-visible-ratio` / `OPERATOR_YOLO_MIN_LABEL_VISIBLE_RATIO` to the operator YOLO tile dataset exporter. diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py index 31ef2b77..bd544b7e 100644 --- a/backend/tests/test_docker_runtime_config.py +++ b/backend/tests/test_docker_runtime_config.py @@ -118,6 +118,14 @@ def test_frontend_uses_same_origin_api_proxy_by_default() -> None: assert "try_files $uri $uri/ /index.html" in nginx_config +def test_nginx_runtime_allows_real_gis_upload_payloads() -> None: + frontend_nginx = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8") + all_in_one_nginx = (ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8") + + assert "client_max_body_size 250m;" in frontend_nginx + assert "client_max_body_size 250m;" in all_in_one_nginx + + def test_compose_does_not_publish_postgis_on_default_host_port() -> None: compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") diff --git a/backend/tests/test_sprint129_operator_yolo_training_dataset.py b/backend/tests/test_sprint129_operator_yolo_training_dataset.py index 79a9ec10..daa36cf7 100644 --- a/backend/tests/test_sprint129_operator_yolo_training_dataset.py +++ b/backend/tests/test_sprint129_operator_yolo_training_dataset.py @@ -60,6 +60,8 @@ def test_operator_yolo_train_smoke_script_contract() -> None: assert "TRAIN_MODEL_OUTPUT_PATH" in script assert "TRAIN_EPOCHS" in script assert "TRAIN_IMGSZ" in script + assert "/opt/geointel/venv/bin/python" in script + assert "PYTHON_BIN=\"python3\"" in script assert "dataset.yaml" in script assert "from ultralytics import YOLO" in script assert "model.train" in script diff --git a/deploy/unraid/nginx-all-in-one.conf b/deploy/unraid/nginx-all-in-one.conf index a9d8abb5..80864302 100644 --- a/deploy/unraid/nginx-all-in-one.conf +++ b/deploy/unraid/nginx-all-in-one.conf @@ -1,6 +1,7 @@ server { listen 80; server_name _; + client_max_body_size 250m; root /usr/share/nginx/html; index index.html; diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index a9f051e4..dcd2eb69 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -5937,3 +5937,33 @@ Open: ## Next recommended pass - Use `yolo-building-tile-uniquehardneg160` as the next safer hard-negative training dataset candidate. Benchmark after training before changing defaults. + +# Sprint 151 - Runtime GIS upload and AOI1024 YOLO candidate + +## What changed + +- Fixed `scripts/train_operator_yolo_detector.sh` so the all-in-one image uses `/opt/geointel/venv/bin/python` by default when that AI venv exists. Explicit `PYTHON_BIN` still wins, and local shells still fall back to `python3`. +- Raised the Nginx request body limit to `250m` in both `frontend/nginx.conf` and `deploy/unraid/nginx-all-in-one.conf` after the live 1024px GeoTIFF upload path returned `413 Request Entity Too Large`. +- Kept the change runtime-only: no API contract, persistence model, migration, model-download behavior or default model selection changed. + +## What was tested + +- Red/green TDD guard for the training wrapper fallback: + - `python -m pytest backend\tests\test_sprint129_operator_yolo_training_dataset.py -q` +- Red/green TDD guard for real GIS upload payload support: + - `python -m pytest backend\tests\test_docker_runtime_config.py::test_nginx_runtime_allows_real_gis_upload_payloads -q` +- `bash -n scripts/train_operator_yolo_detector.sh` +- Tower live model training on `/app/storage/operator-data/yolo-building-aoi1024-visible025`: + - output model: `/app/models/geointel-building-yolov8s-aoi1024visible025e50.pt` + - model asset id: `geointel-building-yolov8s-aoi1024visible025e50-pt` + - final Ultralytics validation: precision approximately `0.275`, recall `0.331`, mAP50 `0.188`, mAP50-95 `0.0716` + +## Known limitations + +- The AOI1024 tile audit is still `needs_attention`: median normalized box area is below gate and small-box share remains high. +- Several dense 1024 GRB reference exports reached the current 1000-feature source cap. Treat those samples as useful but potentially reference-capped until the provider query path supports paging or smaller dense AOIs are chosen. +- The trained model is intentionally inactive. It needs persisted detection QA/QC matrix evidence and background/hard-negative evidence before default promotion. + +## Next recommended pass + +- Rebuild/deploy the runtime upload-limit fix, rerun the four-sample AOI1024 persisted QA matrix, then decide whether label/source paging or additional AOI quality work comes before another training run. diff --git a/docs/TODO.md b/docs/TODO.md index ab3a14e0..1eb96c76 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -464,4 +464,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Prepare the new samples on Tower and build a fresh hard-negative tile dataset. - [x] Rebuild Tower all-in-one image so the newly copied operator scripts are available inside `/app/scripts` without `docker cp`. - [x] Fix YOLO preflight CLI so it respects Tower `.env` runtime configuration. -- [ ] Train a new candidate from `yolo-building-tile-uniquehardneg160` and run the positive/background promotion gates before activating it. +- [x] Train a new inactive AOI1024 YOLOv8s candidate with visible-label filtering. +- [x] Fix the all-in-one/compose Nginx upload limit after live 1024px GeoTIFF uploads hit `413 Request Entity Too Large`. +- [ ] Rerun persisted QA/QC matrix for `geointel-building-yolov8s-aoi1024visible025e50-pt` after redeploying the upload-limit fix. +- [ ] Add GRB paging or smaller dense AOI sampling before trusting 1000-feature-capped dense reference exports as full ground truth. +- [ ] Keep every local YOLO candidate inactive until positive-AOI and hard-negative promotion reports recommend default activation. diff --git a/frontend/nginx.conf b/frontend/nginx.conf index a599e822..f874f105 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -1,6 +1,7 @@ server { listen 80; server_name _; + client_max_body_size 250m; root /usr/share/nginx/html; index index.html; diff --git a/scripts/README.md b/scripts/README.md index 9904ac97..b6c1b91d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -318,7 +318,6 @@ docker exec \ -e TRAIN_BATCH=2 \ -e TRAIN_WORKERS=0 \ -e TRAIN_DEVICE=cpu \ - -e PYTHON_BIN=python3 \ geointel bash /app/scripts/train_operator_yolo_detector.sh ``` @@ -328,6 +327,9 @@ Ultralytics from the existing runtime, copies the best trained artifact to the resulting `.pt` file like any other local model asset: verify preflight, run the real-data matrix and compare persisted QA/QC metrics before activating it as a useful default. +Inside the all-in-one image the wrapper prefers +`/opt/geointel/venv/bin/python` when that AI runtime exists. Set `PYTHON_BIN` +only when intentionally overriding the interpreter. When whole-image training does not improve QA/QC, export a tile-level dataset with overlapping raster windows: @@ -383,6 +385,12 @@ Current Tower audit status: - `yolo-building-tile-uniquehardneg160`: clean expanded-background baseline; 576 tiles, 346 positive, 230 negative, 11,757 labels, 0 invalid labels and 0 repeated background negatives in the first Tower audit. +- `yolo-building-aoi1024-visible025`: larger AOI candidate baseline; 144 + tiles, 117 positive tiles, 27 negative tiles, 15,079 labels and + `min_label_visible_ratio=0.25`. Its audit remains `needs_attention` because + the median normalized box area is still small. The trained + `geointel-building-yolov8s-aoi1024visible025e50-pt` asset is inactive until + persisted QA/QC and hard-negative promotion gates pass. After rebuilding the all-in-one image, the operator scripts are available inside the container at `/app/scripts/...`. Before rebuilding, use the host checkout or @@ -424,7 +432,6 @@ docker exec \ -e TRAIN_BATCH=8 \ -e TRAIN_WORKERS=0 \ -e TRAIN_DEVICE=cpu \ - -e PYTHON_BIN=python3 \ geointel bash /app/scripts/train_operator_yolo_detector.sh ``` diff --git a/scripts/train_operator_yolo_detector.sh b/scripts/train_operator_yolo_detector.sh index c3dce548..8622e981 100644 --- a/scripts/train_operator_yolo_detector.sh +++ b/scripts/train_operator_yolo_detector.sh @@ -21,7 +21,8 @@ Environment variables: TRAIN_BATCH Batch size. Default: 2 TRAIN_WORKERS Data-loader workers. Default: 0 TRAIN_DEVICE Device passed to Ultralytics. Default: cpu - PYTHON_BIN Python executable. Default: python3 + PYTHON_BIN Python executable. Default: /opt/geointel/venv/bin/python + when present, otherwise python3. This helper is an operator/runtime smoke wrapper. It requires an existing local base model and an existing local dataset.yaml. It does not create app features. @@ -43,7 +44,13 @@ TRAIN_IMGSZ="${TRAIN_IMGSZ:-512}" TRAIN_BATCH="${TRAIN_BATCH:-2}" TRAIN_WORKERS="${TRAIN_WORKERS:-0}" TRAIN_DEVICE="${TRAIN_DEVICE:-cpu}" -PYTHON_BIN="${PYTHON_BIN:-python3}" +if [[ -z "${PYTHON_BIN:-}" ]]; then + if [[ -x "/opt/geointel/venv/bin/python" ]]; then + PYTHON_BIN="/opt/geointel/venv/bin/python" + else + PYTHON_BIN="python3" + fi +fi DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml" SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"