From b2fe7fa8bb694e7624f6080237e0c723eaf1ea9b Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 6 Jul 2026 23:55:58 +0200 Subject: [PATCH] Add model asset detection workflow smoke --- CHANGELOG.md | 7 + backend/README.md | 14 + ...20_model_asset_detection_workflow_smoke.py | 27 ++ docs/AI_PIPELINES.md | 14 + docs/CODEX_EXECUTION_LOG.md | 37 +++ docs/TODO.md | 2 + scripts/README.md | 16 ++ scripts/run_readiness_check.sh | 1 + .../verify_model_asset_detection_workflow.sh | 258 ++++++++++++++++++ 9 files changed, 376 insertions(+) create mode 100644 backend/tests/test_sprint120_model_asset_detection_workflow_smoke.py create mode 100644 scripts/verify_model_asset_detection_workflow.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 402f3d56..ccf7ca16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 120 Model asset detection workflow smoke (2026-07-06) + +- Added `scripts/verify_model_asset_detection_workflow.sh` for live Docker/Tower validation of the configured-YOLO path with a selected local model asset. +- The smoke seeds the explicit demo raster, generates a tile manifest, selects a cataloged model asset, checks read-only YOLO preflight, runs the existing detection endpoint and verifies persisted AnalysisRun, Detection list and Detection GeoJSON outputs. +- Registered the new smoke script in the readiness gate as a syntax check so ordinary CI/dev runs do not require AI dependencies or model files. +- Documented that the smoke validates operational routing/provenance only; zero detections are acceptable on the synthetic demo raster and real GIS quality still requires local orthophoto/reference validation. + ## Sprint 118 Local model and reference catalog clarity (2026-07-06) - Added a read-only local model asset catalog endpoint at `GET /api/v1/detection/model-assets`. diff --git a/backend/README.md b/backend/README.md index 73c81e88..67cbea38 100644 --- a/backend/README.md +++ b/backend/README.md @@ -338,6 +338,20 @@ The same read-only status is available through the API and Detection Lab UI: curl http://localhost:1202/api/v1/detection/yolo/preflight ``` +To validate the full configured-YOLO runtime path against Docker/Tower after a +model is mounted and selected, run: + +```bash +bash scripts/verify_model_asset_detection_workflow.sh http://192.168.10.150:1202 +``` + +The smoke uses the existing demo raster to generate a tile manifest, selects a +cataloged local model asset, verifies read-only preflight, submits the existing +detection run endpoint and checks persisted AnalysisRun, Detection list and +Detection GeoJSON output. It does not download weights or inject detector +fixtures. A zero detection result is still a valid runtime smoke outcome on the +synthetic demo raster. + ### Run backend ```bash diff --git a/backend/tests/test_sprint120_model_asset_detection_workflow_smoke.py b/backend/tests/test_sprint120_model_asset_detection_workflow_smoke.py new file mode 100644 index 00000000..28901d6f --- /dev/null +++ b/backend/tests/test_sprint120_model_asset_detection_workflow_smoke.py @@ -0,0 +1,27 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_model_asset_detection_workflow_smoke_is_registered_and_checks_configured_yolo_path() -> None: + script_path = ROOT / "scripts" / "verify_model_asset_detection_workflow.sh" + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert script_path.exists() + script = script_path.read_text(encoding="utf-8") + + assert "bash -n scripts/verify_model_asset_detection_workflow.sh" in readiness + assert "/api/v1/demo/workflow" in script + assert "/api/v1/detection/model-assets" in script + assert "/api/v1/detection/yolo/preflight" in script + assert "model_asset_id" in script + assert "tile_manifest_path" in script + assert "/api/v1/detection/run" in script + assert "/api/v1/detection/runs/" in script + assert "/detections" in script + assert "/geojson" in script + assert "Response is not a canonical GeoIntel data envelope" in script + assert "will_download_models" in script + assert "Fixture detections" not in script + assert "fixture_mode" not in script diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index f5555fd3..2d29b0c2 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -123,6 +123,20 @@ cataloged file for that run. The backend resolves the ID to a local path and persists the selected asset metadata in Job/AnalysisRun parameters. GeoIntel does not download weights or accept arbitrary model paths from the browser. +Operational runtime validation can be run against Docker/Tower with: + +```bash +bash scripts/verify_model_asset_detection_workflow.sh http://192.168.10.150:1202 +``` + +The smoke seeds the explicit offline demo raster, creates a tile manifest, +selects a local model asset, checks read-only preflight, runs the existing +configured-YOLO detection endpoint and verifies persisted AnalysisRun, +Detection list and Detection GeoJSON outputs. It intentionally does not inject +detector fixtures or download weights. A zero detection count is acceptable on +the synthetic demo raster; production usefulness still requires validation on +real georeferenced orthophotos and reference vectors. + ### Sprint 8C detection visualization and QA status Sprint 8C makes persisted detections reviewable: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index fea40ff8..2dc39e5d 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,40 @@ +## Sprint 120 Model asset detection workflow smoke (2026-07-06) + +Changed: +- Added `scripts/verify_model_asset_detection_workflow.sh` to validate the configured-YOLO runtime path against a live Docker/Tower deployment. +- The smoke seeds the explicit offline demo workflow, creates a raster tile manifest, selects the active local model asset from `GET /api/v1/detection/model-assets`, verifies read-only YOLO preflight, submits the existing detection run endpoint and checks persisted AnalysisRun, Detection list and Detection GeoJSON outputs. +- Registered the script in `scripts/run_readiness_check.sh` as a syntax check only, so ordinary readiness runs remain valid on machines without optional AI dependencies or mounted model files. +- Documented the smoke in `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`. + +Validation: +- RED: `python -m pytest backend/tests/test_sprint120_model_asset_detection_workflow_smoke.py -q` failed because `scripts/verify_model_asset_detection_workflow.sh` did not exist yet. +- `python -m pytest backend/tests/test_sprint120_model_asset_detection_workflow_smoke.py -q` passed: 1 test. +- `bash -n scripts/verify_model_asset_detection_workflow.sh` passed. +- Live Tower smoke passed: `bash scripts/verify_model_asset_detection_workflow.sh http://192.168.10.150:1202`. +- Live smoke selected `model_asset_id=yolov8n-building-segmentation-pt`, generated manifest `/app/storage/tiles/c0b00f1f-80bf-4992-be94-f5e5e6f6bf63/f9160f51-ee78-43b3-9353-d5390576fa1d/e9acd488-c376-45ed-b259-0dd79886f21e/manifest.json`, persisted analysis run `7f9e7ecb-c43d-4ed3-9f98-424bc0317805` and returned `detection_count=0`. +- `python -m compileall backend/app` passed. +- `cd backend && python -m pytest -q` passed: 384 tests with the existing Pydantic `model_*` namespace warnings. +- `cd frontend && npm run typecheck` passed. +- `cd frontend && npm run build` passed. +- `cd backend && python -m alembic heads` passed: `202606120900 (head)`. +- `cd backend && python -m alembic upgrade head --sql` passed. +- `bash scripts/run_readiness_check.sh` passed: 384 backend tests, frontend typecheck/build, API contract audit, Alembic head and shell syntax checks. +- Live browser/API smoke passed: `bash scripts/verify_browser_runtime.sh http://192.168.10.150:1202`. +- Live GIS capability smoke passed: `bash scripts/verify_gis_runtime.sh http://192.168.10.150:1202`. +- Live raster workflow smoke passed: `bash scripts/verify_demo_raster_workflow.sh http://192.168.10.150:1202`. +- Live workbench default-state smoke passed: `bash scripts/verify_workbench_default_state.sh http://192.168.10.150:1202`. +- Live workbench backing-state smoke passed: `bash scripts/verify_workbench_interactions.sh http://192.168.10.150:1202`. +- Live demo/export workflow smoke passed: `bash scripts/verify_demo_export_workflow.sh http://192.168.10.150:1202`. +- `bash scripts/verify_ai_handoff_interactions.sh http://192.168.10.150:1202` could not run in this local Codex shell because Node cannot import Playwright; the script remains syntax-checked in readiness and the internal browser was used for live visual verification instead. +- Internal browser validation passed on `http://192.168.10.150:1202`: AI Labs rendered Detection Lab and Segmentation Lab, selecting `yolo-configured` showed the Local model assets selector with `yolov8n-building-segmentation (active)` and `yolov8n`, no-download copy was visible and no console errors were emitted. + +Limitations: +- The smoke proves the configured-YOLO runtime path, provenance and persistence. It does not prove production model quality because it runs against the synthetic demo raster. +- Real operational validation still requires uploading a georeferenced Kempen orthophoto/GeoTIFF, running the configured building model on that raster and comparing persisted detections against reference building vectors through QA/QC. + +Next recommended pass: +- Create the real-data validation path for orthophoto upload, tile generation, configured building-model run and reference-vector QA/QC. + ## Sprint 118 Local model and reference catalog clarity (2026-07-06) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index 43502668..93b0c1be 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -82,6 +82,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add opt-in Docker/Unraid AI build/runtime path for local PyTorch/Ultralytics YOLO operation. - [x] Surface configured-YOLO runtime preflight status through the API and Detection Lab UI. - [x] Add read-only local model asset catalog and Detection Lab model-file selection. +- [x] Add live model asset detection workflow smoke for configured-YOLO runtime/provenance validation. - [x] Add one-click full GIS workflow action for query, derived dataset, QA/QC and export handoff. - [x] Add QA/QC workspace result hierarchy and filter density polish. - [x] Add Change Detection panel hierarchy and analysis workspace density polish. @@ -90,6 +91,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add QA/QC and Exports usability layout pass with calmer evidence review and handoff artifact scanning. - [x] Add AI Labs Detection/Segmentation hierarchy and result density polish. - [x] Add Export/System handoff hierarchy and provider registry density polish. +- [ ] Validate the configured building model on a real georeferenced Kempen orthophoto/GeoTIFF with persisted reference vectors and QA/QC metrics. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index b884ed99..e3c83fba 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -135,6 +135,22 @@ The model-load smoke is opt-in, requires real optional AI dependencies, refuses `--assume-dependencies`, loads only the supplied local file and does not download weights or run prediction. +Verify the full configured-YOLO model asset workflow against a running runtime: + +```bash +bash scripts/verify_model_asset_detection_workflow.sh http://192.168.10.150:1202 +``` + +This smoke is intentionally mutating and requires a real AI-enabled runtime with +at least one mounted local model asset. It seeds the explicit offline demo +workflow, generates a small raster tile manifest, selects the active local model +asset from `GET /api/v1/detection/model-assets`, validates read-only YOLO +preflight, runs `POST /api/v1/detection/run`, and verifies the persisted +AnalysisRun, Detection list and Detection GeoJSON endpoints. A zero detection +count is allowed because the demo raster is a synthetic runtime fixture; the +script validates the operational path and provenance, not production model +quality. The main readiness gate checks this script's syntax only. + Docker images install only the GIS runtime by default. To build a local/Tower image with PyTorch/Ultralytics available for the configured-YOLO preflight and runtime path, set: diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 1b119604..e50e3240 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -53,6 +53,7 @@ bash -n scripts/verify_browser_runtime.sh bash -n scripts/verify_demo_export_workflow.sh bash -n scripts/verify_demo_raster_workflow.sh bash -n scripts/verify_ai_handoff_interactions.sh +bash -n scripts/verify_model_asset_detection_workflow.sh bash -n scripts/verify_workbench_default_state.sh bash -n scripts/verify_workbench_interactions.sh bash -n scripts/verify_gis_runtime.sh diff --git a/scripts/verify_model_asset_detection_workflow.sh b/scripts/verify_model_asset_detection_workflow.sh new file mode 100644 index 00000000..b0a5e985 --- /dev/null +++ b/scripts/verify_model_asset_detection_workflow.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +if ! command -v curl >/dev/null 2>&1; then + echo "curl is required for model asset detection workflow verification" >&2 + exit 1 +fi + +if [ -n "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="${PYTHON_BIN}" +else + PYTHON_BIN="" + for candidate in python3 python.exe python; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then + PYTHON_BIN="${candidate}" + break + fi + done +fi + +if [ -z "${PYTHON_BIN}" ]; then + echo "A Python interpreter is required for JSON parsing" >&2 + exit 1 +fi + +json_field() { + local file_path="$1" + local expression="$2" + "${PYTHON_BIN}" - "$file_path" "$expression" <<'PY' +import json +import sys + +path, expression = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + payload = json.load(handle) +value = payload +for part in expression.split("."): + if part: + value = value[part] +print(value) +PY +} + +require_json_data() { + local file_path="$1" + "${PYTHON_BIN}" - "$file_path" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as handle: + payload = json.load(handle) +if "data" not in payload: + raise SystemExit("Response is not a canonical GeoIntel data envelope") +PY +} + +echo "== GeoIntel model asset detection workflow verification ==" +echo "Base URL: ${BASE_URL}" + +curl -fsS -X POST "${BASE_URL%/}/api/v1/demo/workflow" > "${TMP_DIR}/demo.json" +require_json_data "${TMP_DIR}/demo.json" +project_id="$(json_field "${TMP_DIR}/demo.json" "data.project_id")" +raster_dataset_id="$(json_field "${TMP_DIR}/demo.json" "data.raster_dataset_id")" + +if [ -z "${project_id}" ] || [ "${project_id}" = "None" ] || [ "${project_id}" = "null" ]; then + echo "Demo workflow did not return a project_id" >&2 + exit 1 +fi +if [ -z "${raster_dataset_id}" ] || [ "${raster_dataset_id}" = "None" ] || [ "${raster_dataset_id}" = "null" ]; then + echo "Demo workflow did not return a raster_dataset_id" >&2 + exit 1 +fi + +curl -fsS -X POST "${BASE_URL%/}/api/v1/projects/${project_id}/datasets/${raster_dataset_id}/raster/tile" \ + -H "Content-Type: application/json" \ + -d '{"tile_size":64,"overlap":0,"output_name":"model_asset_detection_smoke_tiles"}' > "${TMP_DIR}/tile.json" +require_json_data "${TMP_DIR}/tile.json" +manifest_path="$(json_field "${TMP_DIR}/tile.json" "data.result_json.manifest_path")" + +if [ -z "${manifest_path}" ] || [ "${manifest_path}" = "None" ] || [ "${manifest_path}" = "null" ]; then + echo "Raster tile response did not include a manifest_path" >&2 + exit 1 +fi + +curl -fsS "${BASE_URL%/}/api/v1/detection/model-assets" > "${TMP_DIR}/model_assets.json" +require_json_data "${TMP_DIR}/model_assets.json" +model_asset_id="$("${PYTHON_BIN}" - "${TMP_DIR}/model_assets.json" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +items = data.get("items") or [] +if not items: + raise SystemExit("No local model assets are available. Mount a local .pt/.onnx/.engine file before running this smoke.") +for item in items: + if item.get("will_download_models") is not False: + raise SystemExit("Model asset catalog must never report automatic model downloads") +selected = next((item for item in items if item.get("active")), items[0]) +print(selected["model_asset_id"]) +PY +)" + +curl -fsS -G "${BASE_URL%/}/api/v1/detection/yolo/preflight" \ + --data-urlencode "tile_manifest_path=${manifest_path}" \ + --data-urlencode "model_asset_id=${model_asset_id}" > "${TMP_DIR}/preflight.json" +require_json_data "${TMP_DIR}/preflight.json" +"${PYTHON_BIN}" - "${TMP_DIR}/preflight.json" "${model_asset_id}" <<'PY' +import json +import sys + +path, expected_model_asset_id = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("model_asset_id") != expected_model_asset_id: + raise SystemExit("YOLO preflight did not use the selected model_asset_id") +if data.get("will_download_models") is not False: + raise SystemExit("YOLO preflight must never download model weights") +if data.get("will_run_inference") is not False: + raise SystemExit("YOLO preflight must remain read-only") +if data.get("status") != "ready": + raise SystemExit(f"YOLO preflight is not ready: {data.get('status')} {data.get('message')}") +checks = data.get("checks") or {} +if checks.get("manifest_valid") is not True: + raise SystemExit("YOLO preflight did not validate the raster tile manifest") +if checks.get("model_file_exists") is not True: + raise SystemExit("YOLO preflight did not confirm the local model file") +PY + +"${PYTHON_BIN}" - "${TMP_DIR}/run_request.json" "${project_id}" "${raster_dataset_id}" "${model_asset_id}" "${manifest_path}" <<'PY' +import json +import sys + +path, project_id, dataset_id, model_asset_id, tile_manifest_path = sys.argv[1:6] +payload = { + "project_id": project_id, + "dataset_id": dataset_id, + "model_id": "yolo-configured", + "model_asset_id": model_asset_id, + "confidence_threshold": 0.5, + "tile_manifest_path": tile_manifest_path, + "parameters_json": {}, +} +with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) +PY + +curl -fsS -X POST "${BASE_URL%/}/api/v1/detection/run" \ + -H "Content-Type: application/json" \ + --data-binary "@${TMP_DIR}/run_request.json" > "${TMP_DIR}/detection_run.json" +require_json_data "${TMP_DIR}/detection_run.json" +"${PYTHON_BIN}" - "${TMP_DIR}/detection_run.json" "${model_asset_id}" <<'PY' +import json +import sys + +path, expected_model_asset_id = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("model_id") != "yolo-configured": + raise SystemExit("Detection run did not use yolo-configured") +if not data.get("analysis_run_id") or not data.get("job_id"): + raise SystemExit("Detection run did not return persisted run/job ids") +if data.get("status") != "success": + raise SystemExit(f"Detection run failed: {data.get('error_code')} {data.get('message')}") +if int(data.get("detection_count") or 0) < 0: + raise SystemExit("Detection count cannot be negative") +PY + +analysis_run_id="$(json_field "${TMP_DIR}/detection_run.json" "data.analysis_run_id")" +detection_count="$(json_field "${TMP_DIR}/detection_run.json" "data.detection_count")" + +curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}" > "${TMP_DIR}/run_detail.json" +require_json_data "${TMP_DIR}/run_detail.json" +"${PYTHON_BIN}" - "${TMP_DIR}/run_detail.json" "${analysis_run_id}" "${model_asset_id}" "${manifest_path}" <<'PY' +import json +import sys + +path, analysis_run_id, model_asset_id, tile_manifest_path = sys.argv[1:5] +with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("id") != analysis_run_id: + raise SystemExit("Detection run detail returned the wrong run id") +if data.get("status") != "success": + raise SystemExit(f"Detection run detail is not successful: {data.get('status')}") +parameters = data.get("parameters_json") or {} +if parameters.get("model_asset_id") != model_asset_id: + raise SystemExit("Persisted run parameters lost model_asset_id provenance") +if parameters.get("tile_manifest_path") != tile_manifest_path: + raise SystemExit("Persisted run parameters lost tile_manifest_path provenance") +PY + +curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}/detections" > "${TMP_DIR}/detections.json" +require_json_data "${TMP_DIR}/detections.json" +"${PYTHON_BIN}" - "${TMP_DIR}/detections.json" "${detection_count}" <<'PY' +import json +import sys + +path, expected_count = sys.argv[1], int(sys.argv[2]) +with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +items = data.get("items") or [] +if int(data.get("total") or 0) != expected_count: + raise SystemExit("Detection list total does not match run detection_count") +if len(items) != expected_count: + raise SystemExit("Detection list item count does not match run detection_count") +for item in items: + if not item.get("geometry"): + raise SystemExit("Persisted detection is missing geometry") + if not item.get("source_tile_path"): + raise SystemExit("Persisted YOLO detection is missing source_tile_path provenance") +PY + +curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}/geojson" > "${TMP_DIR}/geojson.json" +require_json_data "${TMP_DIR}/geojson.json" +"${PYTHON_BIN}" - "${TMP_DIR}/geojson.json" "${detection_count}" <<'PY' +import json +import sys + +path, expected_count = sys.argv[1], int(sys.argv[2]) +with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("type") != "FeatureCollection": + raise SystemExit("Detection GeoJSON response is not a FeatureCollection") +features = data.get("features") or [] +if len(features) != expected_count: + raise SystemExit("Detection GeoJSON feature count does not match run detection_count") +required_properties = { + "detection_id", + "class_name", + "confidence", + "model_name", + "model_version", + "analysis_run_id", + "dataset_id", + "job_id", + "source_tile_path", + "bbox_json", +} +for feature in features: + properties = feature.get("properties") or {} + missing = sorted(required_properties - set(properties)) + if missing: + raise SystemExit(f"Detection GeoJSON feature is missing properties: {missing}") + if not feature.get("geometry"): + raise SystemExit("Detection GeoJSON feature is missing geometry") +PY + +echo "Model asset detection workflow verification passed" +echo "Project: ${project_id}" +echo "Raster dataset: ${raster_dataset_id}" +echo "Model asset: ${model_asset_id}" +echo "Manifest: ${manifest_path}" +echo "Analysis run: ${analysis_run_id}" +echo "Detections: ${detection_count}"