From f0a58011fefa08646644b7b45a5f513229c4de12 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 7 Jul 2026 00:19:23 +0200 Subject: [PATCH] Add real data detection QA workflow smoke --- CHANGELOG.md | 8 + backend/README.md | 18 + ..._sprint121_real_data_detection_qa_smoke.py | 33 ++ docs/AI_PIPELINES.md | 28 + docs/CODEX_EXECUTION_LOG.md | 27 + docs/TODO.md | 1 + scripts/README.md | 21 + scripts/run_readiness_check.sh | 1 + .../verify_real_data_detection_qa_workflow.sh | 504 ++++++++++++++++++ 9 files changed, 641 insertions(+) create mode 100644 backend/tests/test_sprint121_real_data_detection_qa_smoke.py create mode 100644 scripts/verify_real_data_detection_qa_workflow.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index ccf7ca16..6fc7a4d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ # Changelog +## Sprint 121 Real data detection and QA workflow smoke (2026-07-07) + +- Added `scripts/verify_real_data_detection_qa_workflow.sh` for operator-provided GeoTIFF/reference-vector validation against a live runtime. +- The smoke uploads a real raster source dataset and real reference building vector, validates GIS metadata, tiles the raster, selects a mounted local model asset, runs configured YOLO detection, runs persisted detection QA/QC and exports detection GeoJSON. +- Registered the script in the readiness gate as a syntax check so normal development remains green without real local imagery or model files. +- Documented exact Tower usage in `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md` and `docs/TODO.md`. +- The script refuses missing files, unsupported formats, demo workflow seeding, fixture detections, live provider fetching and model downloads. + ## Sprint 120 Model asset detection workflow smoke (2026-07-06) - Added `scripts/verify_model_asset_detection_workflow.sh` for live Docker/Tower validation of the configured-YOLO path with a selected local model asset. diff --git a/backend/README.md b/backend/README.md index 67cbea38..4e0323d9 100644 --- a/backend/README.md +++ b/backend/README.md @@ -352,6 +352,24 @@ Detection GeoJSON output. It does not download weights or inject detector fixtures. A zero detection result is still a valid runtime smoke outcome on the synthetic demo raster. +To validate the configured building model on operator-provided GIS data, mount +or copy a real georeferenced raster and a real reference-building GeoJSON onto +the runtime host, then run: + +```bash +REAL_RASTER_PATH=/mnt/user/appdata/geointel/data/orthophoto.tif \ +REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/data/reference-buildings.geojson \ +bash scripts/verify_real_data_detection_qa_workflow.sh http://192.168.10.150:1202 +``` + +This smoke refuses missing/unsupported inputs, uploads the raster and reference +dataset through the normal dataset service, generates raster tiles, selects a +local model asset, runs configured YOLO detection, compares persisted +detections against persisted `vector_features`, persists QA/QC rows and exports +the detection GeoJSON. It never seeds demo detections, enables fixture mode, +fetches live providers or downloads model weights. Current V1 upload support is +limited to GeoTIFF-style rasters and GeoJSON/JSON reference vectors. + ### Run backend ```bash diff --git a/backend/tests/test_sprint121_real_data_detection_qa_smoke.py b/backend/tests/test_sprint121_real_data_detection_qa_smoke.py new file mode 100644 index 00000000..85cb0a88 --- /dev/null +++ b/backend/tests/test_sprint121_real_data_detection_qa_smoke.py @@ -0,0 +1,33 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_real_data_detection_qa_smoke_requires_operator_inputs_and_checks_full_chain() -> None: + script_path = ROOT / "scripts" / "verify_real_data_detection_qa_workflow.sh" + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert script_path.exists() + script = script_path.read_text(encoding="utf-8") + + assert "bash -n scripts/verify_real_data_detection_qa_workflow.sh" in readiness + assert "REAL_RASTER_PATH" in script + assert "REAL_REFERENCE_VECTOR_PATH" in script + assert "usage()" in script + assert "/api/v1/projects" in script + assert "/datasets/upload" in script + assert "dataset_role=reference" in script + assert "reference_layer_name=buildings" in script + assert "/raster/inspect" in script + assert "/raster/tile" in script + assert "/api/v1/detection/model-assets" in script + assert "/api/v1/detection/yolo/preflight" in script + assert "/api/v1/detection/run" in script + assert "/qa/reference" in script + assert "/api/v1/exports/geojson" in script + assert "Response is not a canonical GeoIntel data envelope" in script + assert "No local model assets are available" in script + assert "demo/workflow" not in script + assert "fixture_mode" not in script + assert "Fixture detections" not in script diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 2d29b0c2..4cf3b209 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -137,6 +137,34 @@ detector fixtures or download weights. A zero detection count is acceptable on the synthetic demo raster; production usefulness still requires validation on real georeferenced orthophotos and reference vectors. +### Real-data detection and QA validation + +The real operational validation path uses operator-provided files rather than +demo fixtures: + +```bash +REAL_RASTER_PATH=/mnt/user/appdata/geointel/data/orthophoto.tif \ +REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/data/reference-buildings.geojson \ +bash scripts/verify_real_data_detection_qa_workflow.sh http://192.168.10.150:1202 +``` + +The script verifies the full persisted chain: + +- source raster upload with CRS and bounds metadata; +- reference building vector upload as `dataset_role=reference`; +- raster inspect and tile manifest generation; +- local model asset selection and read-only YOLO preflight; +- configured-YOLO detection run through Job, AnalysisRun and Detection rows; +- detection GeoJSON generated from persisted geometry; +- detection QA against persisted reference `vector_features` with persisted + `QualityCheck` and `Metric` rows; +- detection run GeoJSON export. + +It refuses to run without a real GeoTIFF-style raster and GeoJSON/JSON reference +vector. It does not seed demo data, use `fixture_mode`, fetch live providers or +download model weights. A zero detection count is valid as runtime evidence but +does not prove the model is useful for the target imagery. + ### Sprint 8C detection visualization and QA status Sprint 8C makes persisted detections reviewable: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 3fa2b1cc..a411706f 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,30 @@ +## Sprint 121 Real data detection and QA workflow smoke (2026-07-07) + +Changed: +- Added `scripts/verify_real_data_detection_qa_workflow.sh` for live-runtime validation with operator-provided real GIS inputs. +- The smoke creates a project, uploads a real GeoTIFF-style raster as a source dataset, uploads a real reference-building GeoJSON/JSON as `dataset_role=reference`, validates CRS/bounds/features, tiles the raster, selects a mounted local model asset, runs read-only YOLO preflight, runs configured YOLO detection, runs detection QA against persisted `vector_features`, and exports the detection run GeoJSON. +- Registered the smoke in `scripts/run_readiness_check.sh` as a syntax check only, so normal readiness does not require real orthophotos, reference vectors, optional AI dependencies or model files. +- Documented Tower usage and limitations in `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`. + +Validation: +- RED: `python -m pytest backend/tests/test_sprint121_real_data_detection_qa_smoke.py -q` failed while `scripts/verify_real_data_detection_qa_workflow.sh` did not exist. +- `python -m pytest backend/tests/test_sprint121_real_data_detection_qa_smoke.py -q` passed: 1 test. +- `bash -n scripts/verify_real_data_detection_qa_workflow.sh` passed. +- `bash scripts/verify_real_data_detection_qa_workflow.sh --help` passed and printed required `REAL_RASTER_PATH` and `REAL_REFERENCE_VECTOR_PATH` usage. +- `python -m compileall backend/app` passed. +- `python scripts/smoke_docs.py` passed. +- `bash scripts/run_readiness_check.sh` passed: 385 backend tests, Alembic head check, frontend typecheck, frontend production build and shell syntax checks. +- `cd backend && python -m alembic upgrade head --sql` passed. +- Missing-input guard passed: `bash scripts/verify_real_data_detection_qa_workflow.sh http://localhost:1202` returned exit code 2 and printed usage. +- Local `docker compose config` could not run in this Windows Codex environment because the `docker` command is not installed. + +Limitations: +- The full real-data smoke was not executed in this Codex workspace because no operator-provided real GeoTIFF and reference GeoJSON were found locally. +- The script enforces real inputs and never seeds demo data, enables fixture detections, fetches live GRB/OSM/Sentinel data or downloads model weights. + +Next recommended pass: +- Place a target orthophoto/GeoTIFF and matching reference-building GeoJSON under the Tower appdata path and run `REAL_RASTER_PATH=... REAL_REFERENCE_VECTOR_PATH=... bash scripts/verify_real_data_detection_qa_workflow.sh http://192.168.10.150:1202`. + ## Sprint 120 Model asset detection workflow smoke (2026-07-06) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index 93b0c1be..71ad0444 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -91,6 +91,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add QA/QC and Exports usability layout pass with calmer evidence review and handoff artifact scanning. - [x] Add AI Labs Detection/Segmentation hierarchy and result density polish. - [x] Add Export/System handoff hierarchy and provider registry density polish. +- [x] Add operator-provided real raster/reference detection + QA workflow smoke. - [ ] Validate the configured building model on a real georeferenced Kempen orthophoto/GeoTIFF with persisted reference vectors and QA/QC metrics. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index e3c83fba..f62624b5 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -151,6 +151,27 @@ count is allowed because the demo raster is a synthetic runtime fixture; the script validates the operational path and provenance, not production model quality. The main readiness gate checks this script's syntax only. +Verify the full operator-provided raster/reference detection and QA path: + +```bash +REAL_RASTER_PATH=/mnt/user/appdata/geointel/data/orthophoto.tif \ +REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/data/reference-buildings.geojson \ +bash scripts/verify_real_data_detection_qa_workflow.sh http://192.168.10.150:1202 +``` + +The real-data smoke is intentionally mutating and refuses to run without +operator-supplied files. Current V1 upload support expects a georeferenced +`.tif`, `.tiff` or `.geotiff` raster and a `.geojson` or `.json` reference +building vector. The script creates a project, uploads the raster as a source +dataset, uploads the vector as a `reference` dataset, validates raster/vector +metadata, tiles the raster, selects a mounted local model asset, verifies +read-only YOLO preflight, runs configured YOLO detection, runs detection QA +against persisted `vector_features`, and exports the detection run as GeoJSON. +It does not seed demo data, enable fixture detections, fetch external data or +download model weights. A zero detection count is accepted operationally, but +must be interpreted as model/data quality evidence rather than as a successful +building extraction result. + Docker images install only the GIS runtime by default. To build a local/Tower image with PyTorch/Ultralytics available for the configured-YOLO preflight and runtime path, set: diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index e50e3240..71b2b8a5 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -54,6 +54,7 @@ bash -n scripts/verify_demo_export_workflow.sh bash -n scripts/verify_demo_raster_workflow.sh bash -n scripts/verify_ai_handoff_interactions.sh bash -n scripts/verify_model_asset_detection_workflow.sh +bash -n scripts/verify_real_data_detection_qa_workflow.sh bash -n scripts/verify_workbench_default_state.sh bash -n scripts/verify_workbench_interactions.sh bash -n scripts/verify_gis_runtime.sh diff --git a/scripts/verify_real_data_detection_qa_workflow.sh b/scripts/verify_real_data_detection_qa_workflow.sh new file mode 100644 index 00000000..b226ce4d --- /dev/null +++ b/scripts/verify_real_data_detection_qa_workflow.sh @@ -0,0 +1,504 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: + REAL_RASTER_PATH=/path/to/orthophoto.tif \ + REAL_REFERENCE_VECTOR_PATH=/path/to/reference-buildings.geojson \ + bash scripts/verify_real_data_detection_qa_workflow.sh [base_url] + +or: + bash scripts/verify_real_data_detection_qa_workflow.sh [base_url] /path/to/orthophoto.tif /path/to/reference-buildings.geojson + +Required inputs: + REAL_RASTER_PATH Georeferenced .tif/.tiff/.geotiff raster. + REAL_REFERENCE_VECTOR_PATH EPSG-aware .geojson/.json reference vector with building polygons. + +Optional environment: + REAL_PROJECT_NAME Project name for the validation run. + REAL_MODEL_ASSET_ID Specific /api/v1/detection/model-assets id to use. + REAL_TILE_SIZE Raster tile size, default 640. + REAL_TILE_OVERLAP Raster tile overlap, default 64. + REAL_CONFIDENCE_THRESHOLD Detection confidence threshold, default 0.5. + REAL_IOU_THRESHOLD QA IoU threshold, default 0.5. +EOF +} + +BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" +REAL_RASTER_PATH="${2:-${REAL_RASTER_PATH:-}}" +REAL_REFERENCE_VECTOR_PATH="${3:-${REAL_REFERENCE_VECTOR_PATH:-}}" +REAL_PROJECT_NAME="${REAL_PROJECT_NAME:-GeoIntel Real Data Validation}" +REAL_TILE_SIZE="${REAL_TILE_SIZE:-640}" +REAL_TILE_OVERLAP="${REAL_TILE_OVERLAP:-64}" +REAL_CONFIDENCE_THRESHOLD="${REAL_CONFIDENCE_THRESHOLD:-0.5}" +REAL_IOU_THRESHOLD="${REAL_IOU_THRESHOLD:-0.5}" +REAL_MODEL_ASSET_ID="${REAL_MODEL_ASSET_ID:-}" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then + usage + exit 0 +fi + +if [ -z "${REAL_RASTER_PATH}" ] || [ -z "${REAL_REFERENCE_VECTOR_PATH}" ]; then + usage + exit 2 +fi + +if [ ! -f "${REAL_RASTER_PATH}" ]; then + echo "REAL_RASTER_PATH does not point to a readable file: ${REAL_RASTER_PATH}" >&2 + exit 2 +fi +if [ ! -f "${REAL_REFERENCE_VECTOR_PATH}" ]; then + echo "REAL_REFERENCE_VECTOR_PATH does not point to a readable file: ${REAL_REFERENCE_VECTOR_PATH}" >&2 + exit 2 +fi + +case "${REAL_RASTER_PATH,,}" in + *.tif|*.tiff|*.geotiff) ;; + *) + echo "REAL_RASTER_PATH must be a .tif, .tiff or .geotiff file for the current V1 raster upload flow." >&2 + exit 2 + ;; +esac + +case "${REAL_REFERENCE_VECTOR_PATH,,}" in + *.geojson|*.json) ;; + *) + echo "REAL_REFERENCE_VECTOR_PATH must be .geojson or .json for the current V1 vector upload flow." >&2 + exit 2 + ;; +esac + +if ! command -v curl >/dev/null 2>&1; then + echo "curl is required for real data detection/QA workflow verification" >&2 + exit 1 +fi + +if [ -n "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="${PYTHON_BIN}" +else + PYTHON_BIN="" + for candidate in python3 python.exe python; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then + PYTHON_BIN="${candidate}" + break + fi + done +fi + +if [ -z "${PYTHON_BIN}" ]; then + echo "A Python interpreter is required for JSON parsing" >&2 + exit 1 +fi + +json_field() { + local file_path="$1" + local expression="$2" + "${PYTHON_BIN}" - "$file_path" "$expression" <<'PY' +import json +import sys + +path, expression = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + payload = json.load(handle) +value = payload +for part in expression.split("."): + if part: + value = value[part] +print(value) +PY +} + +require_json_data() { + local file_path="$1" + "${PYTHON_BIN}" - "$file_path" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as handle: + payload = json.load(handle) +if "data" not in payload: + raise SystemExit("Response is not a canonical GeoIntel data envelope") +PY +} + +echo "== GeoIntel real data detection + QA workflow verification ==" +echo "Base URL: ${BASE_URL}" +echo "Raster: ${REAL_RASTER_PATH}" +echo "Reference vector: ${REAL_REFERENCE_VECTOR_PATH}" + +"${PYTHON_BIN}" - "${TMP_DIR}/project_request.json" "${REAL_PROJECT_NAME}" <<'PY' +import json +import sys +from datetime import datetime, timezone + +path, project_name = sys.argv[1], sys.argv[2] +stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") +payload = { + "name": f"{project_name} {stamp}", + "description": "Operator-provided real data validation: raster upload, reference vector upload, configured YOLO detection, QA/QC and GeoJSON export.", + "region": "Kempen", +} +with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) +PY + +curl -fsS -X POST "${BASE_URL%/}/api/v1/projects" \ + -H "Content-Type: application/json" \ + --data-binary "@${TMP_DIR}/project_request.json" > "${TMP_DIR}/project.json" +require_json_data "${TMP_DIR}/project.json" +project_id="$(json_field "${TMP_DIR}/project.json" "data.id")" +if [ -z "${project_id}" ] || [ "${project_id}" = "None" ] || [ "${project_id}" = "null" ]; then + echo "Project creation did not return a project id" >&2 + exit 1 +fi + +curl -fsS -X POST "${BASE_URL%/}/api/v1/projects/${project_id}/datasets/upload" \ + -F "file=@${REAL_RASTER_PATH}" \ + -F "dataset_type=raster" \ + -F "source=user_upload" \ + -F "dataset_role=source" \ + -F "source_name=manual" \ + -F 'source_metadata_json={"validation_workflow":"real_data_detection_qa","input_kind":"orthophoto"}' \ + -F 'provenance_metadata_json={"operator_supplied":true,"no_external_fetch":true}' \ + > "${TMP_DIR}/raster_upload.json" +require_json_data "${TMP_DIR}/raster_upload.json" +raster_dataset_id="$(json_field "${TMP_DIR}/raster_upload.json" "data.id")" + +"${PYTHON_BIN}" - "${TMP_DIR}/raster_upload.json" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("dataset_type") != "raster": + raise SystemExit(f"Uploaded raster returned wrong dataset_type: {data.get('dataset_type')}") +if data.get("status") != "ready": + raise SystemExit(f"Uploaded raster is not ready: {data.get('status')}") +if not data.get("crs"): + raise SystemExit("Uploaded raster does not expose CRS metadata; refusing geospatial AI validation") +if not data.get("bounds_json"): + raise SystemExit("Uploaded raster does not expose bounds metadata; refusing geospatial AI validation") +PY + +curl -fsS -X POST "${BASE_URL%/}/api/v1/projects/${project_id}/datasets/upload" \ + -F "file=@${REAL_REFERENCE_VECTOR_PATH}" \ + -F "dataset_type=vector" \ + -F "source=user_upload" \ + -F "dataset_role=reference" \ + -F "source_name=manual" \ + -F "reference_layer_name=buildings" \ + -F 'source_metadata_json={"validation_workflow":"real_data_detection_qa","input_kind":"reference_buildings"}' \ + -F 'provenance_metadata_json={"operator_supplied":true,"no_external_fetch":true}' \ + > "${TMP_DIR}/reference_upload.json" +require_json_data "${TMP_DIR}/reference_upload.json" +reference_dataset_id="$(json_field "${TMP_DIR}/reference_upload.json" "data.id")" + +"${PYTHON_BIN}" - "${TMP_DIR}/reference_upload.json" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("dataset_type") != "vector": + raise SystemExit(f"Uploaded reference returned wrong dataset_type: {data.get('dataset_type')}") +if data.get("dataset_role") != "reference": + raise SystemExit("Uploaded reference did not persist dataset_role=reference") +if data.get("reference_layer_name") != "buildings": + raise SystemExit("Uploaded reference did not persist reference_layer_name=buildings") +if data.get("status") != "ready": + raise SystemExit(f"Uploaded reference is not ready: {data.get('status')}") +if int(data.get("feature_count") or 0) < 1: + raise SystemExit("Uploaded reference has no persisted features; QA would be meaningless") +PY + +curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/datasets/${raster_dataset_id}/raster/inspect" > "${TMP_DIR}/raster_inspect.json" +require_json_data "${TMP_DIR}/raster_inspect.json" +"${PYTHON_BIN}" - "${TMP_DIR}/raster_inspect.json" "${raster_dataset_id}" <<'PY' +import json +import sys + +path, raster_id = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("dataset_id") != raster_id: + raise SystemExit("Raster inspect returned the wrong dataset id") +if data.get("ready") is not True: + raise SystemExit("Raster inspect did not report ready=true") +metadata = data.get("metadata") or {} +if not metadata.get("crs"): + raise SystemExit("Raster inspect metadata has no CRS") +if int(metadata.get("width") or 0) < 1 or int(metadata.get("height") or 0) < 1: + raise SystemExit("Raster inspect metadata has invalid dimensions") +PY + +curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/datasets/${reference_dataset_id}/vector/summary" > "${TMP_DIR}/reference_summary.json" +require_json_data "${TMP_DIR}/reference_summary.json" +"${PYTHON_BIN}" - "${TMP_DIR}/reference_summary.json" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if int(data.get("feature_count") or 0) < 1: + raise SystemExit("Reference vector summary reports no features") +geometry_types = set(data.get("geometry_types") or []) +if not geometry_types.intersection({"Polygon", "MultiPolygon"}): + raise SystemExit(f"Reference vector does not expose polygonal building geometries: {sorted(geometry_types)}") +PY + +curl -fsS -X POST "${BASE_URL%/}/api/v1/projects/${project_id}/datasets/${raster_dataset_id}/raster/tile" \ + -H "Content-Type: application/json" \ + -d "{\"tile_size\":${REAL_TILE_SIZE},\"overlap\":${REAL_TILE_OVERLAP},\"output_name\":\"real_data_detection_tiles\"}" > "${TMP_DIR}/tile.json" +require_json_data "${TMP_DIR}/tile.json" +manifest_path="$(json_field "${TMP_DIR}/tile.json" "data.result_json.manifest_path")" +if [ -z "${manifest_path}" ] || [ "${manifest_path}" = "None" ] || [ "${manifest_path}" = "null" ]; then + echo "Raster tile response did not include a manifest_path" >&2 + exit 1 +fi +"${PYTHON_BIN}" - "${TMP_DIR}/tile.json" "${raster_dataset_id}" <<'PY' +import json +import sys + +path, raster_id = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + job = json.load(handle)["data"] +if job.get("job_type") != "raster.tile": + raise SystemExit(f"Unexpected tile job type: {job.get('job_type')}") +if job.get("status") not in {"success", "completed"}: + raise SystemExit(f"Raster tile job did not complete: {job.get('status')}") +result = job.get("result_json") or {} +if result.get("dataset_id") != raster_id: + raise SystemExit("Raster tile result returned the wrong dataset id") +if result.get("ready") is not True: + raise SystemExit("Raster tile result did not report ready=true") +manifest = result.get("manifest") or {} +tiles = manifest.get("tiles") or [] +if not tiles: + raise SystemExit("Raster tile manifest has no tiles") +if manifest.get("source_dataset_id") != raster_id: + raise SystemExit("Raster tile manifest source_dataset_id drifted") +PY + +curl -fsS "${BASE_URL%/}/api/v1/detection/model-assets" > "${TMP_DIR}/model_assets.json" +require_json_data "${TMP_DIR}/model_assets.json" +model_asset_id="$("${PYTHON_BIN}" - "${TMP_DIR}/model_assets.json" "${REAL_MODEL_ASSET_ID}" <<'PY' +import json +import sys + +path, requested = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +items = data.get("items") or [] +if not items: + raise SystemExit("No local model assets are available. Mount a local .pt/.onnx/.engine file before running this smoke.") +for item in items: + if item.get("will_download_models") is not False: + raise SystemExit("Model asset catalog must never report automatic model downloads") +if requested: + matches = [item for item in items if item.get("model_asset_id") == requested] + if not matches: + raise SystemExit(f"Requested REAL_MODEL_ASSET_ID was not found: {requested}") + selected = matches[0] +else: + selected = next((item for item in items if item.get("active")), items[0]) +print(selected["model_asset_id"]) +PY +)" + +curl -fsS -G "${BASE_URL%/}/api/v1/detection/yolo/preflight" \ + --data-urlencode "tile_manifest_path=${manifest_path}" \ + --data-urlencode "model_asset_id=${model_asset_id}" > "${TMP_DIR}/preflight.json" +require_json_data "${TMP_DIR}/preflight.json" +"${PYTHON_BIN}" - "${TMP_DIR}/preflight.json" "${model_asset_id}" <<'PY' +import json +import sys + +path, expected_model_asset_id = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("model_asset_id") != expected_model_asset_id: + raise SystemExit("YOLO preflight did not use the selected model_asset_id") +if data.get("will_download_models") is not False: + raise SystemExit("YOLO preflight must never download model weights") +if data.get("will_run_inference") is not False: + raise SystemExit("YOLO preflight must remain read-only") +if data.get("status") != "ready": + raise SystemExit(f"YOLO preflight is not ready: {data.get('status')} {data.get('message')}") +checks = data.get("checks") or {} +if checks.get("manifest_valid") is not True: + raise SystemExit("YOLO preflight did not validate the raster tile manifest") +if checks.get("model_file_exists") is not True: + raise SystemExit("YOLO preflight did not confirm the local model file") +PY + +"${PYTHON_BIN}" - "${TMP_DIR}/run_request.json" "${project_id}" "${raster_dataset_id}" "${model_asset_id}" "${manifest_path}" "${REAL_CONFIDENCE_THRESHOLD}" <<'PY' +import json +import sys + +path, project_id, dataset_id, model_asset_id, tile_manifest_path, confidence = sys.argv[1:7] +payload = { + "project_id": project_id, + "dataset_id": dataset_id, + "model_id": "yolo-configured", + "model_asset_id": model_asset_id, + "confidence_threshold": float(confidence), + "class_filter": ["building"], + "tile_manifest_path": tile_manifest_path, + "parameters_json": { + "workflow": "real_data_detection_qa", + "operator_supplied_inputs": True, + }, +} +with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) +PY + +curl -fsS -X POST "${BASE_URL%/}/api/v1/detection/run" \ + -H "Content-Type: application/json" \ + --data-binary "@${TMP_DIR}/run_request.json" > "${TMP_DIR}/detection_run.json" +require_json_data "${TMP_DIR}/detection_run.json" +"${PYTHON_BIN}" - "${TMP_DIR}/detection_run.json" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("model_id") != "yolo-configured": + raise SystemExit("Detection run did not use yolo-configured") +if not data.get("analysis_run_id") or not data.get("job_id"): + raise SystemExit("Detection run did not return persisted run/job ids") +if data.get("status") != "success": + raise SystemExit(f"Detection run failed: {data.get('error_code')} {data.get('message')}") +if int(data.get("detection_count") or 0) < 0: + raise SystemExit("Detection count cannot be negative") +PY +analysis_run_id="$(json_field "${TMP_DIR}/detection_run.json" "data.analysis_run_id")" +detection_count="$(json_field "${TMP_DIR}/detection_run.json" "data.detection_count")" + +curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}/detections" > "${TMP_DIR}/detections.json" +require_json_data "${TMP_DIR}/detections.json" +"${PYTHON_BIN}" - "${TMP_DIR}/detections.json" "${detection_count}" <<'PY' +import json +import sys + +path, expected_count = sys.argv[1], int(sys.argv[2]) +with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +items = data.get("items") or [] +if int(data.get("total") or 0) != expected_count: + raise SystemExit("Detection list total does not match run detection_count") +if len(items) != expected_count: + raise SystemExit("Detection list item count does not match run detection_count") +for item in items: + if not item.get("source_tile_path"): + raise SystemExit("Persisted detection is missing source_tile_path provenance") +PY + +curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}/geojson" > "${TMP_DIR}/detection_geojson.json" +require_json_data "${TMP_DIR}/detection_geojson.json" +"${PYTHON_BIN}" - "${TMP_DIR}/detection_geojson.json" "${detection_count}" <<'PY' +import json +import sys + +path, expected_count = sys.argv[1], int(sys.argv[2]) +with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("type") != "FeatureCollection": + raise SystemExit("Detection GeoJSON response is not a FeatureCollection") +if len(data.get("features") or []) != expected_count: + raise SystemExit("Detection GeoJSON feature count does not match detection_count") +PY + +"${PYTHON_BIN}" - "${TMP_DIR}/qa_request.json" "${reference_dataset_id}" "${REAL_IOU_THRESHOLD}" "${REAL_CONFIDENCE_THRESHOLD}" <<'PY' +import json +import sys + +path, reference_dataset_id, iou_threshold, confidence = sys.argv[1:5] +payload = { + "reference_dataset_id": reference_dataset_id, + "iou_threshold": float(iou_threshold), + "class_name": "building", + "min_confidence": float(confidence), +} +with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) +PY + +curl -fsS -X POST "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}/qa/reference" \ + -H "Content-Type: application/json" \ + --data-binary "@${TMP_DIR}/qa_request.json" > "${TMP_DIR}/qa.json" +require_json_data "${TMP_DIR}/qa.json" +"${PYTHON_BIN}" - "${TMP_DIR}/qa.json" "${analysis_run_id}" "${reference_dataset_id}" <<'PY' +import json +import sys + +path, analysis_run_id, reference_dataset_id = sys.argv[1:4] +with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("analysis_run_id") != analysis_run_id: + raise SystemExit("Detection QA returned the wrong analysis_run_id") +if data.get("reference_dataset_id") != reference_dataset_id: + raise SystemExit("Detection QA returned the wrong reference_dataset_id") +if not data.get("quality_check_id"): + raise SystemExit("Detection QA did not persist a quality_check_id") +if int(data.get("reference_feature_count") or 0) < 1: + raise SystemExit("Detection QA reference feature count is empty") +for key in ("matches", "false_positives", "false_negatives"): + if int(data.get(key) or 0) < 0: + raise SystemExit(f"Detection QA returned a negative {key}") +PY +quality_check_id="$(json_field "${TMP_DIR}/qa.json" "data.quality_check_id")" + +"${PYTHON_BIN}" - "${TMP_DIR}/export_request.json" "${analysis_run_id}" <<'PY' +import json +import sys + +path, analysis_run_id = sys.argv[1], sys.argv[2] +payload = { + "export_kind": "detection_run", + "analysis_run_id": analysis_run_id, + "name": "real-data-detection-run", +} +with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) +PY + +curl -fsS -X POST "${BASE_URL%/}/api/v1/exports/geojson" \ + -H "Content-Type: application/json" \ + --data-binary "@${TMP_DIR}/export_request.json" > "${TMP_DIR}/export.json" +require_json_data "${TMP_DIR}/export.json" +export_id="$(json_field "${TMP_DIR}/export.json" "data.export_id")" +if [ -z "${export_id}" ] || [ "${export_id}" = "None" ] || [ "${export_id}" = "null" ]; then + echo "Detection GeoJSON export did not return an export_id" >&2 + exit 1 +fi + +curl -fsS "${BASE_URL%/}/api/v1/exports/${export_id}/content" > "${TMP_DIR}/export_content.json" +require_json_data "${TMP_DIR}/export_content.json" +"${PYTHON_BIN}" - "${TMP_DIR}/export_content.json" "${detection_count}" <<'PY' +import json +import sys + +path, expected_count = sys.argv[1], int(sys.argv[2]) +with open(path, "r", encoding="utf-8") as handle: + content = json.load(handle)["data"]["content"] +if content.get("type") != "FeatureCollection": + raise SystemExit("Detection export content is not a FeatureCollection") +if len(content.get("features") or []) != expected_count: + raise SystemExit("Detection export feature count does not match detection_count") +PY + +echo "Real data detection + QA workflow verification passed" +echo "Project: ${project_id}" +echo "Raster dataset: ${raster_dataset_id}" +echo "Reference dataset: ${reference_dataset_id}" +echo "Model asset: ${model_asset_id}" +echo "Manifest: ${manifest_path}" +echo "Analysis run: ${analysis_run_id}" +echo "Detections: ${detection_count}" +echo "Quality check: ${quality_check_id}" +echo "Detection export: ${export_id}"