From 7c4de36c58c03c2f74b8792361b17e60ef40f014 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 17 Jun 2026 18:44:51 +0200 Subject: [PATCH] Add workbench default state smoke --- CHANGELOG.md | 7 + backend/tests/test_readiness_gate.py | 15 ++ docs/CODEX_EXECUTION_LOG.md | 30 ++++ docs/TODO.md | 3 +- scripts/README.md | 13 ++ scripts/run_readiness_check.sh | 1 + scripts/verify_workbench_default_state.sh | 171 ++++++++++++++++++++++ 7 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 scripts/verify_workbench_default_state.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 6413a343..1f539bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 46 Workbench default-state smoke (2026-06-17) + +- Added `scripts/verify_workbench_default_state.sh` to verify the live frontend/API default demo state through the browser-facing URL. +- The smoke seeds the offline demo workflow and verifies the demo project, AOI geometry, ready candidate/reference datasets and persisted QA/QC result via canonical envelopes. +- Added readiness coverage for the new smoke script syntax and static tests for its expected contract checks. +- No API contracts, migrations, provider fetching, AI behavior or product capabilities changed. + ## Sprint 45 Default demo selection polish (2026-06-17) - Improved frontend project selection so a cold start prefers a populated demo/workbench project over an empty first project. diff --git a/backend/tests/test_readiness_gate.py b/backend/tests/test_readiness_gate.py index cd718007..f5361169 100644 --- a/backend/tests/test_readiness_gate.py +++ b/backend/tests/test_readiness_gate.py @@ -20,6 +20,7 @@ def test_readiness_gate_checks_demo_export_workflow_script_syntax() -> None: content = script.read_text(encoding="utf-8") assert "bash -n scripts/verify_demo_export_workflow.sh" in content + assert "bash -n scripts/verify_workbench_default_state.sh" in content def test_readiness_gate_runs_golden_qa_benchmark() -> None: @@ -67,6 +68,20 @@ def test_demo_export_workflow_script_verifies_export_endpoints() -> None: assert "/download" in content +def test_workbench_default_state_script_verifies_populated_demo_start_state() -> None: + script = Path(__file__).resolve().parents[2] / "scripts" / "verify_workbench_default_state.sh" + content = script.read_text(encoding="utf-8") + + assert "/api/v1/demo/workflow" in content + assert "GeoIntel Demo - Building QA" in content + assert "/areas" in content + assert "/datasets" in content + assert "/quality-checks" in content + assert "data.items" in content + assert "Demo AOI - Geel buildings" in content + assert "2/2 ready" in content + + def test_pass_end_check_excludes_vendor_and_build_outputs() -> None: script = Path(__file__).resolve().parents[2] / "scripts" / "codex_pass_end_check.sh" content = script.read_text(encoding="utf-8") diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 9a6f0aae..1bb3c7bd 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,33 @@ +## Sprint 46 Workbench default-state smoke (2026-06-17) + +Changed: +- Added `scripts/verify_workbench_default_state.sh`, a dependency-light runtime smoke for the browser-facing workbench default demo state. +- The smoke calls the offline demo workflow, then verifies `GeoIntel Demo - Building QA`, `Demo AOI - Geel buildings`, `2/2 ready` datasets and a persisted QA/QC result through canonical `data.items` envelopes. +- Added the script syntax check to `scripts/run_readiness_check.sh`. +- Extended `backend/tests/test_readiness_gate.py` so the readiness gate and script keep covering the default-state smoke contract. +- Updated `scripts/README.md` and `CHANGELOG.md`. + +Validation: +- RED: `cd backend && python -m pytest tests/test_readiness_gate.py -q` failed before implementation because `verify_workbench_default_state.sh` was missing and readiness did not reference it. +- `cd backend && python -m pytest tests/test_readiness_gate.py -q` passed: 9 tests. +- `bash -n scripts/verify_workbench_default_state.sh` passed. +- `bash scripts/verify_workbench_default_state.sh http://192.168.10.150:1202` passed. +- `python -m compileall backend/app` passed. +- `cd backend && python -m pytest -W error::DeprecationWarning` passed: 195 tests. +- `bash scripts/run_readiness_check.sh` passed. +- `cd frontend && npm run typecheck` passed. +- `cd frontend && npm run build` passed. +- `cd backend && python -m alembic heads` passed: `202606120900 (head)`. +- `cd backend && python -m alembic upgrade head --sql` passed. +- `bash -n scripts/live_migration_smoke.sh` passed. + +Limitations: +- The committed smoke validates the connected default workbench state through HTTP/API contracts, not pixel layout. Browser screenshot and overflow validation remain a Codex/browser verification step. +- No API contracts, migrations, provider fetching, AI behavior or product capabilities changed. + +Next recommended pass: +- Commit, deploy to Tower and run the live browser screenshot/overflow pass against the redeployed runtime. + ## Sprint 45 Default demo selection polish (2026-06-17) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index e6326658..29dae886 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -41,6 +41,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Browser-facing demo/export workflow smoke script with connected V1 state checks. - [x] Compact V1 workbench status strip for project, AOI, datasets, map, QA/QC and exports. - [x] Dry-run-first demo export artifact cleanup tooling. +- [x] Browser-facing default workbench state smoke for the offline demo project. - [x] Live Docker/PostGIS validation on Tower/Unraid. - [x] Real YOLO compatibility smoke with optional AI extras and local model file. - [x] Detection and segmentation workflow hook extraction beyond Sprint 10. @@ -56,7 +57,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Demo workflow orchestration hook decomposition. - [x] Final `App.tsx` import/encoding cleanup and size audit. - [x] Optional final bootstrap-effect extraction. -- [ ] Decide next V1 stabilization focus: UI browser regression coverage, backend service contract audit, or golden dataset expansion. +- [ ] Decide next V1 stabilization focus: browser screenshot regression automation, backend service contract audit, or golden dataset expansion. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index 47acc3ae..ef60b7a3 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -26,6 +26,19 @@ persisted QA/QC result is compared against `fixtures/golden/expected_qa_metrics. so runtime demo precision, recall, F1, mean IoU and false-positive/negative counts cannot drift silently. +Verify that the browser-facing workbench can populate the default demo start +state through the frontend proxy: + +```bash +bash scripts/verify_workbench_default_state.sh http://192.168.10.150:1202 +``` + +This smoke is dependency-light and intentionally idempotent: it seeds the +offline demo workflow, then verifies that `GeoIntel Demo - Building QA` exposes +the `Demo AOI - Geel buildings` map geometry, `2/2 ready` demo datasets and a +persisted QA/QC result through canonical `data.items` envelopes. Pair it with a +Codex/browser screenshot pass when checking visual layout or overflow. + Verify the deterministic QA/QC golden benchmark: ```bash diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 81d5644d..356bb09f 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -50,6 +50,7 @@ ${PYTHON_BIN} -m compileall backend/app bash -n scripts/live_migration_smoke.sh bash -n scripts/verify_browser_runtime.sh bash -n scripts/verify_demo_export_workflow.sh +bash -n scripts/verify_workbench_default_state.sh bash -n scripts/verify_gis_runtime.sh bash -n scripts/verify_golden_qa_benchmark.sh echo "== Run readiness check passed ==" diff --git a/scripts/verify_workbench_default_state.sh b/scripts/verify_workbench_default_state.sh new file mode 100644 index 00000000..826c3228 --- /dev/null +++ b/scripts/verify_workbench_default_state.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +if ! command -v curl >/dev/null 2>&1; then + echo "curl is required for workbench default-state verification" >&2 + exit 1 +fi + +if [ -n "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="${PYTHON_BIN}" +else + PYTHON_BIN="" + for candidate in python3 python.exe python; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then + PYTHON_BIN="${candidate}" + break + fi + done +fi + +if [ -z "${PYTHON_BIN}" ]; then + echo "A Python interpreter is required for JSON parsing" >&2 + exit 1 +fi + +require_json_data() { + local file_path="$1" + "${PYTHON_BIN}" - "$file_path" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as handle: + payload = json.load(handle) +if "data" not in payload: + raise SystemExit("Response is not a canonical GeoIntel data envelope") +PY +} + +json_field() { + local file_path="$1" + local expression="$2" + "${PYTHON_BIN}" - "$file_path" "$expression" <<'PY' +import json +import sys + +path, expression = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + payload = json.load(handle) +value = payload +for part in expression.split("."): + if part: + value = value[part] +print(value) +PY +} + +echo "== GeoIntel workbench default-state verification ==" +echo "Base URL: ${BASE_URL}" + +frontend_status="$(curl -fsS -o "${TMP_DIR}/index.html" -w "%{http_code}" "${BASE_URL%/}" 2>/dev/null || true)" +if [ "${frontend_status}" != "200" ]; then + echo "Frontend returned HTTP ${frontend_status:-none}, expected 200" >&2 + exit 1 +fi + +if ! grep -q '
&2 + exit 1 +fi + +curl -fsS -X POST "${BASE_URL%/}/api/v1/demo/workflow" > "${TMP_DIR}/demo.json" +require_json_data "${TMP_DIR}/demo.json" + +project_id="$(json_field "${TMP_DIR}/demo.json" "data.project_id")" +area_id="$(json_field "${TMP_DIR}/demo.json" "data.area_id")" +candidate_dataset_id="$(json_field "${TMP_DIR}/demo.json" "data.candidate_dataset_id")" +reference_dataset_id="$(json_field "${TMP_DIR}/demo.json" "data.reference_dataset_id")" +quality_check_id="$(json_field "${TMP_DIR}/demo.json" "data.quality_check_id")" + +curl -fsS "${BASE_URL%/}/api/v1/projects" > "${TMP_DIR}/projects.json" +require_json_data "${TMP_DIR}/projects.json" +# The workbench lists below must all use canonical data.items envelopes. +"${PYTHON_BIN}" - "${TMP_DIR}/projects.json" "${project_id}" <<'PY' +import json +import sys + +path, project_id = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + payload = json.load(handle) +items = payload["data"]["items"] +matches = [item for item in items if item["id"] == project_id] +if not matches: + raise SystemExit("Seeded demo project is missing from project list") +project = matches[0] +if project.get("name") != "GeoIntel Demo - Building QA": + raise SystemExit(f"Unexpected default demo project name: {project.get('name')}") +PY + +curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/areas" > "${TMP_DIR}/areas.json" +require_json_data "${TMP_DIR}/areas.json" +"${PYTHON_BIN}" - "${TMP_DIR}/areas.json" "${area_id}" <<'PY' +import json +import sys + +path, area_id = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + payload = json.load(handle) +items = payload["data"]["items"] +matches = [item for item in items if item["id"] == area_id] +if not matches: + raise SystemExit("Seeded demo AOI is missing from project area list") +area = matches[0] +if area.get("name") != "Demo AOI - Geel buildings": + raise SystemExit(f"Unexpected demo AOI name: {area.get('name')}") +geometry = area.get("geometry") +if not geometry or geometry.get("type") not in {"Polygon", "MultiPolygon"}: + raise SystemExit("Demo AOI does not expose map-ready GeoJSON geometry") +PY + +curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/datasets" > "${TMP_DIR}/datasets.json" +require_json_data "${TMP_DIR}/datasets.json" +"${PYTHON_BIN}" - "${TMP_DIR}/datasets.json" "${candidate_dataset_id}" "${reference_dataset_id}" <<'PY' +import json +import sys + +path, candidate_id, reference_id = sys.argv[1], sys.argv[2], sys.argv[3] +with open(path, "r", encoding="utf-8") as handle: + payload = json.load(handle) +items = payload["data"]["items"] +ids = {item["id"]: item for item in items} +if candidate_id not in ids: + raise SystemExit("Candidate fixture dataset is missing from default workbench dataset list") +if reference_id not in ids: + raise SystemExit("Reference fixture dataset is missing from default workbench dataset list") +if ids[candidate_id].get("status") != "ready" or ids[reference_id].get("status") != "ready": + raise SystemExit("Expected 2/2 ready demo datasets") +if ids[reference_id].get("dataset_role") != "reference": + raise SystemExit("Default workbench reference dataset is not marked as reference") +PY + +curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks" > "${TMP_DIR}/quality_checks.json" +require_json_data "${TMP_DIR}/quality_checks.json" +"${PYTHON_BIN}" - "${TMP_DIR}/quality_checks.json" "${quality_check_id}" <<'PY' +import json +import sys + +path, quality_check_id = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + payload = json.load(handle) +items = payload["data"]["items"] +matches = [item for item in items if item["id"] == quality_check_id] +if not matches: + raise SystemExit("Seeded QA/QC check is missing from default workbench quality list") +quality_check = matches[0] +if quality_check.get("status") != "ok": + raise SystemExit(f"Unexpected default workbench QA/QC status: {quality_check.get('status')}") +metrics = {metric["metric_key"]: metric.get("metric_value") for metric in quality_check.get("metrics", [])} +for key in ("precision", "recall", "f1", "mean_iou"): + if key not in metrics: + raise SystemExit(f"Default workbench QA/QC metric is missing: {key}") +PY + +echo "Workbench default state verification passed" +echo "Project: GeoIntel Demo - Building QA" +echo "Area: Demo AOI - Geel buildings" +echo "Datasets: 2/2 ready" +echo "QA/QC: seeded check is available"