Add workbench default state smoke
This commit is contained in:
@@ -7,6 +7,13 @@
|
|||||||
|
|
||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## Sprint 46 Workbench default-state smoke (2026-06-17)
|
||||||
|
|
||||||
|
- Added `scripts/verify_workbench_default_state.sh` to verify the live frontend/API default demo state through the browser-facing URL.
|
||||||
|
- The smoke seeds the offline demo workflow and verifies the demo project, AOI geometry, ready candidate/reference datasets and persisted QA/QC result via canonical envelopes.
|
||||||
|
- Added readiness coverage for the new smoke script syntax and static tests for its expected contract checks.
|
||||||
|
- No API contracts, migrations, provider fetching, AI behavior or product capabilities changed.
|
||||||
|
|
||||||
## Sprint 45 Default demo selection polish (2026-06-17)
|
## Sprint 45 Default demo selection polish (2026-06-17)
|
||||||
|
|
||||||
- Improved frontend project selection so a cold start prefers a populated demo/workbench project over an empty first project.
|
- Improved frontend project selection so a cold start prefers a populated demo/workbench project over an empty first project.
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ def test_readiness_gate_checks_demo_export_workflow_script_syntax() -> None:
|
|||||||
content = script.read_text(encoding="utf-8")
|
content = script.read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert "bash -n scripts/verify_demo_export_workflow.sh" in content
|
assert "bash -n scripts/verify_demo_export_workflow.sh" in content
|
||||||
|
assert "bash -n scripts/verify_workbench_default_state.sh" in content
|
||||||
|
|
||||||
|
|
||||||
def test_readiness_gate_runs_golden_qa_benchmark() -> None:
|
def test_readiness_gate_runs_golden_qa_benchmark() -> None:
|
||||||
@@ -67,6 +68,20 @@ def test_demo_export_workflow_script_verifies_export_endpoints() -> None:
|
|||||||
assert "/download" in content
|
assert "/download" in content
|
||||||
|
|
||||||
|
|
||||||
|
def test_workbench_default_state_script_verifies_populated_demo_start_state() -> None:
|
||||||
|
script = Path(__file__).resolve().parents[2] / "scripts" / "verify_workbench_default_state.sh"
|
||||||
|
content = script.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "/api/v1/demo/workflow" in content
|
||||||
|
assert "GeoIntel Demo - Building QA" in content
|
||||||
|
assert "/areas" in content
|
||||||
|
assert "/datasets" in content
|
||||||
|
assert "/quality-checks" in content
|
||||||
|
assert "data.items" in content
|
||||||
|
assert "Demo AOI - Geel buildings" in content
|
||||||
|
assert "2/2 ready" in content
|
||||||
|
|
||||||
|
|
||||||
def test_pass_end_check_excludes_vendor_and_build_outputs() -> None:
|
def test_pass_end_check_excludes_vendor_and_build_outputs() -> None:
|
||||||
script = Path(__file__).resolve().parents[2] / "scripts" / "codex_pass_end_check.sh"
|
script = Path(__file__).resolve().parents[2] / "scripts" / "codex_pass_end_check.sh"
|
||||||
content = script.read_text(encoding="utf-8")
|
content = script.read_text(encoding="utf-8")
|
||||||
|
|||||||
@@ -1,3 +1,33 @@
|
|||||||
|
## Sprint 46 Workbench default-state smoke (2026-06-17)
|
||||||
|
|
||||||
|
Changed:
|
||||||
|
- Added `scripts/verify_workbench_default_state.sh`, a dependency-light runtime smoke for the browser-facing workbench default demo state.
|
||||||
|
- The smoke calls the offline demo workflow, then verifies `GeoIntel Demo - Building QA`, `Demo AOI - Geel buildings`, `2/2 ready` datasets and a persisted QA/QC result through canonical `data.items` envelopes.
|
||||||
|
- Added the script syntax check to `scripts/run_readiness_check.sh`.
|
||||||
|
- Extended `backend/tests/test_readiness_gate.py` so the readiness gate and script keep covering the default-state smoke contract.
|
||||||
|
- Updated `scripts/README.md` and `CHANGELOG.md`.
|
||||||
|
|
||||||
|
Validation:
|
||||||
|
- RED: `cd backend && python -m pytest tests/test_readiness_gate.py -q` failed before implementation because `verify_workbench_default_state.sh` was missing and readiness did not reference it.
|
||||||
|
- `cd backend && python -m pytest tests/test_readiness_gate.py -q` passed: 9 tests.
|
||||||
|
- `bash -n scripts/verify_workbench_default_state.sh` passed.
|
||||||
|
- `bash scripts/verify_workbench_default_state.sh http://192.168.10.150:1202` passed.
|
||||||
|
- `python -m compileall backend/app` passed.
|
||||||
|
- `cd backend && python -m pytest -W error::DeprecationWarning` passed: 195 tests.
|
||||||
|
- `bash scripts/run_readiness_check.sh` passed.
|
||||||
|
- `cd frontend && npm run typecheck` passed.
|
||||||
|
- `cd frontend && npm run build` passed.
|
||||||
|
- `cd backend && python -m alembic heads` passed: `202606120900 (head)`.
|
||||||
|
- `cd backend && python -m alembic upgrade head --sql` passed.
|
||||||
|
- `bash -n scripts/live_migration_smoke.sh` passed.
|
||||||
|
|
||||||
|
Limitations:
|
||||||
|
- The committed smoke validates the connected default workbench state through HTTP/API contracts, not pixel layout. Browser screenshot and overflow validation remain a Codex/browser verification step.
|
||||||
|
- No API contracts, migrations, provider fetching, AI behavior or product capabilities changed.
|
||||||
|
|
||||||
|
Next recommended pass:
|
||||||
|
- Commit, deploy to Tower and run the live browser screenshot/overflow pass against the redeployed runtime.
|
||||||
|
|
||||||
## Sprint 45 Default demo selection polish (2026-06-17)
|
## Sprint 45 Default demo selection polish (2026-06-17)
|
||||||
|
|
||||||
Changed:
|
Changed:
|
||||||
|
|||||||
+2
-1
@@ -41,6 +41,7 @@ This file now starts with the current implementation status. Older preparation/b
|
|||||||
- [x] Browser-facing demo/export workflow smoke script with connected V1 state checks.
|
- [x] Browser-facing demo/export workflow smoke script with connected V1 state checks.
|
||||||
- [x] Compact V1 workbench status strip for project, AOI, datasets, map, QA/QC and exports.
|
- [x] Compact V1 workbench status strip for project, AOI, datasets, map, QA/QC and exports.
|
||||||
- [x] Dry-run-first demo export artifact cleanup tooling.
|
- [x] Dry-run-first demo export artifact cleanup tooling.
|
||||||
|
- [x] Browser-facing default workbench state smoke for the offline demo project.
|
||||||
- [x] Live Docker/PostGIS validation on Tower/Unraid.
|
- [x] Live Docker/PostGIS validation on Tower/Unraid.
|
||||||
- [x] Real YOLO compatibility smoke with optional AI extras and local model file.
|
- [x] Real YOLO compatibility smoke with optional AI extras and local model file.
|
||||||
- [x] Detection and segmentation workflow hook extraction beyond Sprint 10.
|
- [x] Detection and segmentation workflow hook extraction beyond Sprint 10.
|
||||||
@@ -56,7 +57,7 @@ This file now starts with the current implementation status. Older preparation/b
|
|||||||
- [x] Demo workflow orchestration hook decomposition.
|
- [x] Demo workflow orchestration hook decomposition.
|
||||||
- [x] Final `App.tsx` import/encoding cleanup and size audit.
|
- [x] Final `App.tsx` import/encoding cleanup and size audit.
|
||||||
- [x] Optional final bootstrap-effect extraction.
|
- [x] Optional final bootstrap-effect extraction.
|
||||||
- [ ] Decide next V1 stabilization focus: UI browser regression coverage, backend service contract audit, or golden dataset expansion.
|
- [ ] Decide next V1 stabilization focus: browser screenshot regression automation, backend service contract audit, or golden dataset expansion.
|
||||||
|
|
||||||
## Sprint 8 status
|
## Sprint 8 status
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,19 @@ persisted QA/QC result is compared against `fixtures/golden/expected_qa_metrics.
|
|||||||
so runtime demo precision, recall, F1, mean IoU and false-positive/negative
|
so runtime demo precision, recall, F1, mean IoU and false-positive/negative
|
||||||
counts cannot drift silently.
|
counts cannot drift silently.
|
||||||
|
|
||||||
|
Verify that the browser-facing workbench can populate the default demo start
|
||||||
|
state through the frontend proxy:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash scripts/verify_workbench_default_state.sh http://192.168.10.150:1202
|
||||||
|
```
|
||||||
|
|
||||||
|
This smoke is dependency-light and intentionally idempotent: it seeds the
|
||||||
|
offline demo workflow, then verifies that `GeoIntel Demo - Building QA` exposes
|
||||||
|
the `Demo AOI - Geel buildings` map geometry, `2/2 ready` demo datasets and a
|
||||||
|
persisted QA/QC result through canonical `data.items` envelopes. Pair it with a
|
||||||
|
Codex/browser screenshot pass when checking visual layout or overflow.
|
||||||
|
|
||||||
Verify the deterministic QA/QC golden benchmark:
|
Verify the deterministic QA/QC golden benchmark:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ ${PYTHON_BIN} -m compileall backend/app
|
|||||||
bash -n scripts/live_migration_smoke.sh
|
bash -n scripts/live_migration_smoke.sh
|
||||||
bash -n scripts/verify_browser_runtime.sh
|
bash -n scripts/verify_browser_runtime.sh
|
||||||
bash -n scripts/verify_demo_export_workflow.sh
|
bash -n scripts/verify_demo_export_workflow.sh
|
||||||
|
bash -n scripts/verify_workbench_default_state.sh
|
||||||
bash -n scripts/verify_gis_runtime.sh
|
bash -n scripts/verify_gis_runtime.sh
|
||||||
bash -n scripts/verify_golden_qa_benchmark.sh
|
bash -n scripts/verify_golden_qa_benchmark.sh
|
||||||
echo "== Run readiness check passed =="
|
echo "== Run readiness check passed =="
|
||||||
|
|||||||
@@ -0,0 +1,171 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
|
||||||
|
TMP_DIR="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "${TMP_DIR}"' EXIT
|
||||||
|
|
||||||
|
if ! command -v curl >/dev/null 2>&1; then
|
||||||
|
echo "curl is required for workbench default-state verification" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "${PYTHON_BIN:-}" ]; then
|
||||||
|
PYTHON_BIN="${PYTHON_BIN}"
|
||||||
|
else
|
||||||
|
PYTHON_BIN=""
|
||||||
|
for candidate in python3 python.exe python; do
|
||||||
|
if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then
|
||||||
|
PYTHON_BIN="${candidate}"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "${PYTHON_BIN}" ]; then
|
||||||
|
echo "A Python interpreter is required for JSON parsing" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
require_json_data() {
|
||||||
|
local file_path="$1"
|
||||||
|
"${PYTHON_BIN}" - "$file_path" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
with open(sys.argv[1], "r", encoding="utf-8") as handle:
|
||||||
|
payload = json.load(handle)
|
||||||
|
if "data" not in payload:
|
||||||
|
raise SystemExit("Response is not a canonical GeoIntel data envelope")
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
json_field() {
|
||||||
|
local file_path="$1"
|
||||||
|
local expression="$2"
|
||||||
|
"${PYTHON_BIN}" - "$file_path" "$expression" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
path, expression = sys.argv[1], sys.argv[2]
|
||||||
|
with open(path, "r", encoding="utf-8") as handle:
|
||||||
|
payload = json.load(handle)
|
||||||
|
value = payload
|
||||||
|
for part in expression.split("."):
|
||||||
|
if part:
|
||||||
|
value = value[part]
|
||||||
|
print(value)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "== GeoIntel workbench default-state verification =="
|
||||||
|
echo "Base URL: ${BASE_URL}"
|
||||||
|
|
||||||
|
frontend_status="$(curl -fsS -o "${TMP_DIR}/index.html" -w "%{http_code}" "${BASE_URL%/}" 2>/dev/null || true)"
|
||||||
|
if [ "${frontend_status}" != "200" ]; then
|
||||||
|
echo "Frontend returned HTTP ${frontend_status:-none}, expected 200" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! grep -q '<div id="root"' "${TMP_DIR}/index.html"; then
|
||||||
|
echo "Frontend shell does not expose the React root node" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
curl -fsS -X POST "${BASE_URL%/}/api/v1/demo/workflow" > "${TMP_DIR}/demo.json"
|
||||||
|
require_json_data "${TMP_DIR}/demo.json"
|
||||||
|
|
||||||
|
project_id="$(json_field "${TMP_DIR}/demo.json" "data.project_id")"
|
||||||
|
area_id="$(json_field "${TMP_DIR}/demo.json" "data.area_id")"
|
||||||
|
candidate_dataset_id="$(json_field "${TMP_DIR}/demo.json" "data.candidate_dataset_id")"
|
||||||
|
reference_dataset_id="$(json_field "${TMP_DIR}/demo.json" "data.reference_dataset_id")"
|
||||||
|
quality_check_id="$(json_field "${TMP_DIR}/demo.json" "data.quality_check_id")"
|
||||||
|
|
||||||
|
curl -fsS "${BASE_URL%/}/api/v1/projects" > "${TMP_DIR}/projects.json"
|
||||||
|
require_json_data "${TMP_DIR}/projects.json"
|
||||||
|
# The workbench lists below must all use canonical data.items envelopes.
|
||||||
|
"${PYTHON_BIN}" - "${TMP_DIR}/projects.json" "${project_id}" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
path, project_id = sys.argv[1], sys.argv[2]
|
||||||
|
with open(path, "r", encoding="utf-8") as handle:
|
||||||
|
payload = json.load(handle)
|
||||||
|
items = payload["data"]["items"]
|
||||||
|
matches = [item for item in items if item["id"] == project_id]
|
||||||
|
if not matches:
|
||||||
|
raise SystemExit("Seeded demo project is missing from project list")
|
||||||
|
project = matches[0]
|
||||||
|
if project.get("name") != "GeoIntel Demo - Building QA":
|
||||||
|
raise SystemExit(f"Unexpected default demo project name: {project.get('name')}")
|
||||||
|
PY
|
||||||
|
|
||||||
|
curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/areas" > "${TMP_DIR}/areas.json"
|
||||||
|
require_json_data "${TMP_DIR}/areas.json"
|
||||||
|
"${PYTHON_BIN}" - "${TMP_DIR}/areas.json" "${area_id}" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
path, area_id = sys.argv[1], sys.argv[2]
|
||||||
|
with open(path, "r", encoding="utf-8") as handle:
|
||||||
|
payload = json.load(handle)
|
||||||
|
items = payload["data"]["items"]
|
||||||
|
matches = [item for item in items if item["id"] == area_id]
|
||||||
|
if not matches:
|
||||||
|
raise SystemExit("Seeded demo AOI is missing from project area list")
|
||||||
|
area = matches[0]
|
||||||
|
if area.get("name") != "Demo AOI - Geel buildings":
|
||||||
|
raise SystemExit(f"Unexpected demo AOI name: {area.get('name')}")
|
||||||
|
geometry = area.get("geometry")
|
||||||
|
if not geometry or geometry.get("type") not in {"Polygon", "MultiPolygon"}:
|
||||||
|
raise SystemExit("Demo AOI does not expose map-ready GeoJSON geometry")
|
||||||
|
PY
|
||||||
|
|
||||||
|
curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/datasets" > "${TMP_DIR}/datasets.json"
|
||||||
|
require_json_data "${TMP_DIR}/datasets.json"
|
||||||
|
"${PYTHON_BIN}" - "${TMP_DIR}/datasets.json" "${candidate_dataset_id}" "${reference_dataset_id}" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
path, candidate_id, reference_id = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||||
|
with open(path, "r", encoding="utf-8") as handle:
|
||||||
|
payload = json.load(handle)
|
||||||
|
items = payload["data"]["items"]
|
||||||
|
ids = {item["id"]: item for item in items}
|
||||||
|
if candidate_id not in ids:
|
||||||
|
raise SystemExit("Candidate fixture dataset is missing from default workbench dataset list")
|
||||||
|
if reference_id not in ids:
|
||||||
|
raise SystemExit("Reference fixture dataset is missing from default workbench dataset list")
|
||||||
|
if ids[candidate_id].get("status") != "ready" or ids[reference_id].get("status") != "ready":
|
||||||
|
raise SystemExit("Expected 2/2 ready demo datasets")
|
||||||
|
if ids[reference_id].get("dataset_role") != "reference":
|
||||||
|
raise SystemExit("Default workbench reference dataset is not marked as reference")
|
||||||
|
PY
|
||||||
|
|
||||||
|
curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks" > "${TMP_DIR}/quality_checks.json"
|
||||||
|
require_json_data "${TMP_DIR}/quality_checks.json"
|
||||||
|
"${PYTHON_BIN}" - "${TMP_DIR}/quality_checks.json" "${quality_check_id}" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
path, quality_check_id = sys.argv[1], sys.argv[2]
|
||||||
|
with open(path, "r", encoding="utf-8") as handle:
|
||||||
|
payload = json.load(handle)
|
||||||
|
items = payload["data"]["items"]
|
||||||
|
matches = [item for item in items if item["id"] == quality_check_id]
|
||||||
|
if not matches:
|
||||||
|
raise SystemExit("Seeded QA/QC check is missing from default workbench quality list")
|
||||||
|
quality_check = matches[0]
|
||||||
|
if quality_check.get("status") != "ok":
|
||||||
|
raise SystemExit(f"Unexpected default workbench QA/QC status: {quality_check.get('status')}")
|
||||||
|
metrics = {metric["metric_key"]: metric.get("metric_value") for metric in quality_check.get("metrics", [])}
|
||||||
|
for key in ("precision", "recall", "f1", "mean_iou"):
|
||||||
|
if key not in metrics:
|
||||||
|
raise SystemExit(f"Default workbench QA/QC metric is missing: {key}")
|
||||||
|
PY
|
||||||
|
|
||||||
|
echo "Workbench default state verification passed"
|
||||||
|
echo "Project: GeoIntel Demo - Building QA"
|
||||||
|
echo "Area: Demo AOI - Geel buildings"
|
||||||
|
echo "Datasets: 2/2 ready"
|
||||||
|
echo "QA/QC: seeded check is available"
|
||||||
Reference in New Issue
Block a user