diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ac493cf..ee2b822c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ # Changelog +## Sprint 97 demo raster fixture workflow (2026-06-23) + +- Added a deterministic local GeoTIFF raster fixture to the offline demo workflow so raster controls and AI Lab dataset prerequisites have usable V1 context. +- Returned `raster_dataset_id` from the canonical demo workflow response and wired the frontend demo loader to select it for Detection and Segmentation Labs. +- Kept the candidate vector dataset as the default Data/Map/Export context after demo load. +- Hardened workbench default/interactions smoke scripts to require the candidate vector, reference vector and raster fixture datasets as `3/3 ready`. +- No external provider fetching, real AI inference, migrations or API behavior outside the demo response contract changed. + ## Sprint 96 useful default context (2026-06-22) - Auto-open the first ready vector dataset after project data loads so Data, Map and Exports start with usable context. diff --git a/backend/app/schemas/demo.py b/backend/app/schemas/demo.py index 494ef38d..444ed209 100644 --- a/backend/app/schemas/demo.py +++ b/backend/app/schemas/demo.py @@ -10,6 +10,7 @@ class DemoWorkflowResponse(BaseModel): area_id: UUID reference_dataset_id: UUID candidate_dataset_id: UUID + raster_dataset_id: UUID | None = None quality_check_id: UUID metric_count: int status: str diff --git a/backend/app/services/demo_workflow_service.py b/backend/app/services/demo_workflow_service.py index b6a86241..0af85da1 100644 --- a/backend/app/services/demo_workflow_service.py +++ b/backend/app/services/demo_workflow_service.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import os +import importlib from datetime import datetime, timezone from pathlib import Path from uuid import UUID, uuid4 @@ -14,6 +15,7 @@ from app.schemas.demo import DemoWorkflowResponse from app.services.geojson_service import parse_geojson_payload from app.services.qa_service import QaService from app.services.quality_service import QualityService +from app.services.raster_service import extract_raster_metadata from app.services.storage_service import StorageService from app.services.vector_feature_service import VectorFeatureService from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon @@ -24,6 +26,7 @@ class DemoWorkflowService: AREA_NAME = "Demo AOI - Geel buildings" REFERENCE_FILENAME = "demo_reference_buildings.geojson" CANDIDATE_FILENAME = "demo_predicted_buildings.geojson" + RASTER_FILENAME = "demo_context_raster.tif" EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json" @staticmethod @@ -101,15 +104,28 @@ class DemoWorkflowService: .filter(Dataset.project_id == project_id) .filter(Dataset.dataset_role == "source") .filter(Dataset.source_name == "fixture") + .filter(Dataset.dataset_type == "vector") .first() ) + raster = DemoWorkflowService._find_demo_raster_dataset(db, project_id) quality_check = ( db.query(QualityCheck) .filter(QualityCheck.project_id == project_id) .filter(QualityCheck.check_type == "demo_candidate_vs_reference") .first() ) - return bool(area and reference and candidate and quality_check) + return bool(area and reference and candidate and raster and quality_check) + + @staticmethod + def _find_demo_raster_dataset(db: Session, project_id: UUID) -> Dataset | None: + return ( + db.query(Dataset) + .filter(Dataset.project_id == project_id) + .filter(Dataset.dataset_type == "raster") + .filter(Dataset.source_name == "fixture") + .filter(Dataset.name == DemoWorkflowService.RASTER_FILENAME) + .first() + ) @staticmethod def _create_area(db: Session, project_id: UUID) -> Area: @@ -207,6 +223,84 @@ class DemoWorkflowService: ) return dataset + @staticmethod + def _create_demo_raster_bytes() -> bytes: + numpy = importlib.import_module("numpy") + rasterio = importlib.import_module("rasterio") + rasterio_io = importlib.import_module("rasterio.io") + rasterio_transform = importlib.import_module("rasterio.transform") + + width = 64 + height = 48 + data = numpy.linspace(20, 220, num=width * height, dtype=numpy.uint8).reshape((height, width)) + transform = rasterio_transform.from_bounds(4.9895, 51.1595, 4.9930, 51.1615, width, height) + with rasterio_io.MemoryFile() as memfile: + with memfile.open( + driver="GTiff", + width=width, + height=height, + count=1, + dtype="uint8", + crs="EPSG:4326", + transform=transform, + nodata=0, + ) as dataset: + dataset.write(data, 1) + return memfile.read() + + @staticmethod + def _create_raster_dataset(db: Session, *, project_id: UUID, area_id: UUID) -> Dataset: + dataset_id = uuid4() + raw = DemoWorkflowService._create_demo_raster_bytes() + storage_info = StorageService.persist_dataset_file( + project_id=str(project_id), + dataset_id=str(dataset_id), + dataset_type="raster", + original_filename=DemoWorkflowService.RASTER_FILENAME, + content=raw, + content_type="image/tiff", + ) + metadata = extract_raster_metadata(storage_info["storage_path"]) + bounds = metadata.get("bounds") + bounds_json = None + if isinstance(bounds, list) and len(bounds) == 4: + bounds_json = {"minx": bounds[0], "miny": bounds[1], "maxx": bounds[2], "maxy": bounds[3]} + dataset = Dataset( + id=dataset_id, + project_id=project_id, + area_id=area_id, + name=DemoWorkflowService.RASTER_FILENAME, + dataset_type="raster", + source="fixture", + dataset_role="source", + source_name="fixture", + reference_layer_name=None, + source_metadata={ + "fixture": True, + "fixture_name": DemoWorkflowService.RASTER_FILENAME, + "usage": "offline demo raster workflow only", + }, + provenance_metadata={ + "created_by": "demo_workflow", + "source_path": "generated:demo_context_raster", + }, + imported_at=datetime.now(timezone.utc), + storage_path=storage_info["storage_path"], + original_filename=storage_info["original_filename"], + stored_filename=storage_info["stored_filename"], + content_type=storage_info["content_type"], + size_bytes=storage_info["size_bytes"], + checksum_sha256=storage_info["checksum_sha256"], + crs=metadata.get("crs"), + bounds_json=bounds_json, + metadata_json=metadata, + status="ready", + ) + db.add(dataset) + db.commit() + db.refresh(dataset) + return dataset + @staticmethod def _persist_qa( db: Session, @@ -306,8 +400,10 @@ class DemoWorkflowService: .filter(Dataset.project_id == existing.id) .filter(Dataset.dataset_role == "source") .filter(Dataset.source_name == "fixture") + .filter(Dataset.dataset_type == "vector") .first() ) + raster = DemoWorkflowService._find_demo_raster_dataset(db, existing.id) quality_check = ( db.query(QualityCheck) .filter(QualityCheck.project_id == existing.id) @@ -316,6 +412,8 @@ class DemoWorkflowService: .first() ) if area and reference and candidate and quality_check: + if not raster: + raster = DemoWorkflowService._create_raster_dataset(db=db, project_id=existing.id, area_id=area.id) if not DemoWorkflowService._quality_check_matches_expected(db, quality_check): area = DemoWorkflowService._sync_demo_area(db, area) quality_check = DemoWorkflowService._persist_qa( @@ -330,6 +428,7 @@ class DemoWorkflowService: area_id=area.id, reference_dataset_id=reference.id, candidate_dataset_id=candidate.id, + raster_dataset_id=raster.id, quality_check_id=quality_check.id, metric_count=db.query(Metric).filter(Metric.quality_check_id == quality_check.id).count(), status="ready", @@ -352,6 +451,7 @@ class DemoWorkflowService: area = None reference = None candidate = None + raster = None quality_check = None created = True @@ -381,6 +481,8 @@ class DemoWorkflowService: source_name="fixture", reference_layer_name=None, ) + if not raster: + raster = DemoWorkflowService._create_raster_dataset(db=db, project_id=project.id, area_id=area.id) if not quality_check: quality_check = DemoWorkflowService._persist_qa( db=db, @@ -395,6 +497,7 @@ class DemoWorkflowService: area_id=area.id, reference_dataset_id=reference.id, candidate_dataset_id=candidate.id, + raster_dataset_id=raster.id, quality_check_id=quality_check.id, metric_count=6, status="ready", diff --git a/backend/tests/test_readiness_gate.py b/backend/tests/test_readiness_gate.py index 7b5fa8e1..397c35ac 100644 --- a/backend/tests/test_readiness_gate.py +++ b/backend/tests/test_readiness_gate.py @@ -121,7 +121,7 @@ def test_workbench_default_state_script_verifies_populated_demo_start_state() -> assert "/quality-checks" in content assert "data.items" in content assert "Demo AOI - Geel buildings" in content - assert "2/2 ready" in content + assert "3/3 ready" in content def test_pass_end_check_excludes_vendor_and_build_outputs() -> None: diff --git a/backend/tests/test_sprint97_demo_raster_fixture.py b/backend/tests/test_sprint97_demo_raster_fixture.py new file mode 100644 index 00000000..72a1fca5 --- /dev/null +++ b/backend/tests/test_sprint97_demo_raster_fixture.py @@ -0,0 +1,30 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_demo_workflow_contract_includes_raster_fixture_dataset() -> None: + schema = (ROOT / "backend" / "app" / "schemas" / "demo.py").read_text(encoding="utf-8") + service = (ROOT / "backend" / "app" / "services" / "demo_workflow_service.py").read_text(encoding="utf-8") + + assert "raster_dataset_id: UUID | None = None" in schema + assert 'RASTER_FILENAME = "demo_context_raster.tif"' in service + assert "_create_demo_raster_bytes" in service + assert "_create_raster_dataset" in service + assert "dataset_type=\"raster\"" in service + assert "source_name=\"fixture\"" in service + assert "raster_dataset_id=raster.id" in service + + +def test_demo_workflow_frontend_uses_raster_fixture_for_ai_labs() -> None: + types = (ROOT / "frontend" / "src" / "types.ts").read_text(encoding="utf-8") + app = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8") + hook = (ROOT / "frontend" / "src" / "hooks" / "useDemoWorkflow.ts").read_text(encoding="utf-8") + + assert "raster_dataset_id?: string | null" in types + assert "setSelectedDetectionDatasetId" in app + assert "setSelectedSegmentationDatasetId" in app + assert "setSelectedDetectionDatasetId(result.raster_dataset_id ?? '')" in hook + assert "setSelectedSegmentationDatasetId(result.raster_dataset_id ?? '')" in hook + assert "rasterDataset" in hook diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 877b7449..d522fc5d 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,29 @@ +## Sprint 97 demo raster fixture workflow (2026-06-23) + +Changed: +- Added `raster_dataset_id` to the demo workflow response contract. +- Extended `DemoWorkflowService` with a deterministic in-memory `demo_context_raster.tif` GeoTIFF fixture persisted through `StorageService` and the existing `datasets` table as a ready `raster`/`fixture` source dataset. +- Updated the frontend demo workflow hook so Detection and Segmentation Labs receive the seeded raster dataset while the candidate vector remains selected for Data, Map and Export review. +- Updated default-state and interaction smokes to require candidate vector, reference vector and raster fixture datasets as `3/3 ready`. +- Updated `scripts/README.md`, `frontend/README.md` and `CHANGELOG.md`. + +Validation: +- RED: `python -m pytest backend\tests\test_sprint97_demo_raster_fixture.py -q` failed before implementation because the demo schema/service and frontend hook did not expose or select a raster fixture. +- `python -m pytest backend\tests\test_sprint97_demo_raster_fixture.py -q` passed: 2 tests. +- `python -m compileall backend/app` passed. +- `python -m pytest backend\tests\test_sprint97_demo_raster_fixture.py backend\tests\test_sprint15_demo_workflow.py backend\tests\test_sprint21_demo_workflow_smoke.py backend\tests\test_readiness_gate.py -q` passed: 21 tests. +- `bash -n scripts/verify_workbench_default_state.sh` passed. +- `bash -n scripts/verify_workbench_interactions.sh` passed. +- `cd frontend && npm run typecheck` passed. +- `python -m pytest backend\tests\test_sprint96_useful_default_context.py backend\tests\test_sprint39_frontend_orchestration_hooks.py -q` passed: 11 tests. + +Limitations: +- The raster fixture is a tiny generated local GeoTIFF for V1 workflow validation only. It does not represent external imagery and does not enable real AI inference. +- No migrations, provider fetching, real YOLO/SAM behavior or product scope beyond the existing offline demo workflow changed. + +Next recommended pass: +- Run full readiness, rebuild/redeploy, then verify that the live browser-facing workbench exposes three ready demo datasets and preselects the raster fixture for AI Labs. + ## Sprint 48 Backend API contract audit (2026-06-17) Changed: diff --git a/frontend/README.md b/frontend/README.md index 3fc84e1a..5b4c71b2 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -260,6 +260,7 @@ AI Lab run controls explicitly explain when no raster dataset is available, inst - The Data workspace surfaces selected project, AOI and dataset context before creation/upload forms, then separates form and catalog/list regions for faster scanning. - The Export Center includes a handoff readiness summary, grouped artifact actions and provenance-rich export cards so report/GeoJSON handoff stays understandable in long-running demo projects. - The Export Center highlights the latest project report, project metadata, dataset GeoJSON, detection GeoJSON and segmentation GeoJSON artifacts with direct preview/download actions. +- The offline demo workflow now selects the seeded raster fixture for Detection and Segmentation Lab prerequisites while keeping the candidate vector dataset open for Data/Map/Export review. ## Workbench shell refactor diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 94c890b2..bc87dc4e 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -363,7 +363,9 @@ function App(): JSX.Element { setQaCandidateDatasetId, setQaReferenceDatasetId, setQaAreaId, + setSelectedDetectionDatasetId, setDetectionReferenceDatasetId, + setSelectedSegmentationDatasetId, setSegmentationReferenceDatasetId, setErrorMessage, }) diff --git a/frontend/src/hooks/useDemoWorkflow.ts b/frontend/src/hooks/useDemoWorkflow.ts index c4f3e650..8d3e3c81 100644 --- a/frontend/src/hooks/useDemoWorkflow.ts +++ b/frontend/src/hooks/useDemoWorkflow.ts @@ -17,7 +17,9 @@ interface DemoWorkflowOptions { setQaCandidateDatasetId: (datasetId: string) => void setQaReferenceDatasetId: (datasetId: string) => void setQaAreaId: (areaId: string) => void + setSelectedDetectionDatasetId: (datasetId: string) => void setDetectionReferenceDatasetId: (datasetId: string) => void + setSelectedSegmentationDatasetId: (datasetId: string) => void setSegmentationReferenceDatasetId: (datasetId: string) => void setErrorMessage: (message: string | null) => void } @@ -36,7 +38,9 @@ export function useDemoWorkflow({ setQaCandidateDatasetId, setQaReferenceDatasetId, setQaAreaId, + setSelectedDetectionDatasetId, setDetectionReferenceDatasetId, + setSelectedSegmentationDatasetId, setSegmentationReferenceDatasetId, setErrorMessage, }: DemoWorkflowOptions) { @@ -55,7 +59,9 @@ export function useDemoWorkflow({ setQaCandidateDatasetId(result.candidate_dataset_id) setQaReferenceDatasetId(result.reference_dataset_id) setQaAreaId(result.area_id) + setSelectedDetectionDatasetId(result.raster_dataset_id ?? '') setDetectionReferenceDatasetId(result.reference_dataset_id) + setSelectedSegmentationDatasetId(result.raster_dataset_id ?? '') setSegmentationReferenceDatasetId(result.reference_dataset_id) setDemoWorkflowMessage(result.message) await loadProjects(result.project_id) @@ -67,8 +73,11 @@ export function useDemoWorkflow({ loadExports(result.project_id), ]) const candidateDataset = projectData?.datasets.find((dataset) => dataset.id === result.candidate_dataset_id) + const rasterDataset = projectData?.datasets.find((dataset) => dataset.id === result.raster_dataset_id) if (candidateDataset) { await loadDatasetDetails(result.project_id, candidateDataset) + } else if (rasterDataset) { + await loadDatasetDetails(result.project_id, rasterDataset) } } catch (error) { setErrorMessage(formatError(error, 'Failed to load demo workflow')) diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 1649b99f..284eab9b 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -40,6 +40,7 @@ export interface DemoWorkflowResponse { area_id: string reference_dataset_id: string candidate_dataset_id: string + raster_dataset_id?: string | null quality_check_id: string metric_count: number status: string diff --git a/scripts/README.md b/scripts/README.md index 08bdce3b..f972f460 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -46,9 +46,10 @@ bash scripts/verify_workbench_default_state.sh http://192.168.10.150:1202 This smoke is dependency-light and intentionally idempotent: it seeds the offline demo workflow, then verifies that `GeoIntel Demo - Building QA` exposes -the `Demo AOI - Geel buildings` map geometry, `2/2 ready` demo datasets and a -persisted QA/QC result through canonical `data.items` envelopes. Pair it with a -Codex/browser screenshot pass when checking visual layout or overflow. +the `Demo AOI - Geel buildings` map geometry, `3/3 ready` demo datasets +(candidate vector, reference vector and raster fixture) and a persisted QA/QC +result through canonical `data.items` envelopes. Pair it with a Codex/browser +screenshot pass when checking visual layout or overflow. Verify the backing state for the core workbench interactions: diff --git a/scripts/verify_workbench_default_state.sh b/scripts/verify_workbench_default_state.sh index 826c3228..44e8340e 100644 --- a/scripts/verify_workbench_default_state.sh +++ b/scripts/verify_workbench_default_state.sh @@ -79,6 +79,7 @@ project_id="$(json_field "${TMP_DIR}/demo.json" "data.project_id")" area_id="$(json_field "${TMP_DIR}/demo.json" "data.area_id")" candidate_dataset_id="$(json_field "${TMP_DIR}/demo.json" "data.candidate_dataset_id")" reference_dataset_id="$(json_field "${TMP_DIR}/demo.json" "data.reference_dataset_id")" +raster_dataset_id="$(json_field "${TMP_DIR}/demo.json" "data.raster_dataset_id")" quality_check_id="$(json_field "${TMP_DIR}/demo.json" "data.quality_check_id")" curl -fsS "${BASE_URL%/}/api/v1/projects" > "${TMP_DIR}/projects.json" @@ -123,11 +124,11 @@ PY curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/datasets" > "${TMP_DIR}/datasets.json" require_json_data "${TMP_DIR}/datasets.json" -"${PYTHON_BIN}" - "${TMP_DIR}/datasets.json" "${candidate_dataset_id}" "${reference_dataset_id}" <<'PY' +"${PYTHON_BIN}" - "${TMP_DIR}/datasets.json" "${candidate_dataset_id}" "${reference_dataset_id}" "${raster_dataset_id}" <<'PY' import json import sys -path, candidate_id, reference_id = sys.argv[1], sys.argv[2], sys.argv[3] +path, candidate_id, reference_id, raster_id = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] with open(path, "r", encoding="utf-8") as handle: payload = json.load(handle) items = payload["data"]["items"] @@ -136,10 +137,14 @@ if candidate_id not in ids: raise SystemExit("Candidate fixture dataset is missing from default workbench dataset list") if reference_id not in ids: raise SystemExit("Reference fixture dataset is missing from default workbench dataset list") -if ids[candidate_id].get("status") != "ready" or ids[reference_id].get("status") != "ready": - raise SystemExit("Expected 2/2 ready demo datasets") +if raster_id not in ids: + raise SystemExit("Raster fixture dataset is missing from default workbench dataset list") +if ids[candidate_id].get("status") != "ready" or ids[reference_id].get("status") != "ready" or ids[raster_id].get("status") != "ready": + raise SystemExit("Expected 3/3 ready demo datasets") if ids[reference_id].get("dataset_role") != "reference": raise SystemExit("Default workbench reference dataset is not marked as reference") +if ids[raster_id].get("dataset_type") != "raster" or ids[raster_id].get("source_name") != "fixture": + raise SystemExit("Default workbench raster fixture lost its raster/source metadata") PY curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks" > "${TMP_DIR}/quality_checks.json" @@ -167,5 +172,5 @@ PY echo "Workbench default state verification passed" echo "Project: GeoIntel Demo - Building QA" echo "Area: Demo AOI - Geel buildings" -echo "Datasets: 2/2 ready" +echo "Datasets: 3/3 ready" echo "QA/QC: seeded check is available" diff --git a/scripts/verify_workbench_interactions.sh b/scripts/verify_workbench_interactions.sh index 24b35b7f..41dfe224 100644 --- a/scripts/verify_workbench_interactions.sh +++ b/scripts/verify_workbench_interactions.sh @@ -67,6 +67,7 @@ project_id="$(json_field "${TMP_DIR}/demo.json" "data.project_id")" area_id="$(json_field "${TMP_DIR}/demo.json" "data.area_id")" candidate_dataset_id="$(json_field "${TMP_DIR}/demo.json" "data.candidate_dataset_id")" reference_dataset_id="$(json_field "${TMP_DIR}/demo.json" "data.reference_dataset_id")" +raster_dataset_id="$(json_field "${TMP_DIR}/demo.json" "data.raster_dataset_id")" quality_check_id="$(json_field "${TMP_DIR}/demo.json" "data.quality_check_id")" curl -fsS "${BASE_URL%/}/api/v1/projects" > "${TMP_DIR}/projects.json" @@ -87,12 +88,13 @@ require_json_data "${TMP_DIR}/quality_checks.json" "${area_id}" \ "${candidate_dataset_id}" \ "${reference_dataset_id}" \ + "${raster_dataset_id}" \ "${quality_check_id}" <<'PY' import json import sys projects_path, areas_path, datasets_path, quality_path = sys.argv[1:5] -project_id, area_id, candidate_id, reference_id, quality_check_id = sys.argv[5:10] +project_id, area_id, candidate_id, reference_id, raster_id, quality_check_id = sys.argv[5:11] def load(path): with open(path, "r", encoding="utf-8") as handle: @@ -116,14 +118,19 @@ if not area.get("geometry"): dataset_ids = {item["id"]: item for item in datasets} candidate = dataset_ids.get(candidate_id) reference = dataset_ids.get(reference_id) +raster = dataset_ids.get(raster_id) if not candidate: raise SystemExit("candidate dataset backing state is missing") if not reference: raise SystemExit("reference dataset backing state is missing") -if candidate.get("status") != "ready" or reference.get("status") != "ready": - raise SystemExit("Dataset selection backing state is not 2/2 ready") +if not raster: + raise SystemExit("raster dataset backing state is missing") +if candidate.get("status") != "ready" or reference.get("status") != "ready" or raster.get("status") != "ready": + raise SystemExit("Dataset selection backing state is not 3/3 ready") if reference.get("dataset_role") != "reference": raise SystemExit("reference dataset backing state lost its reference role") +if raster.get("dataset_type") != "raster" or raster.get("source_name") != "fixture": + raise SystemExit("raster dataset backing state lost its fixture raster metadata") quality_check = next((item for item in checks if item["id"] == quality_check_id), None) if not quality_check or quality_check.get("status") != "ok": @@ -158,6 +165,6 @@ PY echo "Workbench interaction backing-state verification passed" echo "Project switch: GeoIntel Demo - Building QA" echo "Area selection: Demo AOI - Geel buildings" -echo "Dataset selection: candidate dataset and reference dataset ready" +echo "Dataset selection: candidate, reference and raster fixture datasets ready" echo "QA refresh: seeded metrics available" echo "Export refresh: metadata export listed"