From 9e20cc82ae56a3a52a93f06044b428c3dc8fa1dd Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 6 Jul 2026 20:47:02 +0200 Subject: [PATCH] Document model and reference catalog plan --- .../2026-07-06-model-reference-catalog.md | 410 ++++++++++++++++++ ...26-07-06-model-reference-catalog-design.md | 217 +++++++++ 2 files changed, 627 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-06-model-reference-catalog.md create mode 100644 docs/superpowers/specs/2026-07-06-model-reference-catalog-design.md diff --git a/docs/superpowers/plans/2026-07-06-model-reference-catalog.md b/docs/superpowers/plans/2026-07-06-model-reference-catalog.md new file mode 100644 index 00000000..645386ad --- /dev/null +++ b/docs/superpowers/plans/2026-07-06-model-reference-catalog.md @@ -0,0 +1,410 @@ +# Model And Reference Catalog Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a read-only local model asset catalog and make official reference providers visually distinct from AI model selection. + +**Architecture:** Model files remain runtime assets in the mounted model directory; persisted domain state remains in jobs, analysis runs, detections, datasets and vector features. Detection runs keep `model_id="yolo-configured"` and optionally include a validated `model_asset_id` resolved by a dedicated backend service. + +**Tech Stack:** FastAPI, Pydantic, SQLAlchemy, React, TypeScript, Vite, MapLibre, existing GeoIntel API envelope helpers. + +--- + +## File Structure + +- Create `backend/app/services/model_asset_catalog_service.py`: read-only scanner and resolver for local model assets. +- Modify `backend/app/core/config.py`: add `YOLO_MODELS_DIR` setting with safe default `/app/models`. +- Modify `backend/app/schemas/detection.py`: add model asset response schemas and optional request fields. +- Modify `backend/app/api/routes/detection.py`: add model asset endpoint and pass selected asset IDs to preflight/detection. +- Modify `backend/app/services/detection_service.py`: resolve selected model asset into run-local YOLO settings and persist selection in parameters. +- Modify `backend/app/services/yolo_preflight_service.py`: accept selected asset ID and preflight against that asset path. +- Add `backend/tests/test_model_asset_catalog.py`: unit tests for scanning, resolving and envelope behavior. +- Add or extend `backend/tests/test_sprint8b_yolo_foundation.py`: detection run parameter persistence for selected assets. +- Modify `frontend/src/types.ts`: add model asset types and optional request fields. +- Modify `frontend/src/services/api/detection.ts`: add model asset endpoint and preflight query param. +- Modify `frontend/src/hooks/useDetectionWorkflow.ts`: load/select model assets and send selected asset ID. +- Modify `frontend/src/components/detection/DetectionLab.tsx`: add local model asset picker and clearer model state. +- Modify `frontend/src/components/providers/ProviderPanel.tsx`: sharpen reference-provider catalog copy. +- Update docs listed in the design document. + +## Task 1: Backend Catalog Tests + +**Files:** + +- Create: `backend/tests/test_model_asset_catalog.py` + +- [ ] **Step 1: Write catalog unit tests** + +```python +from pathlib import Path + +from app.core.config import Settings +from app.services.model_asset_catalog_service import ModelAssetCatalogService + + +def test_model_asset_catalog_lists_supported_files(tmp_path: Path) -> None: + model_file = tmp_path / "building-detector.pt" + model_file.write_bytes(b"local model") + ignored_file = tmp_path / "notes.txt" + ignored_file.write_text("ignore me", encoding="utf-8") + settings = Settings(yolo_models_dir=str(tmp_path), yolo_model_path=str(model_file), yolo_enabled=True) + + response = ModelAssetCatalogService.list_assets(settings=settings) + + assert response.total == 1 + asset = response.items[0] + assert asset.filename == "building-detector.pt" + assert asset.active is True + assert asset.model_path == str(model_file) + assert asset.size_bytes == len(b"local model") + assert len(asset.sha256) == 64 + assert asset.will_download_models is False + + +def test_model_asset_catalog_resolves_known_asset(tmp_path: Path) -> None: + model_file = tmp_path / "building-detector.pt" + model_file.write_bytes(b"local model") + settings = Settings(yolo_models_dir=str(tmp_path), yolo_enabled=True) + + asset = ModelAssetCatalogService.resolve_asset("building-detector-pt", settings=settings) + + assert asset.filename == "building-detector.pt" + assert asset.model_path == str(model_file) + + +def test_model_asset_catalog_rejects_unknown_asset(tmp_path: Path) -> None: + settings = Settings(yolo_models_dir=str(tmp_path), yolo_enabled=True) + + try: + ModelAssetCatalogService.resolve_asset("missing-model", settings=settings) + except Exception as exc: + assert getattr(exc, "code", "") == "DETECTION_MODEL_ASSET_NOT_FOUND" + else: + raise AssertionError("unknown model asset should fail") +``` + +- [ ] **Step 2: Run the tests to verify they fail before implementation** + +Run: + +```bash +cd backend +python -m pytest tests/test_model_asset_catalog.py -q +``` + +Expected: import failure for `model_asset_catalog_service`. + +## Task 2: Backend Catalog Implementation + +**Files:** + +- Create: `backend/app/services/model_asset_catalog_service.py` +- Modify: `backend/app/core/config.py` +- Modify: `backend/app/schemas/detection.py` + +- [ ] **Step 1: Add settings and schemas** + +Add `yolo_models_dir` to settings: + +```python +yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR") +``` + +Add schemas: + +```python +class ModelAssetRead(BaseModel): + model_asset_id: str + filename: str + display_name: str + model_path: str + suffix: str + framework: str + task_type: str + size_bytes: int + sha256: str + active: bool + status: str + limitation_message: str + will_download_models: bool = False + + +class ModelAssetListResponse(BaseModel): + items: list[ModelAssetRead] + total: int + model_directory: str +``` + +- [ ] **Step 2: Implement the scanner and resolver** + +Implement a service that: + +- resolves `YOLO_MODELS_DIR`; +- scans `.pt`, `.onnx`, `.engine`; +- uses deterministic slug IDs such as `building-detector-pt`; +- computes SHA-256; +- marks active file by comparing resolved paths; +- raises `AppError(code="DETECTION_MODEL_ASSET_NOT_FOUND", status_code=404)` for unknown IDs. + +- [ ] **Step 3: Run focused tests** + +Run: + +```bash +cd backend +python -m pytest tests/test_model_asset_catalog.py -q +``` + +Expected: pass. + +## Task 3: API And Service Wiring + +**Files:** + +- Modify: `backend/app/api/routes/detection.py` +- Modify: `backend/app/services/detection_service.py` +- Modify: `backend/app/services/yolo_preflight_service.py` +- Modify: `backend/app/schemas/detection.py` +- Test: `backend/tests/test_model_asset_catalog.py` + +- [ ] **Step 1: Add endpoint test** + +Add a FastAPI test that calls `GET /api/v1/detection/model-assets` and asserts: + +```python +assert "data" in response.json() +assert response.json()["data"]["total"] == 1 +``` + +- [ ] **Step 2: Add optional request fields** + +Extend `DetectionRunRequest` with: + +```python +model_asset_id: str | None = None +``` + +Extend preflight query handling with `model_asset_id: str | None = None`. + +- [ ] **Step 3: Wire endpoint** + +Add route: + +```python +@router.get("/model-assets", response_model=dict) +def list_detection_model_assets() -> dict: + return envelope(ModelAssetCatalogService.list_assets().model_dump()) +``` + +- [ ] **Step 4: Resolve selected asset in detection service** + +When `model_asset_id` is present and `model_id` equals the configured YOLO model ID: + +- resolve the asset through `ModelAssetCatalogService`; +- copy settings with `yolo_model_path=asset.model_path`; +- persist `model_asset_id`, `model_asset_path`, and `model_asset_sha256` in run parameters. + +- [ ] **Step 5: Resolve selected asset in preflight** + +When `model_asset_id` is present: + +- resolve the asset; +- preflight against that asset path; +- include `model_asset_id` in the response. + +- [ ] **Step 6: Run focused tests** + +Run: + +```bash +cd backend +python -m pytest tests/test_model_asset_catalog.py tests/test_sprint8b_yolo_foundation.py -q +``` + +Expected: pass. + +## Task 4: Frontend Model Asset Selection + +**Files:** + +- Modify: `frontend/src/types.ts` +- Modify: `frontend/src/services/api/detection.ts` +- Modify: `frontend/src/hooks/useDetectionWorkflow.ts` +- Modify: `frontend/src/components/detection/DetectionLab.tsx` + +- [ ] **Step 1: Add frontend types and API call** + +Add: + +```ts +export interface ModelAssetRead { + model_asset_id: string + filename: string + display_name: string + model_path: string + suffix: string + framework: string + task_type: string + size_bytes: number + sha256: string + active: boolean + status: string + limitation_message: string + will_download_models: boolean +} + +export interface ModelAssetListResponse { + items: ModelAssetRead[] + total: number + model_directory: string +} +``` + +Add `model_asset_id?: string | null` to `DetectionRunRequest` and preflight params. + +- [ ] **Step 2: Add hook state** + +Add: + +```ts +const [modelAssets, setModelAssets] = useState([]) +const [selectedModelAssetId, setSelectedModelAssetId] = useState('') +const [modelAssetError, setModelAssetError] = useState(null) +``` + +Load assets alongside detection models and default to the active asset when one exists. + +- [ ] **Step 3: Send selected asset** + +Send `model_asset_id: selectedModelAssetId || null` in: + +- `getYoloPreflight`; +- `run`. + +- [ ] **Step 4: Add UI picker** + +In `DetectionLab`, render a local model asset picker when `selectedDetectionModelId === "yolo-configured"`. + +Show: + +- filename; +- active badge; +- size in MB; +- checksum prefix; +- no-download warning. + +- [ ] **Step 5: Typecheck** + +Run: + +```bash +cd frontend +npm run typecheck +``` + +Expected: pass. + +## Task 5: Provider Catalog Clarity + +**Files:** + +- Modify: `frontend/src/components/providers/ProviderPanel.tsx` + +- [ ] **Step 1: Update copy and grouping** + +Keep existing provider cards but add a concise heading that states: + +- GRB/OSM are source capabilities, not model choices; +- manual upload is the currently configured reference data path; +- fixture is test/demo only. + +- [ ] **Step 2: Build** + +Run: + +```bash +cd frontend +npm run build +``` + +Expected: pass. + +## Task 6: Documentation + +**Files:** + +- Modify: `docs/API_CONTRACTS.md` +- Modify: `docs/AI_PIPELINES.md` +- Modify: `backend/README.md` +- Modify: `frontend/README.md` +- Modify: `docs/CODEX_EXECUTION_LOG.md` +- Modify: `CHANGELOG.md` +- Modify: `docs/TODO.md` + +- [ ] **Step 1: Document backend API** + +Document: + +- `GET /api/v1/detection/model-assets`; +- optional `model_asset_id` on preflight and run; +- no model downloads; +- `YOLO_MODELS_DIR`. + +- [ ] **Step 2: Document UI behavior** + +Document: + +- model assets are local runtime files; +- GRB/OSM are reference data providers; +- manual upload remains the configured reference path. + +## Task 7: Full Validation + +**Files:** none unless validation reveals a bug. + +- [ ] **Step 1: Run backend compile** + +```bash +python -m compileall backend/app +``` + +- [ ] **Step 2: Run backend tests** + +```bash +cd backend +python -m pytest +``` + +- [ ] **Step 3: Run readiness** + +```bash +bash scripts/run_readiness_check.sh +``` + +- [ ] **Step 4: Run frontend typecheck and build** + +```bash +cd frontend +npm run typecheck +npm run build +``` + +- [ ] **Step 5: Run migration checks** + +```bash +cd backend +python -m alembic heads +python -m alembic upgrade head --sql +bash ../scripts/live_migration_smoke.sh +``` + +- [ ] **Step 6: Runtime smoke after deploy** + +Call: + +```bash +curl http://192.168.10.150:1202/api/v1/detection/model-assets +curl "http://192.168.10.150:1202/api/v1/detection/yolo/preflight?model_asset_id=&tile_manifest_path=" +``` + +Expected: canonical envelopes and no model downloads. diff --git a/docs/superpowers/specs/2026-07-06-model-reference-catalog-design.md b/docs/superpowers/specs/2026-07-06-model-reference-catalog-design.md new file mode 100644 index 00000000..6b68d8a3 --- /dev/null +++ b/docs/superpowers/specs/2026-07-06-model-reference-catalog-design.md @@ -0,0 +1,217 @@ +# Model And Reference Catalog Design + +## Goal + +Make GeoIntel operationally clearer for V1 users by separating two concepts that currently look too similar in the UI: + +- local AI model assets that can be selected for configured YOLO inference; +- official or contextual reference data providers such as GRB, OSM, manual uploads and fixtures. + +This pass must not download model weights, fetch live GRB/OSM data, add training, add auth, or bypass the existing dataset, job, analysis run and detection persistence architecture. + +## Current State + +Detection currently exposes a model capability list with: + +- `yolo-placeholder`; +- one configured slot, `yolo-configured`, backed by `YOLO_MODEL_PATH`; +- `manual-fixture-detector`. + +This is import-safe and honest, but it does not feel like a model picker. A user can place multiple files in `/app/models`, yet the UI can only show the single configured environment slot. + +Reference data currently exposes provider capabilities for: + +- `grb`; +- `osm`; +- `manual`; +- `fixture`. + +This is architecturally correct, but the UI does not yet make the distinction explicit enough between authoritative reference sources and AI model assets. + +## Non-Goals + +- No automatic model downloads. +- No bundled production model files in git. +- No live GRB WFS or OSM Overpass import. +- No direct writes from providers into `vector_features`. +- No new database tables for model assets in this pass. +- No training studio, model management workflow, LiDAR, reports, copilot or multi-user scope. + +## Options Considered + +### Option A: Keep Only `YOLO_MODEL_PATH` + +Keep the current single configured model slot and document that users must edit `.env`. + +Benefits: + +- smallest code change; +- preserves all existing contracts. + +Drawbacks: + +- poor operator experience; +- no visible list of available local model files; +- users cannot tell whether the model directory contains other usable files. + +### Option B: Filesystem-Backed Model Asset Catalog + +Scan a configured model directory, expose local model files through an API, and let the UI select one asset for `yolo-configured` runs. + +Benefits: + +- aligns with the current runtime model mount (`/app/models`); +- no database migration; +- no downloads or fake model metadata; +- can show file existence, size, checksum and active environment model; +- keeps actual inference inside `DetectionService` and `YoloDetectionAdapter`. + +Drawbacks: + +- metadata is limited unless optional sidecar files are added later; +- model classes are not guaranteed without loading the model. + +### Option C: Persisted Model Registry + +Create database tables for model registry records, model versions, model artifacts and model lifecycle state. + +Benefits: + +- strong long-term foundation for training studio and MLOps; +- full metadata and auditability. + +Drawbacks: + +- too broad for V1; +- adds migration and lifecycle complexity before runtime needs justify it; +- risks distracting from core GIS workflow completion. + +## Recommended Approach + +Use Option B. + +Add a filesystem-backed model asset catalog for local runtime model files. It should scan `YOLO_MODELS_DIR`, defaulting to `/app/models`, and fall back to the parent directory of `YOLO_MODEL_PATH` when appropriate. It should only report local files with known model suffixes such as `.pt`, `.onnx` and `.engine`. + +The catalog must be read-only. It must never download, create, mutate, move or delete model files. + +Detection runs should still use `model_id="yolo-configured"` for the real YOLO execution path, but may include a selected `model_asset_id`. The backend resolves that ID to a path inside the configured model directory and uses that path for the run. This avoids arbitrary path injection while keeping the existing detection contract compatible. + +## Backend Design + +Create `ModelAssetCatalogService`. + +Responsibilities: + +- resolve the model directory from settings; +- scan known model file suffixes; +- return deterministic asset IDs derived from file names; +- compute SHA-256 and size for visible provenance; +- mark which asset matches the active `YOLO_MODEL_PATH`; +- resolve a selected asset ID to a local path; +- reject missing, unknown or out-of-directory model assets. + +Add schemas: + +- `ModelAssetRead`; +- `ModelAssetListResponse`. + +Add endpoint: + +- `GET /api/v1/detection/model-assets` + +Extend existing endpoints without breaking older clients: + +- `GET /api/v1/detection/yolo/preflight` accepts optional `model_asset_id`; +- `POST /api/v1/detection/run` accepts optional `model_asset_id`. + +When `model_asset_id` is supplied, `DetectionService` should use a settings copy with `yolo_model_path` replaced by the resolved asset path. The run parameters should persist the selected asset ID and path for reproducibility. + +## Frontend Design + +Detection Lab should show: + +- model capability cards; +- a local model asset picker for configured YOLO; +- active model indicator; +- asset size and checksum prefix; +- selected asset passed to preflight and detection run; +- clear warning that GeoIntel does not download weights. + +Provider panel should show: + +- official reference source catalog; +- GRB as authoritative but not configured for live fetch; +- OSM as contextual and not configured for live fetch; +- manual uploads as the configured way to add real reference datasets now; +- fixtures as demo/test only. + +## Data Flow + +```text +/app/models/*.pt + -> ModelAssetCatalogService + -> GET /api/v1/detection/model-assets + -> Detection Lab model asset picker + -> POST /api/v1/detection/run model_id=yolo-configured + model_asset_id + -> DetectionService resolves local path + -> YoloDetectionAdapter loads selected local model + -> Job + AnalysisRun + Detection persistence +``` + +Reference data remains: + +```text +Provider registry + -> capabilities/status/limitations + -> manual upload or future provider import + -> DatasetService / VectorFeatureService + -> vector_features + -> detection/segmentation QA +``` + +## Error Handling + +- Unknown `model_asset_id`: `DETECTION_MODEL_ASSET_NOT_FOUND`. +- Model asset outside configured directory: `DETECTION_MODEL_ASSET_INVALID`. +- Missing configured YOLO dependencies: existing dependency unavailable behavior. +- Missing tile manifest: existing tile manifest required behavior. +- Missing model file after catalog resolution: existing model unavailable behavior. + +## Tests + +Backend tests should cover: + +- catalog lists only supported local model files; +- catalog marks the active model; +- checksum and size are reported; +- invalid assets are ignored; +- unknown asset ID fails cleanly; +- selected model asset is persisted in job and analysis run parameters; +- model assets endpoint uses canonical envelope; +- preflight accepts selected asset without model downloads. + +Frontend tests should cover: + +- Detection Lab exposes local model asset selection; +- selected model asset is sent to run and preflight requests; +- Provider Panel copy distinguishes reference providers from model assets. + +## Documentation + +Update: + +- `docs/API_CONTRACTS.md`; +- `docs/AI_PIPELINES.md`; +- `backend/README.md`; +- `frontend/README.md`; +- `docs/CODEX_EXECUTION_LOG.md`; +- `CHANGELOG.md`; +- `docs/TODO.md`. + +## Future Work + +- Sidecar model metadata files, for example `model.pt.json`, for class names, source, license and intended task; +- optional model compatibility smoke per selected asset; +- persisted model registry after V1 foundation is stable; +- live GRB/OSM imports through provider contracts; +- official reference dataset browser after live provider imports exist.