commit 6ea3586a3edf27ea6f3e420539d611c0ddd25ad2 Author: Codex Date: Tue Jun 16 23:36:32 2026 +0200 Initial GeoIntel V1 foundation diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..d80a6c69 --- /dev/null +++ b/.env.example @@ -0,0 +1,22 @@ +# Backend +GEOINTEL_ENV=development +GEOINTEL_API_PREFIX=/api/v1 +DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1 +STORAGE_ROOT=./storage +MAX_UPLOAD_MB=500 +CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202 +YOLO_ENABLED=false +YOLO_MODEL_PATH= +YOLO_MODEL_VERSION= +YOLO_MAX_TILES=100 +ENABLE_GRB_WFS=false +GRB_WFS_URL= +OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter + +# Install backend raster dependencies when needed: +# python -m pip install rasterio + +# Frontend +VITE_API_BASE_URL= +VITE_API_PROXY_TARGET=http://localhost:8000 +VITE_MAP_STYLE_URL=https://demotiles.maplibre.org/style.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..02acc8de --- /dev/null +++ b/.gitattributes @@ -0,0 +1,12 @@ +*.sh text eol=lf +*.py text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.toml text eol=lf +*.ini text eol=lf +Dockerfile text eol=lf +*.md text eol=lf +*.tsx text eol=lf +*.ts text eol=lf +*.css text eol=lf +*.json text eol=lf diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..e83cd15d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,43 @@ +--- +name: Bug report +about: Report a reproducible defect +--- + +## Summary + +## Steps to reproduce + +1. +2. +3. + +## Expected behavior + +## Actual behavior + +## Affected module + +- [ ] Backend +- [ ] Frontend +- [ ] Database +- [ ] Raster +- [ ] Vector +- [ ] AI/Detection +- [ ] QA/QC +- [ ] Export +- [ ] Docs + +## Logs/screenshots + +## Data involved + +- Dataset: +- CRS: +- Geometry type: + +## Risk + +- [ ] Blocks build +- [ ] Data correctness issue +- [ ] UX issue +- [ ] Documentation issue diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..390e579e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,26 @@ +--- +name: Feature request +about: Propose an improvement without breaking scope +--- + +## Problem + +## Proposed solution + +## Scope category + +- [ ] V1 in scope +- [ ] V1 adjacent +- [ ] V2+ +- [ ] RFC required + +## Affected modules + +## Acceptance criteria + +- [ ] +- [ ] + +## Risks + +## Notes diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..832b1f27 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,21 @@ +# Summary + +## Changed files + +## Acceptance criteria + +- [ ] Meets pass prompt +- [ ] Meets M6 quality gates +- [ ] Tests run +- [ ] Docs updated +- [ ] No architecture drift + +## Tests + +```bash +# commands +``` + +## Known limitations + +## Next pass recommendation diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..d5d1cdcd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: GeoIntel CI + +on: + push: + branches: [ main, develop, 'build/**' ] + pull_request: + branches: [ main, develop ] + +jobs: + docs-smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Validate repository docs + run: | + python scripts/smoke_docs.py + python scripts/validate_fixtures.py + + contract-smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Run contract smoke checks + run: python scripts/smoke_contracts.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..42425ac6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,40 @@ +# Python +__pycache__/ +*.py[cod] +.venv/ +venv/ +.env +*.egg-info/ + +# Node +node_modules/ +dist/ +build/ +*.tsbuildinfo + +# Large local data +/datasets/raw/* +/datasets/processed/* +/datasets/cache/* +/storage/uploads/* +/storage/tiles/* +/storage/masks/* +/storage/reports/* +/storage/exports/* +/storage/rasters/* +/exports/* +/models/* +/backend/storage/uploads/* +/backend/storage/tiles/* +/backend/storage/masks/* +/backend/storage/reports/* +/backend/storage/exports/* + +# Keep folder placeholders +!**/.gitkeep +!**/README.md + +# OS/editor +.DS_Store +.vscode/ +.idea/ diff --git a/.gitkeep b/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..72c9ff4e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,42 @@ +# AI Agent Instructions for GeoIntel + +## Project identity + +GeoIntel Kempen is a GeoAI Workbench, not a generic CRUD app and not a generic dashboard. + +## Required behavior + +- Read `docs/CODEX_BOOTSTRAP_PROMPT.md` first. +- Respect `docs/V1_SCOPE_FREEZE.md`. +- Use `docs/API_CONTRACTS.md` as source of truth for endpoints. +- Use `docs/DATABASE_IMPLEMENTATION_PLAN.md` as source of truth for persistence. +- Use `docs/DEFINITION_OF_DONE.md` to decide whether work is complete. + +## Agent roles + +### Architecture Agent + +Owns repository layout, API contracts, database migrations and service boundaries. + +### GIS Agent + +Owns GeoPandas, Shapely, Rasterio, CRS, clipping, buffering, spatial joins and metadata extraction. + +### AI Agent + +Owns YOLO/SAM abstractions, inference contracts, model configuration, detection/segmentation persistence and `not_configured` behavior. + +### QA Agent + +Owns tests, QA/QC metrics, regression checks and acceptance criteria. + +### Frontend Agent + +Owns React, TypeScript, MapLibre, API client, UI states and workbench UX. + +## Never do this + +- Do not fake production AI outputs. +- Do not silently skip geospatial validation. +- Do not add auth/multi-user/LiDAR/training before V1 foundation is stable. +- Do not remove documentation to avoid conflicts. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..2f7b68fb --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,310 @@ +# M13 — Codex Optimization Pack + +- Added reusable Codex skills under `skills/`. +- Added prompt discipline, token/context budget policy, secrets policy and parallel-agent strategy. +- Added M13 day-one optimized master prompt and pass completion report prompt. +- Added M13 validation script and included it in readiness checks. + +# Changelog + +## Release hardening audit pass (2026-06-15) + +- Replaced remaining backend `datetime.utcnow()` usage with timezone-aware UTC timestamps. +- Verified the affected backend tests with `DeprecationWarning` promoted to errors. +- Split the frontend production bundle into explicit app, React vendor and MapLibre vendor chunks. +- Raised the Vite chunk warning threshold to match the isolated MapLibre GIS dependency rather than masking app-code growth. +- Hardened the main readiness gate so backend deprecation warnings fail release readiness. +- Added API contract smoke validation to the main readiness gate. +- Hardened the pass-end placeholder scan to skip dependency, build-output and bytecode-cache folders. +- Added backend tests for the release/readiness script expectations. +- Updated `docs/TODO.md` with current implementation status while preserving older planning context. +- No API contracts, migrations, product features, AI dependencies or provider behavior were changed. + +## Docker runtime hardening (2026-06-15) + +- Fixed the backend Docker build by copying `README.md` and `app/` before `pip install .`. +- Removed mandatory Compose `.env` references so `docker compose up` works with checked-in local defaults. +- Published the Docker Compose frontend on host port `1202`. +- Added backend CORS defaults for `http://localhost:1202` and `http://127.0.0.1:1202`. +- Stopped publishing PostGIS on host port `5432`; backend uses Docker-internal `db:5432`. +- Added a PostGIS healthcheck and made the backend wait for a healthy database. +- Added a backend Docker start script that retries a real SQL connection before running migrations, avoiding first-start database race conditions. +- Made the backend container run `alembic upgrade head` before starting Uvicorn. +- Added backend/frontend `.dockerignore` files to keep dependency folders, build outputs and caches out of Docker build contexts. +- Added Docker runtime configuration regression tests. +- Fixed Alembic logging format so Docker migration logs no longer print literal `%(levelname)` formatter strings. +- Changed the frontend API default to same-origin requests and added a Vite proxy for `/api` and `/health`, with Docker routing to `http://backend:8000`. +- Added a browser runtime verification script that fails when the frontend `/api` proxy returns Vite HTML instead of the backend JSON envelope. +- Updated environment and local runbook documentation so Docker/LAN browser clients use same-origin API calls through the frontend proxy by default. +- Corrected example YOLO environment variables to the names the backend actually reads: `YOLO_ENABLED`, `YOLO_MODEL_PATH` and `YOLO_MAX_TILES`. +- Replaced the Docker frontend runtime with an nginx-served production build and explicit `/api` plus `/health` reverse proxy to the backend service, avoiding Vite HTML fallback for API requests. + +## M6 — Codex Autonomy Pack + +Added: + +- M6 autonomy boundaries. +- M6 quality gates. +- Codex self-review checklist. +- Failure recovery playbook. +- Gap registry. +- Next-day execution checklist. +- Final handoff template. +- Codex pass prompts PASS 00 through PASS 12. +- GitHub issue templates and PR template. +- GitHub Actions docs/contract smoke workflow. +- Codex preflight and pass-end scripts. + +Purpose: + +- Prepare the repository so Codex can build with strict guidance and bounded improvement freedom. + +## M7 - Implementation Control Layer + +Added: + +- M7 implementation control layer. +- Locked build sequence. +- Regression trap catalogue. +- Codex self-review checklist. +- Geospatial calculation rules. +- Frontend state rules. +- API response rules. +- Module completion matrix. +- Codex decision boundaries. +- Proposed improvements backlog. +- End-of-pass review prompts. +- Regression and contract drift audit prompts. +- Module contracts for project/area/dataset, detection boundary and QA/QC. +- M7 self-review scripts. + +## M8 - Tomorrow Execution Pack + +- Added Day 1 Codex execution pack. +- Added pass-by-pass Day 1 prompts. +- Added autonomy boundaries, failure recovery and quality gate matrix. +- Added operator checklist and smoke script scaffold. +- Added next-pass guidance for Day 2 GeoAI loop. + +## v0.9 — M9 Max Preparation + +- Added Codex day-one master prompt. +- Added autonomous build doctrine and pass scorecards. +- Added real-vs-demo data policy and detailed data contracts. +- Added geospatial edge cases, UI state spec and API validation examples. +- Added implementation review script, regression map and gap-to-task conversion rules. +- Added final pre-code checklist and long-form prompt variants. + +## Sprint 1 readiness hardening (2026-06-11) + +- Added cross-platform backend/runtime scripts with `python`/`python3` fallback in readiness tooling. +- Fixed backend packaging metadata so editable install works in current flat repo layout. +- Added dependency and smoke test script updates for Sprint 1 services. +- Added minimal Sprint 1 tests for health endpoint, GeoJSON metadata extraction, invalid payload rejection, and dataset content reads. +- Fixed frontend README doc reference typo for repository conventions. +- Updated Sprint 1 docs to include backend import smoke and concrete local setup commands. + +## Sprint 2 foundation (2026-06-11) + +- Added canonical vector/raster dataset handling and lifecycle status transitions (`uploaded`, `validating`, `ready`, `failed`). +- Added vector metadata extraction (feature count, geometry types, bounds, approximate area, CRS assumptions). +- Added raster metadata extraction service with dependency-aware fallback (`RASTER_PROCESSING_UNAVAILABLE`). +- Added deterministic storage metadata capture (`original_filename`, `stored_filename`, `content_type`, `size_bytes`, `checksum_sha256`) and upload folder layout. +- Added dataset inspection/vector summary/raster metadata API endpoints and frontend detail panel support. +- Added Sprint 2 tests for vector metadata, invalid GeoJSON handling, legacy geojson compatibility and raster dependency fallback. + +## Sprint 3 foundation (2026-06-11) + +- Added job model and database migration for queued/running/success/failed operations. +- Added job APIs for create/list/read/status under project scope. +- Added vector operation services and route wiring for inspect/bbox/stats/clip/buffer/intersect. +- Added raster operation scaffolding for inspect/metadata/preview, with dependency-aware clip/tile unavailability. +- Added frontend operation controls, job status display, and derived dataset link-through in dataset detail panel. +- Updated API contracts and execution log for Sprint 3 foundations. + +## Sprint 5 raster analytics hardening (2026-06-11) + +- Added raster band statistics operation: + - min, max, mean, std, nodata count, nodata ratio, valid pixel count, dtype, band index and optional histogram. + - chunked raster reads to reduce memory pressure and explicit dependency-aware unavailable mode when raster libs are missing. +- Added raster reproject operation foundation with CRS validation: + - supports target CRS selection via explicit parameter, + - persists derived output dataset with operation provenance, + - records operation parameters and error details when invalid. +- Hardened raster clip and tile manifest flow: + - explicit empty clip failure behavior, + - bounds/metadata refresh and improved tile manifest fields. +- Added raster operation job persistence tests: + - result_json and error_message persistence, + - dependency-aware statistics failure behavior, + - invalid CRS handling, + - output linkage for reprojected datasets. +- Extended dataset UI dataset detail job panel: + - raster metadata visibility (CRS, bounds, resolution), + - raster band statistics rendering, + - reproject form and job result visibility. + +## Sprint 6 local spectral indices (2026-06-12) + +- Added local spectral index operations: + - NDVI endpoint + - NDWI endpoint + - NDBI endpoint +- Added explicit spectral index input validation: + - positive integer checks + - source raster band-count bounds checks +- Implemented dependency-aware index execution for missing raster dependencies. +- Added local spectral raster output generation using float32 and `NaN` invalid handling. +- Stored index-derived dataset provenance metadata: + - `source_dataset_id` + - `operation` (`raster.ndvi`, `raster.ndwi`, `raster.ndbi`) + - `band_mapping` + - `formula` + - `output_dtype` + - `nodata_strategy` + - `value_range_note` + - `created_at` + - `output_dataset_id` + - `path` +- Extended dataset detail UI with spectral index controls and result dataset actions. +- Updated: + - `docs/API_CONTRACTS.md` + - `docs/RASTER_OPERATIONS_SPEC.md` + - `backend/README.md` + - `frontend/README.md` + - `docs/CODEX_EXECUTION_LOG.md` + +## Sprint 7A persistence and QA foundation (2026-06-12) + +- Added `vector_features` as first-class queryable vector state while preserving original uploaded files as source artifacts. +- Added `quality_checks` and `metrics` as persisted QA/QC domain records. +- Added Alembic migration `202606120700_sprint7a_persistence_foundation.py` for vector features, quality checks, metrics and required indexes. +- Persisted uploaded vector GeoJSON feature properties and geometries into PostGIS-backed feature rows. +- Updated QA candidate-vs-reference jobs to persist quality checks and metric rows and return `quality_check_id`. +- Hardened GRB/OSM provider contracts as honest `not_configured` capability stubs only. +- Added tests for Sprint 7A persistence, dataset role validation, provider contracts, migration integrity and QA route persistence. + +## Sprint 7B provider integration skeleton (2026-06-12) + +- Added central provider registry entries for `grb`, `osm`, `manual` and `fixture`. +- Added provider capability, layer, status and future import-contract endpoints using the existing API envelope style. +- Kept GRB and OSM as explicit `not_configured` providers with no live WFS, Overpass or download behavior. +- Added provider-to-dataset mapping rules for future imports through `DatasetService` and `VectorFeatureService`. +- Added lightweight frontend Provider Capabilities panel with status, authority, layers, query modes and limitations. +- Added live PostGIS migration smoke script for opt-in local database verification. +- Added Sprint 7B provider registry/API tests. + +## Sprint 8 Detection Lab foundation (2026-06-12) + +- Added `detections` as first-class persisted PostGIS records linked to project, dataset, job and analysis run. +- Hardened `analysis_runs` with dataset/job/model/result metadata for future detection and segmentation workflows. +- Added model registry capabilities for `yolo-placeholder` and `manual-fixture-detector`. +- Added Detection Lab service and API foundation with dependency-aware `DETECTION_MODEL_UNAVAILABLE` responses. +- Added explicit fixture detector mode for tests/demo fixtures only; no fake production inference was introduced. +- Added minimal frontend Detection Lab panel for model status, raster dataset selection, confidence threshold and run status. +- Added Sprint 8 tests for persistence, model capabilities, unavailable model behavior, invalid dataset validation, explicit fixture persistence and API envelope shape. + +## Sprint 8B configured YOLO foundation (2026-06-12) + +- Added optional `ai` backend dependency group for Ultralytics/Torch without making AI dependencies mandatory for normal startup. +- Added `yolo-configured` model registry capability with `not_configured`, `dependency_unavailable` and `configured` status behavior. +- Added import-safe YOLO adapter that loads only an existing local model path and does not auto-download weights. +- Added raster tile manifest validation, configured tile limits and pixel bbox to EPSG:4326 detection polygon conversion. +- Added mocked YOLO persistence tests that verify first-class detection records without requiring YOLO dependencies. +- Added Detection Lab tile manifest path input for configured YOLO runs. +- Updated AI/API/backend/frontend docs for Sprint 8B configuration and limitations. + +## Sprint 8C detection visualization and QA integration (2026-06-12) + +- Added detection result review endpoints for run lists, filtered detections, detection detail and GeoJSON FeatureCollection output. +- Added detection QA against persisted reference `vector_features` using existing `quality_checks` and `metrics`. +- Added frontend Detection Lab run selection, detection table, class/confidence filters and MapLibre detection GeoJSON overlay. +- Added frontend detection QA controls and metric summary display. +- Added tests for detection GeoJSON shape, filters, detail, QA persistence, no-match QA and Sprint 8B manifest edge cases. +- Segmentation, LiDAR, Copilot, Training Studio and Reports remain out of scope. + +## Sprint 9 Segmentation Lab foundation (2026-06-12) + +- Added `segmentations` as first-class persisted PostGIS MultiPolygon records linked to project, dataset, job and analysis run. +- Added deterministic segmentation mask path convention under `storage/masks/{project_id}/{analysis_run_id}/tile_{tile_index}/`. +- Added segmentation model registry capabilities: + - `segmentation-placeholder` + - `fixture-segmenter` + - `yolo-seg-configured` + - `sam-configured` +- Added Segmentation Lab service and API foundation for model listing, run creation, run/result listing, detail, GeoJSON output and reference QA. +- Added segmentation QA against persisted reference `vector_features` using existing `quality_checks` and `metrics`. +- Added minimal frontend Segmentation Lab UI with model status, raster selection, run/result table, map overlay and QA metric display. +- Real SAM, real YOLO-seg, model downloads, new AI dependencies, LiDAR, Copilot, Training Studio and Reports remain out of scope. + +## Sprint 10 release hardening and modularization (2026-06-13) + +- Extracted project, area, provider capabilities, Detection Lab and Segmentation Lab UI sections from `frontend/src/App.tsx` into focused components. +- Preserved existing API client usage, state ownership, MapLibre overlay behavior and workbench UX. +- Hardened readiness checks to include Alembic head verification and `scripts/live_migration_smoke.sh` syntax validation. +- No new product features, migrations, AI dependencies or live external provider fetching were introduced. + +## Sprint 11 live Docker/PostGIS runtime validation (2026-06-13) + +- Hardened `scripts/live_migration_smoke.sh` so fresh databases run Alembic migrations before checking `PostGIS_Version()`. +- Added live runtime schema-object checks for core migrated tables and geometry indexes. +- Added backend tests that lock the live migration smoke ordering and schema-check contract. +- Documented exact Docker/PostGIS validation commands, expected `DATABASE_URL` and local cleanup commands. +- Docker was unavailable in the current shell, so live container execution remains pending on a Docker-enabled machine. + +## Sprint 12 QA/QC golden dataset and benchmarking (2026-06-15) + +- Added deterministic golden building QA/QC fixtures and expected metric baseline. +- Added `scripts/run_golden_qa_benchmark.py` to run existing QA/QC logic against the golden fixtures and fail on metric drift. +- Added backend tests covering expected golden metrics and `QualityCheck`/`Metric` persistence verification. +- Documented benchmark purpose, command, expected outputs, tolerance and limitations. +- No product features, API contracts, migrations, live providers, AI models or new dependencies were introduced. + +## Sprint 13 real YOLO operational hardening (2026-06-15) + +- Added `YoloPreflightService` for local configured-YOLO readiness checks without loading models or running inference. +- Added `scripts/yolo_preflight.py` for checking enabled state, optional dependency availability, local model path, tile manifest validity, tile limit and referenced tile paths. +- Added backend tests for disabled, dependency-unavailable and ready preflight states plus CLI JSON output. +- Documented preflight usage in backend and AI pipeline docs. +- No model downloads, API contracts, migrations, new dependencies, segmentation behavior or provider fetching were introduced. + +## Sprint 14 Docker GIS runtime enablement (2026-06-16) + +- Added a backend `gis` optional dependency group for the approved raster/vector runtime stack. +- Updated the backend Docker image to install the `gis` extra plus GDAL/GEOS/PROJ system packages. +- Added `scripts/verify_gis_runtime.sh` to verify browser-facing PostGIS, Rasterio and GeoPandas capabilities through the frontend proxy. +- Added `scripts/gis_import_smoke.py` and made the backend Docker build fail if Rasterio, GeoPandas or pyogrio cannot be imported. +- Moved the Docker build-time GIS import smoke into the backend build context and kept the root script as a local wrapper. +- Included the GIS runtime script syntax check in the main readiness gate. +- Added backend and frontend Docker Compose healthchecks and made the frontend wait for a healthy backend. +- Added regression tests for Docker GIS dependency installation and capability verification script coverage. +- Documented Docker GIS runtime verification commands for local and LAN deployments. +- No API contracts, migrations, AI dependencies, provider fetching or product features were changed. + +## Sprint 15 explicit demo workflow seed (2026-06-16) + +- Added `POST /api/v1/demo/workflow` to seed or return an explicit offline demo workflow. +- The demo workflow creates a project, AOI, fixture reference building dataset, fixture candidate building dataset and persisted QA/QC metrics. +- Added `scripts/seed_demo_workflow.py` for CLI-based demo seeding. +- Added frontend "Load demo workflow" action in the Projects panel. +- Added tests for the demo endpoint envelope and fixture contract. +- No live GRB/OSM fetching, AI inference, migrations or new dependencies were introduced. + +## Sprint 16 QA/QC result visibility (2026-06-16) + +- Added `GET /api/v1/projects/{project_id}/quality-checks` to list persisted quality checks and metric rows. +- Added a frontend QA/QC Results panel for project-level persisted QA output. +- Demo workflow loading and QA actions now refresh visible QA/QC results. +- Added backend tests for quality check listing and canonical response envelopes. +- No migrations, new dependencies, live provider fetching or AI inference were introduced. + +## Sprint 17 export foundation (2026-06-16) + +- Hardened `POST /api/v1/exports/geojson` so exports persist `Export` rows instead of returning dataset ids as export ids. +- Added GeoJSON export support for vector datasets, detection runs and segmentation runs using existing persisted geometry services. +- Added project metadata JSON export, lightweight project report HTML export and export list/read/content/download endpoints. +- Added a frontend Export Center panel for creating exports, listing export records, previewing JSON artifact content and downloading artifacts. +- Added export history to project metadata/report artifacts. +- Added `scripts/verify_demo_export_workflow.sh` to smoke test demo seeding, QA/QC visibility, metadata/report/vector exports, export listing and artifact downloads through the browser-facing URL. +- Added backend tests for export persistence, artifact writing, raster rejection, HTML report creation, canonical export envelopes and raw file downloads. +- No migrations, new dependencies, live provider fetching, AI inference, LiDAR, Copilot, Training Studio or separate Reports module were introduced. diff --git a/CODEX_START.md b/CODEX_START.md new file mode 100644 index 00000000..51cadc53 --- /dev/null +++ b/CODEX_START.md @@ -0,0 +1,60 @@ +# CODEX START — Use This First + +This is the shortest possible entry point for the first implementation run. + +## Mandatory order + +1. Read `docs/00-start/START_HERE.md`. +2. Read `docs/30-codex-optimization/CODEX_RUN_CHECKLIST.md`. +3. Read `docs/30-codex-optimization/PROMPT_DISCIPLINE.md`. +4. Read `docs/20-run-readiness/RUN_READINESS_FINAL.md`. +5. Read `docs/20-run-readiness/CODEX_TOMORROW_RUNBOOK.md`. +6. Use `prompts/codex/m14/CODEX_FIRST_DAY_MASTER_PROMPT.md` as the first Codex prompt. +7. Follow `docs/20-run-readiness/PASS_SEQUENCE_FINAL.md` exactly. +8. Select the relevant skill from `skills/` for the active pass. + +## First build objective + +Build the V1 foundation vertical slice: + +Project → Area → Dataset metadata → Reference polygons → Predicted detections → QA/QC → GeoJSON export → Minimal UI. + +Do not start with heavy AI inference, LiDAR, training, MLOps, Sentinel automation, or advanced report generation before the foundation passes. + +## Pass completion rule + +A pass is not done until: + +- commands were run; +- tests/smoke checks were attempted; +- docs/status were updated; +- limitations are explicit; +- next pass is clear. + + +## M13 additions + +Before implementing, Codex must respect: + +- `docs/30-codex-optimization/CODEX_OPTIMIZATION_OVERVIEW.md` +- `docs/30-codex-optimization/TOKEN_BUDGET_POLICY.md` +- `docs/30-codex-optimization/SECRETS_AND_ENV_POLICY.md` +- `docs/30-codex-optimization/PARALLEL_AGENT_STRATEGY.md` when using multiple agents/worktrees +- `docs/30-codex-optimization/CODEX_SKILLS_INDEX.md` + +The preferred first prompt is now: + +- `prompts/codex/m14/CODEX_FIRST_DAY_MASTER_PROMPT.md` + + +## M14 launch controls + +Before the first implementation pass, Codex must read: + +- `docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md` +- `docs/40-build-launch/BUILD_SUCCESS_DEFINITION.md` +- `docs/40-build-launch/BUILD_ORDER_GRAPH.md` +- `docs/40-build-launch/CODEX_STOP_RULES.md` +- `docs/40-build-launch/MODULE_ACCEPTANCE_CRITERIA.md` + +The first implementation run is Sprint 1 only. Do not implement detection, segmentation, Sentinel, LiDAR, training, AI Copilot or advanced reports during Sprint 1. diff --git a/M10_UPDATE_MANIFEST.txt b/M10_UPDATE_MANIFEST.txt new file mode 100644 index 00000000..e8e6e54a --- /dev/null +++ b/M10_UPDATE_MANIFEST.txt @@ -0,0 +1,44 @@ +docs/18-ultra-prep/README.md +docs/18-ultra-prep/AUTONOMOUS_BUILD_CHARTER.md +docs/18-ultra-prep/BUILD_PASS_TEMPLATE.md +docs/18-ultra-prep/CRITICAL_PATH_TO_V1.md +docs/18-ultra-prep/CODEX_START_HERE.md +prompts/codex/M10_MASTER_AUTONOMOUS_PROMPT.md +prompts/codex/M10_PASS_SEQUENCE.md +docs/18-ultra-prep/FEATURE_FLAG_STRATEGY.md +docs/18-ultra-prep/ERROR_TAXONOMY.md +docs/18-ultra-prep/GEOMETRY_CONTRACTS.md +docs/18-ultra-prep/CRS_POLICY.md +docs/18-ultra-prep/SECURITY_AND_SECRET_HANDLING.md +docs/18-ultra-prep/PERFORMANCE_BUDGETS.md +docs/18-ultra-prep/OBSERVABILITY_PLAN.md +docs/18-ultra-prep/CONNECTOR_IMPLEMENTATION_GUIDE.md +docs/18-ultra-prep/MODEL_ADAPTER_GUIDE.md +docs/18-ultra-prep/QA_QC_MATCHING_ALGORITHM.md +docs/18-ultra-prep/FRONTEND_STATE_MACHINE.md +docs/18-ultra-prep/UI_COPY_BANK.md +docs/18-ultra-prep/REPO_HYGIENE_RULES.md +docs/18-ultra-prep/RELEASE_GATE_V1.md +docs/18-ultra-prep/KNOWN_LIMITATIONS_TEMPLATE.md +docs/18-ultra-prep/FINAL_PRE_CODEX_CHECKLIST.md +tickets/TICKET_INDEX.md +tickets/T-001-backend-skeleton.md +tickets/T-002-database-foundation.md +tickets/T-003-project-area-domain.md +tickets/T-010-dataset-manager.md +tickets/T-011-vector-processing.md +tickets/T-012-raster-processing.md +tickets/T-020-frontend-foundation.md +tickets/T-021-map-workbench.md +tickets/T-022-dataset-ui.md +tickets/T-030-detection-adapter.md +tickets/T-031-qaqc-engine.md +tickets/T-032-export-engine.md +tickets/T-033-demo-workflow.md +contracts/api/examples/project_create.json +contracts/api/examples/area_create.geojson +contracts/api/examples/error_feature_disabled.json +contracts/api/examples/qaqc_result.json +scripts/smoke_m10.sh +docs/TODO.md +RELEASE_NOTES/M10_ultra_preparation.md diff --git a/M11_UPDATE_MANIFEST.txt b/M11_UPDATE_MANIFEST.txt new file mode 100644 index 00000000..1dac4502 --- /dev/null +++ b/M11_UPDATE_MANIFEST.txt @@ -0,0 +1,23 @@ +M11 Architect Audit & Control Layer + +Added: +- docs/00-start/START_HERE.md +- docs/governance/GEOINTEL_CONSTITUTION.md +- docs/governance/FORBIDDEN_DECISIONS.md +- docs/governance/ARCHITECTURE_INVARIANTS.md +- docs/governance/DECISION_PRECEDENCE.md +- docs/specs/CANONICAL_DOMAIN_MODELS.md +- docs/specs/GIS_STANDARDS.md +- docs/specs/RASTER_STANDARDS.md +- docs/specs/STATE_MACHINES.md +- docs/specs/DATA_LIFECYCLE.md +- docs/specs/ERROR_CATALOG.md +- docs/specs/PERFORMANCE_BUDGETS_CANONICAL.md +- docs/workflows/GOLDEN_PATHS.md +- docs/build/BUILD_ORDER_DEPENDENCY_GRAPH.md +- docs/build/CODEX_OPERATING_SYSTEM.md +- docs/19-architect-audit/ARCHITECT_AUDIT_REPORT_M11.md +- prompts/codex/M11_ARCHITECT_MASTER_PROMPT.md + +Changed: +- README.md now points to the single canonical M11 start path. diff --git a/M12_UPDATE_MANIFEST.txt b/M12_UPDATE_MANIFEST.txt new file mode 100644 index 00000000..935f4af7 --- /dev/null +++ b/M12_UPDATE_MANIFEST.txt @@ -0,0 +1,20 @@ +M12 Final Run Readiness Layer + +Added: +- CODEX_START.md +- docs/20-run-readiness/RUN_READINESS_FINAL.md +- docs/20-run-readiness/PASS_SEQUENCE_FINAL.md +- docs/20-run-readiness/CODEX_TOMORROW_RUNBOOK.md +- docs/20-run-readiness/IMPLEMENTATION_READINESS_CHECKLIST.md +- docs/20-run-readiness/REPO_CONFLICT_RESOLUTION.md +- prompts/codex/final/DAY_1_MASTER_PROMPT.md +- prompts/codex/final/PASS_00_REPO_AUDIT_FINAL.md +- prompts/codex/final/PASS_01_BACKEND_FOUNDATION_FINAL.md +- prompts/codex/final/PASS_02_DOMAIN_DATABASE_FINAL.md +- scripts/preimplementation_audit.py +- scripts/run_readiness_check.sh +- Makefile +- RELEASE_NOTES/v0.12-m12-final-run-readiness.md + +Changed: +- README.md diff --git a/M13_UPDATE_MANIFEST.txt b/M13_UPDATE_MANIFEST.txt new file mode 100644 index 00000000..709fccc1 --- /dev/null +++ b/M13_UPDATE_MANIFEST.txt @@ -0,0 +1,25 @@ +M13 — Codex Optimization Pack + +Purpose: +- Improve Codex execution quality after M12 final run readiness. +- Add reusable skills, prompt discipline, token policy, secrets policy, parallel agent strategy and pass completion prompts. + +Added: +- docs/30-codex-optimization/CODEX_OPTIMIZATION_OVERVIEW.md +- docs/30-codex-optimization/CODEX_RUN_CHECKLIST.md +- docs/30-codex-optimization/PROMPT_DISCIPLINE.md +- docs/30-codex-optimization/TOKEN_BUDGET_POLICY.md +- docs/30-codex-optimization/SECRETS_AND_ENV_POLICY.md +- docs/30-codex-optimization/PARALLEL_AGENT_STRATEGY.md +- docs/30-codex-optimization/CODEX_SKILLS_INDEX.md +- docs/30-codex-optimization/M13_HANDOFF_SUMMARY.md +- skills/*/SKILL.md +- prompts/codex/m13/*.md +- scripts/validate_m13_codex_assets.py + +Updated: +- README.md +- CODEX_START.md +- Makefile +- scripts/run_readiness_check.sh +- CHANGELOG.md diff --git a/M14_UPDATE_MANIFEST.txt b/M14_UPDATE_MANIFEST.txt new file mode 100644 index 00000000..7d79dda1 --- /dev/null +++ b/M14_UPDATE_MANIFEST.txt @@ -0,0 +1,25 @@ +M14 Build Launch Package + +Added: +- docs/40-build-launch/BUILD_SUCCESS_DEFINITION.md +- docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md +- docs/40-build-launch/DATA_ACQUISITION_PLAYBOOK.md +- docs/40-build-launch/GOLDEN_DATASET_PACKAGE.md +- docs/40-build-launch/BUILD_ORDER_GRAPH.md +- docs/40-build-launch/MODULE_ACCEPTANCE_CRITERIA.md +- docs/40-build-launch/CODEX_STOP_RULES.md +- docs/40-build-launch/RELEASE_STRATEGY.md +- docs/40-build-launch/RISK_REGISTER.md +- docs/40-build-launch/BACKLOG_PRIORITIES_MOSCOW.md +- docs/40-build-launch/FOLDER_OWNERSHIP.md +- prompts/codex/m14/CODEX_FIRST_DAY_MASTER_PROMPT.md +- checklists/SPRINT_1_OPERATOR_CHECKLIST.md +- release/v0.1-foundation-target.md +- scripts/validate_m14_launch_assets.py + +Updated: +- README.md +- CODEX_START.md +- docs/00-start/START_HERE.md +- Makefile +- scripts/run_readiness_check.sh diff --git a/M5_UPDATE_MANIFEST.txt b/M5_UPDATE_MANIFEST.txt new file mode 100644 index 00000000..b73d01cb --- /dev/null +++ b/M5_UPDATE_MANIFEST.txt @@ -0,0 +1,29 @@ +docs/OBSERVABILITY_PLAN.md +docs/TROUBLESHOOTING_RUNBOOK.md +docs/RELEASE_PROCESS.md +docs/ROLLBACK_AND_RECOVERY.md +docs/DEPENDENCY_LOCK_PLAN.md +docs/SECURITY_CHECKLIST.md +docs/DATA_PRIVACY_AND_LICENSING.md +docs/EXTERNAL_SERVICES_ADAPTERS.md +docs/GEOSPATIAL_VALIDATION_RULES.md +docs/BUILD_GOVERNANCE.md +docs/M5_OPERATIONAL_READINESS.md +docs/CI_CD_SPECIFICATION.md +docs/HEALTHCHECK_CONTRACTS.md +docs/CODEX_PASS_0_REPO_AUDIT.md +docs/CODEX_PASS_1_BACKEND_FOUNDATION.md +docs/CODEX_PASS_2_DATABASE_AND_MODELS.md +docs/CODEX_PASS_3_DATASET_MANAGER.md +docs/CODEX_PASS_4_RASTER_VECTOR_CORE.md +docs/CODEX_PASS_5_FRONTEND_WORKBENCH_SHELL.md +docs/CODEX_PASS_6_DETECTION_QA_SKELETON.md +docs/CODEX_PROMPT_M5_LONG_AUTONOMOUS_BUILD.md +scripts/check_repo_structure.sh +scripts/smoke_backend_import.sh +scripts/smoke_docs.py +scripts/smoke_contracts.py +scripts/validate_fixtures.py +RELEASE_NOTES/v0.5-m5-operational-readiness.md +CHANGELOG.md +docs/TODO.md diff --git a/M9_UPDATE_MANIFEST.txt b/M9_UPDATE_MANIFEST.txt new file mode 100644 index 00000000..c99bb16d --- /dev/null +++ b/M9_UPDATE_MANIFEST.txt @@ -0,0 +1,23 @@ +# M9 Update Manifest + +New/changed files: + +- `docs/17-max-prep/M9_MAX_PREPARATION_PACK.md` +- `prompts/codex/M9_DAY_ONE_MASTER_PROMPT.md` +- `docs/17-max-prep/M9_AUTONOMOUS_BUILD_DOCTRINE.md` +- `docs/17-max-prep/M9_PASS_SCORECARDS.md` +- `docs/17-max-prep/M9_BUILD_BLOCKERS_AND_RECOVERY.md` +- `docs/17-max-prep/M9_REAL_VS_DEMO_DATA_POLICY.md` +- `docs/17-max-prep/M9_DATA_CONTRACTS_DETAILED.md` +- `docs/17-max-prep/M9_GEOSPATIAL_EDGE_CASES.md` +- `docs/17-max-prep/M9_UI_STATE_SPEC.md` +- `docs/17-max-prep/M9_API_VALIDATION_EXAMPLES.md` +- `docs/17-max-prep/M9_IMPLEMENTATION_REVIEW_SCRIPT.md` +- `docs/17-max-prep/M9_REGRESSION_MAP.md` +- `docs/17-max-prep/M9_GAP_TO_TASK_CONVERSION.md` +- `docs/17-max-prep/M9_MODULE_DATAFLOW_CHECKLIST.md` +- `docs/17-max-prep/M9_FINAL_PRE_CODE_CHECKLIST.md` +- `docs/17-max-prep/M9_LONG_FORM_CODEX_PROMPT_VARIANTS.md` +- `docs/IMPLEMENTATION_GAP_REPORT.md` +- `RELEASE_NOTES/v0.9-m9-max-preparation.md` +- `CHANGELOG.md` diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..236f81b7 --- /dev/null +++ b/Makefile @@ -0,0 +1,50 @@ +PYTHON_BIN := $(shell command -v python3 >/dev/null 2>&1 && echo python3 || echo python) + +.PHONY: readiness docs fixtures preflight backend-install backend-test backend-dev frontend-install frontend-typecheck frontend-build m13 m14 + +readiness: + bash scripts/run_readiness_check.sh + +backend-install: + cd backend && \ + $(PYTHON_BIN) -m pip install -e .[dev] + +backend-test: + cd backend && \ + $(PYTHON_BIN) -m pytest + +backend-dev: + cd backend && \ + $(PYTHON_BIN) -m uvicorn app.main:app --reload + +frontend-install: + cd frontend && \ + npm install + +frontend-typecheck: + cd frontend && \ + npm run typecheck + +frontend-build: + cd frontend && \ + npm run build + +docs: + $(PYTHON_BIN) scripts/smoke_docs.py + +fixtures: + $(PYTHON_BIN) scripts/validate_fixtures.py + +preflight: + bash scripts/codex_preflight.sh || true + $(PYTHON_BIN) scripts/preimplementation_audit.py + + +.PHONY: m13 +m13: + $(PYTHON_BIN) scripts/validate_m13_codex_assets.py + + +.PHONY: m14 +m14: + $(PYTHON_BIN) scripts/validate_m14_launch_assets.py diff --git a/README.md b/README.md new file mode 100644 index 00000000..cd6ec5ee --- /dev/null +++ b/README.md @@ -0,0 +1,250 @@ +# GeoIntel Kempen + +GeoIntel Kempen is a GeoAI Workbench for the Belgian Kempen. It is designed as a portfolio-grade project combining GIS, remote sensing, raster/vector processing, computer vision, QA/QC and geospatial exports. + +GeoIntel is not a generic dashboard or chatbot. The core product is: + +> data → processing → geospatial output → QA/QC → export + +## Current milestone + +**M14 — Build Launch Package** + +The canonical start point is now: + +- `CODEX_START.md` +- `docs/00-start/START_HERE.md` +- `docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md` +- `docs/40-build-launch/BUILD_SUCCESS_DEFINITION.md` +- `docs/40-build-launch/BUILD_ORDER_GRAPH.md` + +Older M0-M13 handoff files are retained as historical preparation artifacts. The M14 build launch docs, M13 optimization docs, M12 final run-readiness docs, M11 governance docs and canonical specs take precedence. + +## Core V1 vertical slice + +The first implementation target is: + +1. Project + Area creation. +2. Dataset registration/upload and metadata extraction. +3. Reference building layer loading. +4. Predicted detection layer loading/import. +5. QA/QC matching against reference polygons. +6. Metrics and false positive/false negative outputs. +7. GeoJSON export. +8. Minimal map/workbench UI. + +## Primary stack + +- Frontend: React, TypeScript, MapLibre GL, Deck.gl, Tailwind. +- Backend: FastAPI, Python. +- Database: PostgreSQL + PostGIS. +- GIS processing: GeoPandas, Shapely, Rasterio, PyProj, GDAL. +- AI: PyTorch, Ultralytics YOLO, SAM-compatible architecture. +- Jobs: Redis + RQ. +- Storage: local filesystem first, MinIO-compatible later. + +## Codex instructions + +Codex must start with: + +1. `docs/00-start/START_HERE.md` +2. `prompts/codex/M11_ARCHITECT_MASTER_PROMPT.md` + +Then follow the build order in: + +- `docs/build/BUILD_ORDER_DEPENDENCY_GRAPH.md` +- `docs/build/CODEX_OPERATING_SYSTEM.md` + +Before every implementation pass, run available preflight/smoke scripts where applicable. + +## Repo principle + +This is a documentation-driven engineering repo. The documentation is not decorative; it is the control system for autonomous implementation. + +## Fastest Day 1 command path + +```bash +make readiness +``` + +## Sprint 2 quick start + +- Update dependencies: + +```bash +python -m pip install -e backend/.[dev] +cd frontend && npm install +``` + +- Run full readiness checks (with no scope expansion): + +```bash +python -m compileall backend/app +cd backend && python -m pytest +cd ../frontend && npm run typecheck && npm run build +bash scripts/run_readiness_check.sh +``` + +- Raster workflow validation command (backend only): + +```bash +bash scripts/smoke_backend_import.sh +cd backend && python -c "from app.main import app; print(app.title)" +``` + +If `rasterio` is not installed, raster metadata endpoints return `RASTER_PROCESSING_UNAVAILABLE` and the frontend displays the +state as failed until the dependency is added. + +## Sprint 4 raster foundation + +- Raster operations now support: + - raster metadata extraction, + - raster preview generation, + - raster clip by area (with provenance on derived datasets), + - raster tile generation with manifest output. +- Raster services are dependency-aware: + - if `rasterio` is unavailable, endpoints return `RASTER_PROCESSING_UNAVAILABLE`. + - if preview dependencies (`numpy`, `pillow`) are unavailable, preview generation is unavailable with a clear error. +- Enable raster stack explicitly when needed: + +```bash +cd backend && python -m pip install -e .[dev,raster] +``` + +## Sprint 5 raster analytics hardening + +- Added raster band statistics (min/max/mean/std, nodata ratio/count, valid pixel count, dtype, optional histograms). +- Added raster reproject workflow with CRS validation and provenance persistence. +- Extended tile manifest expectations (`tile_set_id`, `tile_size`, `overlap`, `bounds`, `source_raster_id`, `tile_paths`, `tile_server`). +- Clarified raster operation availability in frontend/backend docs (`RASTER_PROCESSING_UNAVAILABLE` and invalid-CRS cases). + +- Raster workflow command set (where available): + +```bash +cd backend +python -m pip install -e .[dev,raster] +python -m pytest +cd ../frontend +npm run typecheck +npm run build +``` + +Then give Codex the prompt in: + +- `prompts/codex/final/DAY_1_MASTER_PROMPT.md` + + +## M13 Codex optimization + +For the first serious Codex build run, use: + +- `prompts/codex/m13/DAY_1_OPTIMIZED_MASTER_PROMPT.md` + +Codex should also use the relevant reusable skill under `skills/` for each implementation pass. Validate the optimization assets with: + +```bash +make m13 +``` + +The full readiness path remains: + +```bash +make readiness +``` + + +## M14 Build Launch + +For the first serious implementation run, use: + +- `docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md` +- `docs/40-build-launch/BUILD_SUCCESS_DEFINITION.md` +- `docs/40-build-launch/CODEX_STOP_RULES.md` +- `prompts/codex/m14/CODEX_FIRST_DAY_MASTER_PROMPT.md` + +Validate launch assets with: + +```bash +make m14 +``` + +Full readiness remains: + +```bash +make readiness +``` + +## Sprint 1 execution (Sprint 1 only) + +From a clean machine: + +```bash +cd backend && python -m pip install -e .[dev] +cd .. +make backend-install +make frontend-install +make readiness +``` + +Copy `.env.example` to `.env` only when you want local overrides. Docker Compose has safe defaults for the local PostGIS/backend/frontend stack and does not require a root `.env` file to exist. + +With Docker Compose, open the workbench at `http://localhost:1202`. + +The Docker frontend is served by nginx and proxies `/api` and `/health` to the backend container, so browser clients should use the frontend URL only, for example `http://192.168.10.150:1202` on a LAN host. + +Runtime containers include healthchecks for PostGIS, backend and frontend. After +startup, inspect them with: + +```bash +docker compose ps +``` + +Verify the browser-facing API proxy after rebuilding Docker images: + +```bash +bash scripts/verify_browser_runtime.sh http://localhost:1202 http://localhost:8000/health +``` + +Verify the Docker GIS runtime after rebuilding the backend image: + +```bash +bash scripts/verify_gis_runtime.sh http://localhost:1202 +``` + +On the LAN host use the published browser URL, for example: + +```bash +bash scripts/verify_gis_runtime.sh http://192.168.10.150:1202 +``` + +Load the explicit offline demo workflow: + +```bash +curl -X POST http://192.168.10.150:1202/api/v1/demo/workflow +``` + +If `/api/v1/projects` returns frontend HTML instead of a JSON envelope, rebuild +and restart the frontend container. + +Useful direct verification commands: + +```bash +python -m compileall backend/app +cd backend && python -c "from app.main import app; print(app.title)" +python -m pytest +cd ../frontend && npm run typecheck +cd ../frontend && npm run build +docker compose config +bash scripts/run_readiness_check.sh +``` + +If `make` or `docker` are unavailable in your shell, run the equivalent script entrypoints directly: + +```bash +bash scripts/backend_install.sh +bash scripts/backend_test.sh +bash scripts/frontend_install.sh +bash scripts/frontend_typecheck.sh +bash scripts/frontend_build.sh +bash scripts/run_readiness_check.sh +``` diff --git a/RELEASE_NOTES/M10_ultra_preparation.md b/RELEASE_NOTES/M10_ultra_preparation.md new file mode 100644 index 00000000..0288b6d3 --- /dev/null +++ b/RELEASE_NOTES/M10_ultra_preparation.md @@ -0,0 +1,25 @@ +# M10 Ultra Preparation + +M10 adds a stronger Codex autonomy layer: + +- autonomous build charter; +- Codex start-here guide; +- pass sequence; +- master prompt; +- geometry contracts; +- CRS policy; +- security and secret handling; +- performance budgets; +- observability plan; +- connector guide; +- model adapter guide; +- QA/QC matching algorithm; +- frontend state machine; +- UI copy bank; +- repo hygiene rules; +- V1 release gate; +- implementation tickets; +- API example payloads; +- final pre-Codex checklist. + +This milestone aims to make tomorrow's Codex build significantly more autonomous while preserving strict product boundaries. diff --git a/RELEASE_NOTES/v0.0-M2.md b/RELEASE_NOTES/v0.0-M2.md new file mode 100644 index 00000000..3d5955c1 --- /dev/null +++ b/RELEASE_NOTES/v0.0-M2.md @@ -0,0 +1,11 @@ +# Release Notes — v0.0 M2 Engineering Package + +This is not an application release. It is a repository preparation milestone for autonomous Codex development. + +## Main value + +Codex now has fewer architecture choices to invent. The repo contains decision records, contracts, engineering rules, fixtures, and build prompts. + +## Next recommended action + +Run Codex Pass 01 using `prompts/codex/PASS_01_BACKEND_FOUNDATION.md`. diff --git a/RELEASE_NOTES/v0.0-M3.md b/RELEASE_NOTES/v0.0-M3.md new file mode 100644 index 00000000..27d518b5 --- /dev/null +++ b/RELEASE_NOTES/v0.0-M3.md @@ -0,0 +1,22 @@ +# Release Notes — v0.0 M3 Implementation Readiness + +This is a documentation and repository preparation release. + +## Added + +- implementation epics +- build tickets +- migration plan +- seed data plan +- local dev runbook +- backend package map +- frontend route map +- module contracts +- job lifecycle +- Codex pass matrix +- additional Codex prompts +- known limitations + +## Purpose + +Prepare the repository for Codex-driven implementation without requiring major architecture decisions during coding. diff --git a/RELEASE_NOTES/v0.11-m11-architect-audit-control-layer.md b/RELEASE_NOTES/v0.11-m11-architect-audit-control-layer.md new file mode 100644 index 00000000..865d9c49 --- /dev/null +++ b/RELEASE_NOTES/v0.11-m11-architect-audit-control-layer.md @@ -0,0 +1,18 @@ +# v0.11 — M11 Architect Audit & Control Layer + +This release turns the GeoIntel preparation repo into a stricter architecture-controlled implementation repo. + +## Highlights + +- One canonical `START_HERE` document. +- Constitution, forbidden decisions and architecture invariants. +- Canonical domain model definitions. +- GIS/raster standards. +- State machines and data lifecycle. +- Golden paths and build dependency graph. +- Error catalog and canonical performance budgets. +- M11 Codex architect master prompt. + +## Purpose + +The goal is to reduce Codex ambiguity before implementation starts. Older handoff documents remain available, but M11 defines the precedence and operating model. diff --git a/RELEASE_NOTES/v0.12-m12-final-run-readiness.md b/RELEASE_NOTES/v0.12-m12-final-run-readiness.md new file mode 100644 index 00000000..aa4cd959 --- /dev/null +++ b/RELEASE_NOTES/v0.12-m12-final-run-readiness.md @@ -0,0 +1,21 @@ +# v0.12 — M12 Final Run Readiness Layer + +This release turns the M11 architect audit repo into a directly executable Codex preparation package. + +## Added + +- Root `CODEX_START.md` as the shortest canonical entry point. +- Final run-readiness docs under `docs/20-run-readiness/`. +- Final Day 1 Codex master prompt under `prompts/codex/final/`. +- Final pass prompts for Pass 00, Pass 01 and Pass 02. +- `scripts/preimplementation_audit.py`. +- `scripts/run_readiness_check.sh`. +- Root `Makefile` with `make readiness`. + +## Changed + +- README now points to M12 and the final run path. + +## Intent + +Reduce manual work tomorrow by giving Codex one obvious entry point, one pass sequence, one first prompt, and a simple readiness command. diff --git a/RELEASE_NOTES/v0.4-m4-autonomous-build-readiness.md b/RELEASE_NOTES/v0.4-m4-autonomous-build-readiness.md new file mode 100644 index 00000000..c3a42297 --- /dev/null +++ b/RELEASE_NOTES/v0.4-m4-autonomous-build-readiness.md @@ -0,0 +1,17 @@ +# v0.4 — M4 Autonomous Build Readiness + +This release adds the documentation and fixtures required for longer autonomous Codex implementation passes. + +## Highlights +- Clear sprint board. +- Module build contracts. +- Acceptance tests. +- Service IO contracts. +- UI route/state contracts. +- Job lifecycle contract. +- Demo model registry seed. +- Geel demo fixtures. +- Codex prompts per pass. + +## Next +M5 should add concrete migration SQL, OpenAPI draft, component prop contracts and test skeletons. diff --git a/RELEASE_NOTES/v0.5-m5-operational-readiness.md b/RELEASE_NOTES/v0.5-m5-operational-readiness.md new file mode 100644 index 00000000..403d9e02 --- /dev/null +++ b/RELEASE_NOTES/v0.5-m5-operational-readiness.md @@ -0,0 +1,22 @@ +# GeoIntel v0.5 — M5 Operational Readiness + +## Toegevoegd +- Operational readiness documentatie. +- CI/CD-specificatie. +- Healthcheck-contracten. +- Observability plan. +- Troubleshooting runbook. +- Releaseproces. +- Rollback- en recoveryregels. +- Dependency lock plan. +- Security checklist. +- Data privacy en licensing notities. +- External services adaptercontracten. +- Geospatial validation rules. +- Build governance. +- Codex passdocumenten voor Pass 0 tot Pass 6. +- Long autonomous Codex build prompt. +- Smoke scripts voor repo/docs/contracts/backend import. + +## Volgende logische stap +M6 kan zich richten op echte code-scaffolding: backend app, database migrations, API schemas, frontend shell en eerste project/dataset flows. diff --git a/RELEASE_NOTES/v0.9-m9-max-preparation.md b/RELEASE_NOTES/v0.9-m9-max-preparation.md new file mode 100644 index 00000000..09c4e230 --- /dev/null +++ b/RELEASE_NOTES/v0.9-m9-max-preparation.md @@ -0,0 +1,27 @@ +# GeoIntel v0.9 — M9 Max Preparation + +This release adds a heavy preparation layer intended to maximize Codex autonomy before implementation. + +## Added + +- M9 max preparation pack. +- Day-one Codex master prompt. +- Autonomous build doctrine. +- Build pass scorecards. +- Build blocker and recovery guide. +- Real vs demo data policy. +- Detailed data contracts. +- Geospatial edge case catalog. +- UI state specification. +- API validation examples. +- Implementation review script. +- Regression map. +- Gap-to-task conversion rules. +- Module dataflow checklist. +- Final pre-code checklist. +- Long-form Codex prompt variants. +- Implementation gap report template. + +## Purpose + +Make the repository as ready as possible for a long autonomous Codex build session. diff --git a/adr/ADR-001-technology-stack.md b/adr/ADR-001-technology-stack.md new file mode 100644 index 00000000..7cded6c2 --- /dev/null +++ b/adr/ADR-001-technology-stack.md @@ -0,0 +1,25 @@ +# ADR-001 — Technology Stack + +## Status +Accepted for V1. + +## Context +GeoIntel Kempen must demonstrate modern web development, geospatial processing, and GeoAI engineering. The stack must be realistic for a portfolio project while remaining close to professional workflows. + +## Decision +Use: + +- Frontend: React + TypeScript. +- Map UI: MapLibre GL with Deck.gl where advanced overlays are useful. +- Backend: FastAPI. +- Database: PostgreSQL + PostGIS. +- Processing: GeoPandas, Shapely, Rasterio, PyProj, GDAL-compatible tools. +- AI: PyTorch with Ultralytics YOLO first; SAM/segmentation later. +- Jobs: Redis + RQ for V1. +- Storage: local filesystem with explicit storage abstraction. + +## Consequences +This stack keeps the first build achievable while matching the vacancy profile closely: Python, raster/vector processing, computer vision, AI pipelines, and GIS outputs. + +## Non-goals +Do not introduce Django, Flask, MongoDB, Firebase, or a second frontend framework unless a future ADR explicitly replaces this decision. diff --git a/adr/ADR-002-postgis-choice.md b/adr/ADR-002-postgis-choice.md new file mode 100644 index 00000000..ffe60450 --- /dev/null +++ b/adr/ADR-002-postgis-choice.md @@ -0,0 +1,27 @@ +# ADR-002 — PostGIS as Spatial Source of Truth + +## Status +Accepted for V1. + +## Context +GeoIntel stores areas, datasets, AI detections, segmentations, QA geometries, and exports. Spatial operations need to be queryable and persistent. + +## Decision +Use PostgreSQL with PostGIS as the canonical database for: + +- project areas, +- dataset spatial bounds, +- vector features, +- detection polygons/boxes, +- segmentation polygons, +- QA/QC geometries, +- spatial metadata, +- analysis outputs. + +Raw rasters, tiles, masks, and large binary artifacts stay on disk/object storage. PostGIS stores metadata and vectorized results. + +## Consequences +The backend can do spatial filtering, intersections, bounding-box queries, and QA matching without reloading every file. The portfolio visibly demonstrates professional GIS database skills. + +## Non-goals +Do not store full large rasters as database blobs in V1. diff --git a/adr/ADR-003-grb-strategy.md b/adr/ADR-003-grb-strategy.md new file mode 100644 index 00000000..522874b8 --- /dev/null +++ b/adr/ADR-003-grb-strategy.md @@ -0,0 +1,23 @@ +# ADR-003 — GRB as Authoritative Reference Dataset + +## Status +Accepted for V1 research and implementation planning. + +## Context +The Basiskaart Vlaanderen / GRB is a professional Flemish geospatial reference dataset. GeoIntel is scoped to the Kempen, so Flemish official data is highly relevant. + +## Decision +Treat GRB as the primary QA/QC reference where available. Use it for building/reference geometry validation and later for roads, water, and other base-map objects. + +V1 integration strategy: + +1. Implement a GRB provider abstraction. +2. Start with WFS or downloaded sample/cache depending on practical availability. +3. Normalize GRB features into a common `reference_features` model. +4. Compare AI detections against GRB with IoU/overlap metrics. + +## Consequences +GeoIntel becomes more relevant to real Flemish GeoAI workflows than a generic OSM-only demo. GRB validation becomes a portfolio killer feature. + +## Non-goals +Do not block the entire build on live GRB integration. Provide fixtures and provider interfaces first, then connect real GRB when endpoint details are tested. diff --git a/adr/ADR-004-storage-strategy.md b/adr/ADR-004-storage-strategy.md new file mode 100644 index 00000000..022b2ba5 --- /dev/null +++ b/adr/ADR-004-storage-strategy.md @@ -0,0 +1,27 @@ +# ADR-004 — Storage Strategy + +## Status +Accepted for V1. + +## Context +GeoIntel stores multiple artifact types: uploaded rasters, vector uploads, generated tiles, model outputs, masks, exports, and reports. + +## Decision +Use local filesystem storage for V1 with a strict directory convention: + +- `storage/uploads/` for original user uploads, +- `storage/originals/` for normalized source copies, +- `storage/tiles/` for generated raster tiles, +- `storage/masks/` for segmentation masks, +- `storage/derived/` for processed artifacts, +- `storage/exports/` for GeoJSON/COCO/YOLO exports, +- `storage/reports/` for reports, +- `storage/models/` for model artifacts. + +Database rows reference files by relative path and content hash. + +## Consequences +Simple local development and predictable repo behavior. Future MinIO/S3 migration remains possible because storage calls must go through a service boundary. + +## Non-goals +No direct random file writes from routes or frontend-specific paths. diff --git a/adr/ADR-005-ai-model-strategy.md b/adr/ADR-005-ai-model-strategy.md new file mode 100644 index 00000000..3801c160 --- /dev/null +++ b/adr/ADR-005-ai-model-strategy.md @@ -0,0 +1,18 @@ +# ADR-005 — AI Model Strategy + +## Status +Accepted for V1. + +## Context +The vacancy emphasizes PyTorch, object detection, segmentation, and GeoAI. A portfolio build should show a real inference pipeline, not only AI text generation. + +## Decision +Use Ultralytics YOLO as the first object detection runtime because it is practical, PyTorch-based, well documented, and fast to integrate. Add segmentation through YOLO-seg or SAM after the detection pipeline is reliable. + +Model execution must be wrapped behind `ModelRegistryService` and `DetectionService` interfaces so the UI and API do not depend directly on Ultralytics internals. + +## Consequences +GeoIntel can demonstrate model inference, georeferencing, output conversion, confidence thresholds, and QA/QC against GRB. + +## Non-goals +Do not train a custom model in V1. Fine-tuning becomes V2/V3 after annotation and dataset export exist. diff --git a/adr/ADR-006-job-processing.md b/adr/ADR-006-job-processing.md new file mode 100644 index 00000000..3a07b4bc --- /dev/null +++ b/adr/ADR-006-job-processing.md @@ -0,0 +1,25 @@ +# ADR-006 — Job Processing + +## Status +Accepted for V1. + +## Context +Raster tiling, detection, segmentation, QA, and exports can take longer than a normal HTTP request. + +## Decision +Use Redis + RQ for V1 background jobs. Every long-running operation creates an `analysis_run` or `job` record, updates status, stores outputs, and emits events. + +Supported statuses: + +- pending, +- queued, +- running, +- completed, +- failed, +- cancelled. + +## Consequences +The UI can show progress and status without blocking. RQ is easier than Celery for an initial solo/portfolio project. + +## Non-goals +No Kubernetes-native queues, no Airflow, no full workflow engine in V1. diff --git a/adr/ADR-007-api-design.md b/adr/ADR-007-api-design.md new file mode 100644 index 00000000..c125e50e --- /dev/null +++ b/adr/ADR-007-api-design.md @@ -0,0 +1,28 @@ +# ADR-007 — API Design + +## Status +Accepted for V1. + +## Context +The frontend must be API-driven and Codex must not invent inconsistent response shapes. + +## Decision +Use REST-style FastAPI endpoints with typed Pydantic schemas. Responses use stable envelopes for long-running jobs and direct resources for simple CRUD operations. + +Errors use a common structure: + +```json +{ + "error": { + "code": "DATASET_NOT_FOUND", + "message": "Dataset not found.", + "details": {} + } +} +``` + +## Consequences +Frontend API clients, tests, and docs stay consistent. + +## Non-goals +No GraphQL in V1. diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 00000000..8cd3b7a8 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,10 @@ +__pycache__ +*.pyc +.pytest_cache +.mypy_cache +.ruff_cache +geointel_backend.egg-info +storage +dist +node_modules +.env diff --git a/backend/.gitkeep b/backend/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 00000000..effc85f4 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + gdal-bin \ + libgdal-dev \ + libgeos-dev \ + libproj-dev \ + libpq-dev \ + proj-bin \ + && rm -rf /var/lib/apt/lists/* + +COPY pyproject.toml README.md /app/ +COPY app /app/app +RUN pip install --no-cache-dir --upgrade pip setuptools +RUN pip install --no-cache-dir ".[gis]" + +COPY . /app +RUN python scripts/gis_import_smoke.py + +EXPOSE 8000 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 00000000..9fe412c4 --- /dev/null +++ b/backend/README.md @@ -0,0 +1,537 @@ +# GeoIntel Backend (Sprint 3 foundation layer) + +FastAPI backend for GeoIntel Kempen Foundation Sprints. + +## Scope implemented +- Project CRUD +- Area CRUD with PostGIS geometry +- Vector and raster dataset upload/registration +- Deterministic local storage metadata capture +- PostGIS migration and database foundation +- Job foundation for async-ready GIS operations + +## Sprint 2 additions +- Dataset typing and lifecycle support: + - `uploaded` + - `validating` + - `ready` + - `failed` +- Vector metadata extraction: + - feature count + - geometry type summary + - bounds + - approximate area + - CRS and CRS assumption +- Raster metadata endpoint: + - returns raster profile when `rasterio` is available + - returns clear `RASTER_PROCESSING_UNAVAILABLE` error when dependency is missing +- Deterministic storage metadata capture: + - original filename + - stored filename + - MIME/content type + - size bytes + - checksum SHA-256 + +## Sprint 3 additions +- Lightweight job architecture: + - `jobs` table and migrations + - job create/list/read/status API + - synchronous execution behind job abstraction +- Vector operations foundation: + - inspect + - bbox + - stats + - clip by area + - buffer + - intersect + - invalid geometry rejection with typed errors +- Raster operation foundation: + - inspect + - metadata + - preview readiness + - clip by area (dependency-aware with unavailable fallback) + - tile generation with manifest output + - real preview image generation when dependencies are installed + +## Sprint 4 additions +- Raster foundation is now implemented with real extraction and deterministic artifact outputs: + - metadata returns width, height, band count, CRS, bounds, resolution, dtype, nodata, transform + - preview endpoint generates and reuses PNG previews with width/height + - clip operation persists a derived raster dataset with: + - `source_dataset_id` + - `operation` + - `operation_parameters` + - tile operation writes deterministic raster tiles under `tiles/{project_id}/{source_dataset_id}/{tile_set_id}` + - tile manifest includes tile path, pixel window, bounds, transform, and count +- Dependency behavior: + - when `rasterio` is missing, raster processing returns `RASTER_PROCESSING_UNAVAILABLE` + - preview endpoint additionally requires numpy/pillow and returns `RASTER_PROCESSING_UNAVAILABLE` when missing + +## Sprint 5 additions +- Raster analytics hardening: + - raster band statistics now include: + - min, max, mean, std + - nodata count and ratio + - valid pixel count + - dtype + - optional histogram bins (default 16 bins) + - raster reproject operation implemented (CRS transform + rasterio reprojection) using dependency-aware raster processing checks. + - reproject failures are explicit (`INVALID_PARAMETERS`, `INVALID_DATASET_CRS`, `RASTER_PROCESSING_UNAVAILABLE`). +- Raster clip and tile hardening: + - clip validates area presence and CRS alignment constraints. + - tile manifest records `tile_set_id`, `tile_size`, `overlap`, `source_dataset_id`, `source_raster_id`, bounds, parameters, count, tile paths, `ai_inference`, and `tile_server`. +- Job result persistence for raster ops: + - raster clip/reproject/tile job payloads persist derived dataset references when outputs are produced. + +## Sprint 6 additions +- Added local spectral index operations: + - NDVI endpoint: `POST /raster/indices/ndvi` + - NDWI endpoint: `POST /raster/indices/ndwi` + - NDBI endpoint: `POST /raster/indices/ndbi` +- Spectral index input validation: + - band parameters must be positive integers + - band parameters must exist in source raster band count +- Dependency-aware execution: + - returns `RASTER_PROCESSING_UNAVAILABLE` when rasterio or numpy are unavailable +- Real index output handling: + - local windowed float32 GeoTIFF generation + - `NaN` strategy for invalid pixels / division by zero +- Provenance capture for derived index datasets: + - `source_dataset_id`, `operation`, `band_mapping`, `formula` + - `output_dtype`, `nodata_strategy`, `value_range_note` + - `output_dataset_id`, `created_at`, `path` + +## Sprint 7B additions +- Added provider registry skeleton for `grb`, `osm`, `manual` and `fixture`. +- Added provider capability endpoints: + - `GET /api/v1/external/providers` + - `GET /api/v1/external/providers/{provider_name}` + - `GET /api/v1/external/providers/{provider_name}/layers` + - `GET /api/v1/external/providers/{provider_name}/status` + - `POST /api/v1/external/providers/{provider_name}/import` +- GRB and OSM imports return explicit `not_configured` responses; no live WFS or Overpass calls are made. +- Manual and fixture providers describe existing upload/fixture flows only. +- Added live PostGIS migration smoke script for environments with a real database: + +```bash +bash scripts/live_migration_smoke.sh +``` + +## Sprint 8 additions +- Added Detection Lab foundation: + - `detections` ORM model and Alembic migration with PostGIS geometry storage. + - hardened `analysis_runs` for dataset/job/model/result metadata. + - model registry capability service for `yolo-placeholder` and `manual-fixture-detector`. + - detection service boundary for creating jobs, analysis runs and dependency-aware unavailable responses. +- Added detection endpoints: + - `GET /api/v1/detection/models` + - `POST /api/v1/detection/run` + - `GET /api/v1/detection/runs/{analysis_run_id}` + - `GET /api/v1/detection/runs/{analysis_run_id}/detections` +- YOLO/PyTorch real inference is not enabled in Sprint 8. +- Fixture detector mode is test/demo-only and requires explicit `fixture_mode=true`. + +## Sprint 8B additions +- Added optional configured YOLO integration foundation: + - `yolo-configured` model registry capability. + - import-safe adapter for local Ultralytics model files. + - raster tile manifest validation and tile limit enforcement. + - pixel bbox to EPSG:4326 detection polygon conversion. + - persisted detections through the existing detection/job/analysis-run path. +- YOLO dependencies are optional extras and are not required for backend startup. +- GeoIntel does not download YOLO model weights automatically. + +## Sprint 8C additions +- Added detection visualization/review API support: + - list detection runs + - list detections by run or dataset with class/confidence filters + - get detection detail + - return persisted detections as GeoJSON FeatureCollections +- Added detection QA against reference vector datasets: + - compares persisted detection geometries against persisted `vector_features` + - persists `quality_checks` and `metrics` + - returns precision, recall, F1, mean IoU and false positive/negative counts +- Segmentation, LiDAR, AI Copilot, Training Studio and Reports remain out of scope. + +## Sprint 9 additions +- Added Segmentation Lab foundation: + - `segmentations` ORM model and Alembic migration with PostGIS MultiPolygon geometry storage. + - segmentation model registry capabilities for `segmentation-placeholder`, `fixture-segmenter`, `yolo-seg-configured` and `sam-configured`. + - segmentation service boundary for creating jobs, analysis runs and unavailable model responses. + - explicit fixture segmenter mode for tests/demo fixtures only. +- Added segmentation endpoints: + - `GET /api/v1/segmentation/models` + - `POST /api/v1/segmentation/run` + - `GET /api/v1/segmentation/runs` + - `GET /api/v1/segmentation/runs/{analysis_run_id}` + - `GET /api/v1/segmentation/runs/{analysis_run_id}/segmentations` + - `GET /api/v1/segmentation/runs/{analysis_run_id}/geojson` + - `POST /api/v1/segmentation/runs/{analysis_run_id}/qa/reference` +- Real SAM and YOLO-seg inference are not enabled in Sprint 9. +- Mask paths are provenance/debug artifacts; persisted PostGIS geometry is authoritative for QA, map display and GeoJSON. + +## Sprint 17 additions +- Added export foundation backed by the existing `exports` table. +- GeoJSON exports now persist export records and write JSON artifacts for: + - vector datasets + - detection analysis runs + - segmentation analysis runs +- Added project metadata JSON export for project, dataset and QA/QC summary state. +- Added export read/list/content endpoints: + - `POST /api/v1/exports/geojson` + - `POST /api/v1/exports/metadata` + - `GET /api/v1/exports/projects/{project_id}/exports` + - `GET /api/v1/exports/{export_id}` + - `GET /api/v1/exports/{export_id}/content` +- Exported detection and segmentation GeoJSON is generated from persisted first-class geometry rows. +- No new migrations, product lines, live providers or AI dependencies are introduced by this export pass. + +## Run locally + +### Prerequisites + +- Python 3.11+ +- PostgreSQL with PostGIS + +### Install dependencies + +```bash +cd backend +python -m pip install -e .[dev] +``` + +Optional AI dependencies for configured local YOLO inference: + +```bash +cd backend +python -m pip install -e .[ai] +``` + +Configured YOLO requires: + +```bash +YOLO_ENABLED=true +YOLO_MODEL_PATH=/absolute/path/to/local-model.pt +``` + +Optional tuning: + +```bash +YOLO_MODEL_ID=yolo-configured +YOLO_MODEL_DISPLAY_NAME="Configured YOLO detector" +YOLO_MODEL_VERSION=local-v1 +YOLO_DEVICE=cpu +YOLO_IMAGE_SIZE=640 +YOLO_MAX_TILES=100 +YOLO_BATCH_SIZE=1 +``` + +### YOLO local preflight + +Sprint 13 adds a local-only preflight for configured YOLO paths: + +```bash +python scripts/yolo_preflight.py --model-path /absolute/path/to/local-model.pt --tile-manifest-path /absolute/path/to/manifest.json +``` + +Machine-readable output: + +```bash +python scripts/yolo_preflight.py --model-path /absolute/path/to/local-model.pt --tile-manifest-path /absolute/path/to/manifest.json --json +``` + +To validate only local model/manifest paths on a machine without optional AI dependencies: + +```bash +python scripts/yolo_preflight.py --model-path /absolute/path/to/local-model.pt --tile-manifest-path /absolute/path/to/manifest.json --assume-dependencies --json +``` + +The preflight checks configuration, dependency availability, local model file existence, tile manifest validity, tile count and referenced tile paths. It does not load a YOLO model, run inference or download weights. + +### Run backend + +```bash +cd backend +python -m uvicorn app.main:app --reload +``` + +### Run backend tests + +```bash +cd backend +python -m pytest +``` + +For warning-sensitive release checks, the backend is expected to pass with Python deprecation warnings promoted to errors for the timestamp-heavy service paths: + +```bash +cd backend +python -m pytest -W error::DeprecationWarning tests/test_geojson_dataset_service.py tests/test_qa_service.py tests/test_sprint7a_persistence_foundation.py tests/test_sprint8c_detection_visualization_qa.py tests/test_sprint9_segmentation_foundation.py tests/test_vector_operations_service.py +``` + +The repository readiness gate now applies the same warning policy to the full backend suite: + +```bash +bash scripts/run_readiness_check.sh +``` + +That readiness gate also runs the API contract smoke check before backend/frontend compilation and tests. + +### Golden QA/QC benchmark + +Sprint 12 includes a deterministic QA/QC regression benchmark using explicit fixture data: + +```bash +python scripts/run_golden_qa_benchmark.py +``` + +Machine-readable output: + +```bash +python scripts/run_golden_qa_benchmark.py --json +``` + +The benchmark compares `fixtures/golden/predicted_buildings.geojson` against `fixtures/golden/reference_buildings.geojson` and fails on metric drift. Expected baseline: + +- precision: `0.5` +- recall: `0.5` +- F1: `0.5` +- mean IoU: `0.8339768339761133` +- false positives: `1` +- false negatives: `1` + +The command uses existing QA/QC service logic and verifies `QualityCheck`/`Metric` persistence through an in-memory test session. It does not require live providers, AI models, Docker or PostGIS. + +### Demo workflow seed + +Sprint 15 adds an explicit offline demo workflow seed. It creates or returns a +demo project, AOI, fixture reference buildings, fixture candidate buildings and +a persisted QA/QC result. It does not fetch live GRB/OSM data and does not run +AI inference. + +API: + +```bash +curl -X POST http://localhost:1202/api/v1/demo/workflow +``` + +CLI: + +```bash +python scripts/seed_demo_workflow.py --json +``` + +In Docker Compose on a LAN host: + +```bash +curl -X POST http://192.168.10.150:1202/api/v1/demo/workflow +``` + +### QA/QC result listing + +Persisted project quality checks and metric rows can be listed with: + +```bash +curl http://localhost:1202/api/v1/projects/{project_id}/quality-checks +``` + +The frontend QA/QC Results panel uses this endpoint after loading the demo +workflow or running QA. + +### Export foundation + +Persisted exports can be created from the existing workbench state: + +```bash +curl -X POST http://localhost:1202/api/v1/exports/metadata \ + -H "Content-Type: application/json" \ + -d '{"project_id":"PROJECT_UUID"}' +``` + +Vector dataset GeoJSON export: + +```bash +curl -X POST http://localhost:1202/api/v1/exports/geojson \ + -H "Content-Type: application/json" \ + -d '{"export_kind":"dataset","dataset_id":"DATASET_UUID"}' +``` + +Detection or segmentation run GeoJSON export: + +```bash +curl -X POST http://localhost:1202/api/v1/exports/geojson \ + -H "Content-Type: application/json" \ + -d '{"export_kind":"detection_run","analysis_run_id":"ANALYSIS_RUN_UUID"}' +``` + +List and inspect exports: + +```bash +curl http://localhost:1202/api/v1/exports/projects/PROJECT_UUID/exports +curl http://localhost:1202/api/v1/exports/EXPORT_UUID/content +``` + +Download an artifact as a browser/file response: + +```bash +curl -OJ http://localhost:1202/api/v1/exports/EXPORT_UUID/download +``` + +Create a lightweight HTML project report artifact: + +```bash +curl -X POST http://localhost:1202/api/v1/exports/report \ + -H "Content-Type: application/json" \ + -d '{"project_id":"PROJECT_UUID"}' +``` + +The report contains project, dataset, QA/QC summary and export history state +only. It is not a PDF designer and does not add a separate reporting module. + +After rebuilding a Docker/LAN deployment, verify the end-to-end demo and export +flow through the browser-facing frontend proxy: + +```bash +bash scripts/verify_demo_export_workflow.sh http://192.168.10.150:1202 +``` + +The script seeds the explicit demo workflow, verifies persisted QA/QC results, +creates metadata/report/vector GeoJSON exports, lists exports and downloads the +JSON/GeoJSON/HTML artifacts. + +### Backend import smoke + +```bash +cd backend +python -c "from app.main import app; print(app.title)" +``` + +### Dockerized backend + +```bash +docker compose up --build backend db +``` + +The Docker Compose stack does not require a root `.env` file for the default local runtime. The database service exposes a container-internal Postgres healthcheck, and the backend also runs `docker_start.sh`, which retries an actual SQL `SELECT 1` connection before running `python -m alembic upgrade head` and starting Uvicorn. + +PostGIS is not published on the host `5432` port by default. This avoids conflicts with existing Postgres/PostGIS services on NAS or server hosts. The backend connects over Docker networking with `db:5432`. + +Backend and frontend Docker build contexts exclude dependency folders, build outputs and Python bytecode caches via `.dockerignore`. + +The Docker Compose frontend is published at `http://localhost:1202`. + +Compose healthchecks are enabled for all runtime services: + +- `db` uses `pg_isready`. +- `backend` checks `http://127.0.0.1:8000/health` inside the container. +- `frontend` checks `http://127.0.0.1/health` through nginx, which also verifies the frontend-to-backend proxy path. + +The frontend waits for a healthy backend before starting. Check runtime state: + +```bash +docker compose ps +docker compose logs --tail=80 backend +docker compose logs --tail=80 frontend +``` + +The backend Docker image installs the approved GIS runtime extra (`.[gis]`) so +browser-facing Docker deployments can report raster/vector processing +capabilities accurately: + +- `rasterio` +- `numpy` +- `pillow` +- `geopandas` +- `pyogrio` +- GDAL/GEOS/PROJ system libraries + +After rebuilding the backend image, verify the LAN/browser runtime from the +repository root: + +```bash +bash scripts/verify_gis_runtime.sh http://localhost:1202 +``` + +On a NAS or server host, use the published LAN URL: + +```bash +bash scripts/verify_gis_runtime.sh http://192.168.10.150:1202 +``` + +The script calls `/api/v1/system/capabilities` through the frontend proxy and +fails if `postgis`, `rasterio` or `geopandas` are not reported as available. + +The backend Docker build also runs: + +```bash +python scripts/gis_import_smoke.py +``` + +Inside the backend Docker build context this resolves to +`backend/scripts/gis_import_smoke.py`. The root `scripts/gis_import_smoke.py` +wrapper calls the same smoke locally. The smoke imports `rasterio`, `geopandas` +and `pyogrio`; if one of those imports fails, the backend image build fails +before deployment. + +### Live Docker/PostGIS migration smoke + +Sprint 11 validates the real PostGIS runtime path with the existing database service. From the repository root: + +```bash +docker compose config +docker compose up -d db +DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel bash scripts/live_migration_smoke.sh +``` + +The smoke script: + +- opens a backend SQLAlchemy connection and runs `SELECT 1` +- runs `alembic upgrade head` +- checks `PostGIS_Version()` after migrations have created the extension +- verifies one Alembic head +- verifies required migrated tables and GiST indexes exist + +Expected local environment: + +```bash +DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel +``` + +If the database is not reachable, confirm Docker Desktop is running and that port `5432` is not already occupied. To clean up the local database container without deleting the named volume: + +```bash +docker compose stop db +``` + +To remove the local PostGIS volume as well, use only when you explicitly want a fresh database: + +```bash +docker compose down -v +``` + +## Key docs +- `docs/API_CONTRACTS.md` +- `docs/DATABASE_IMPLEMENTATION_PLAN.md` +- `docs/DEFINITION_OF_DONE.md` +- `docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md` + +## Raster dependency note + +Raster metadata and raster operations depend on local GDAL/rasterio availability. + +To enable raster processing locally: + +```bash +python -m pip install rasterio +``` + +If `rasterio` is unavailable: +- raster metadata responses return `503` with `RASTER_PROCESSING_UNAVAILABLE` +- raster clip/tile endpoints return explicit unavailable responses + +## Helpful repository scripts + +- `bash scripts/backend_install.sh` +- `bash scripts/backend_test.sh` +- `bash scripts/backend_dev.sh` +- `bash scripts/smoke_backend_import.sh` diff --git a/backend/alembic.ini b/backend/alembic.ini new file mode 100644 index 00000000..92c1ff36 --- /dev/null +++ b/backend/alembic.ini @@ -0,0 +1,38 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +sqlalchemy.url = postgresql+psycopg://geointel:geointel@localhost:5432/geointel + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = INFO +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +class_ = logging.Formatter diff --git a/backend/alembic/env.py b/backend/alembic/env.py new file mode 100644 index 00000000..1ae0d3e0 --- /dev/null +++ b/backend/alembic/env.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import os +import sys +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) + +from app.core.config import get_settings +from app.db.base import Base +import app.models.entities # noqa: F401 + +settings = get_settings() +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +config.set_main_option("sqlalchemy.url", settings.database_url) + +target_metadata = Base.metadata + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url, target_metadata=target_metadata, literal_binds=True) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/alembic/script.py.mako b/backend/alembic/script.py.mako new file mode 100644 index 00000000..030095e7 --- /dev/null +++ b/backend/alembic/script.py.mako @@ -0,0 +1,20 @@ +""" +${message} +""" +from alembic import op +import sqlalchemy as sa + +${imports} + +revision = ${repr(revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/backend/alembic/versions/202601110001_initial.py b/backend/alembic/versions/202601110001_initial.py new file mode 100644 index 00000000..a4254dc8 --- /dev/null +++ b/backend/alembic/versions/202601110001_initial.py @@ -0,0 +1,108 @@ +"""Initial PostGIS schema for Sprint 1 foundation.""" + +from alembic import op +import sqlalchemy as sa +from geoalchemy2 import Geometry + +revision = "202601110001" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("CREATE EXTENSION IF NOT EXISTS postgis") + op.execute("CREATE EXTENSION IF NOT EXISTS postgis_topology") + op.execute('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"') + + op.create_table( + "projects", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("region", sa.Text(), nullable=False, server_default="Kempen"), + sa.Column("status", sa.Text(), nullable=False, server_default="active"), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()")), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()")), + ) + + op.create_table( + "areas", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("project_id", sa.UUID(as_uuid=True), sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("geometry", Geometry("MULTIPOLYGON", srid=4326), nullable=False), + sa.Column("original_crs", sa.Text(), nullable=True), + sa.Column("area_m2", sa.Float(), nullable=True), + sa.Column("bbox", Geometry("POLYGON", srid=4326), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()")), + ) + + op.create_table( + "datasets", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("project_id", sa.UUID(as_uuid=True), sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=False), + sa.Column("area_id", sa.UUID(as_uuid=True), sa.ForeignKey("areas.id", ondelete="SET NULL"), nullable=True), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("dataset_type", sa.Text(), nullable=False), + sa.Column("source", sa.Text(), nullable=False), + sa.Column("storage_path", sa.Text(), nullable=True), + sa.Column("derived_from_dataset_id", sa.UUID(as_uuid=True), sa.ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True), + sa.Column("crs", sa.Text(), nullable=True), + sa.Column("bounds_json", sa.JSON(), nullable=True), + sa.Column("resolution_json", sa.JSON(), nullable=True), + sa.Column("bands_json", sa.JSON(), nullable=True), + sa.Column("metadata_json", sa.JSON(), nullable=True), + sa.Column("status", sa.Text(), nullable=False, server_default="created"), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()")), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()")), + ) + + op.create_table( + "dataset_versions", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("dataset_id", sa.UUID(as_uuid=True), sa.ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False), + sa.Column("version", sa.Integer(), nullable=False, server_default="1"), + sa.Column("storage_path", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()")), + ) + + op.create_table( + "analysis_runs", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("project_id", sa.UUID(as_uuid=True), sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=False), + sa.Column("area_id", sa.UUID(as_uuid=True), sa.ForeignKey("areas.id", ondelete="SET NULL"), nullable=True), + sa.Column("analysis_type", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("parameters_json", sa.JSON(), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + ) + + op.create_table( + "exports", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("project_id", sa.UUID(as_uuid=True), sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=False), + sa.Column("analysis_run_id", sa.UUID(as_uuid=True), sa.ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True), + sa.Column("export_type", sa.Text(), nullable=False), + sa.Column("storage_path", sa.Text(), nullable=False), + sa.Column("metadata_json", sa.JSON(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()")), + ) + + op.create_index("ix_areas_geometry", "areas", ["geometry"], postgresql_using="gist") + op.create_index("ix_areas_project_id", "areas", ["project_id"]) + op.create_index("ix_datasets_project_id", "datasets", ["project_id"]) + + +def downgrade() -> None: + op.drop_index("ix_datasets_project_id", table_name="datasets") + op.drop_index("ix_areas_project_id", table_name="areas") + op.drop_index("ix_areas_geometry", table_name="areas", postgresql_using="gist") + op.drop_table("exports") + op.drop_table("analysis_runs") + op.drop_table("dataset_versions") + op.drop_table("datasets") + op.drop_table("areas") + op.drop_table("projects") diff --git a/backend/alembic/versions/202601120001_dataset_storage_metadata.py b/backend/alembic/versions/202601120001_dataset_storage_metadata.py new file mode 100644 index 00000000..c7795a3b --- /dev/null +++ b/backend/alembic/versions/202601120001_dataset_storage_metadata.py @@ -0,0 +1,27 @@ +"""Add dataset storage metadata columns.""" + +from alembic import op +import sqlalchemy as sa + + +revision = "202601120001" +down_revision = "202601110001" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("datasets", sa.Column("original_filename", sa.Text(), nullable=True)) + op.add_column("datasets", sa.Column("stored_filename", sa.Text(), nullable=True)) + op.add_column("datasets", sa.Column("content_type", sa.Text(), nullable=True)) + op.add_column("datasets", sa.Column("size_bytes", sa.Integer(), nullable=True)) + op.add_column("datasets", sa.Column("checksum_sha256", sa.Text(), nullable=True)) + op.alter_column("datasets", "status", server_default="uploaded") + + +def downgrade() -> None: + op.drop_column("datasets", "checksum_sha256") + op.drop_column("datasets", "size_bytes") + op.drop_column("datasets", "content_type") + op.drop_column("datasets", "stored_filename") + op.drop_column("datasets", "original_filename") diff --git a/backend/alembic/versions/20260611212435_add_jobs_table.py b/backend/alembic/versions/20260611212435_add_jobs_table.py new file mode 100644 index 00000000..663a6556 --- /dev/null +++ b/backend/alembic/versions/20260611212435_add_jobs_table.py @@ -0,0 +1,38 @@ +"""Add lightweight job table for sprint-3 async architecture foundation.""" + +from alembic import op +import sqlalchemy as sa + + +revision = "20260611212435" +down_revision = "202601120001" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "jobs", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("job_type", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False, server_default="queued"), + sa.Column("project_id", sa.UUID(as_uuid=True), sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=False), + sa.Column("dataset_id", sa.UUID(as_uuid=True), sa.ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True), + sa.Column("input_dataset_id", sa.UUID(as_uuid=True), sa.ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True), + sa.Column("output_dataset_id", sa.UUID(as_uuid=True), sa.ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True), + sa.Column("parameters_json", sa.JSON(), nullable=False), + sa.Column("result_json", sa.JSON(), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()")), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + ) + + op.create_index("ix_jobs_project_id", "jobs", ["project_id"]) + op.create_index("ix_jobs_status", "jobs", ["status"]) + + +def downgrade() -> None: + op.drop_index("ix_jobs_status", table_name="jobs") + op.drop_index("ix_jobs_project_id", table_name="jobs") + op.drop_table("jobs") diff --git a/backend/alembic/versions/202606120001_add_dataset_reference_metadata.py b/backend/alembic/versions/202606120001_add_dataset_reference_metadata.py new file mode 100644 index 00000000..11204da8 --- /dev/null +++ b/backend/alembic/versions/202606120001_add_dataset_reference_metadata.py @@ -0,0 +1,28 @@ +"""Add dataset reference and provenance metadata columns.""" + +from alembic import op +import sqlalchemy as sa + + +revision = "202606120001" +down_revision = "20260611212435" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("datasets", sa.Column("dataset_role", sa.Text(), nullable=False, server_default="source")) + op.add_column("datasets", sa.Column("source_name", sa.Text(), nullable=True)) + op.add_column("datasets", sa.Column("reference_layer_name", sa.Text(), nullable=True)) + op.add_column("datasets", sa.Column("source_metadata", sa.JSON(), nullable=True)) + op.add_column("datasets", sa.Column("provenance_metadata", sa.JSON(), nullable=True)) + op.add_column("datasets", sa.Column("imported_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()"), nullable=False)) + + +def downgrade() -> None: + op.drop_column("datasets", "imported_at") + op.drop_column("datasets", "provenance_metadata") + op.drop_column("datasets", "source_metadata") + op.drop_column("datasets", "reference_layer_name") + op.drop_column("datasets", "source_name") + op.drop_column("datasets", "dataset_role") diff --git a/backend/alembic/versions/202606120700_sprint7a_persistence_foundation.py b/backend/alembic/versions/202606120700_sprint7a_persistence_foundation.py new file mode 100644 index 00000000..1d3c80e1 --- /dev/null +++ b/backend/alembic/versions/202606120700_sprint7a_persistence_foundation.py @@ -0,0 +1,76 @@ +"""Add Sprint 7A vector feature and QA persistence foundation.""" + +from alembic import op +import sqlalchemy as sa +from geoalchemy2 import Geometry + + +revision = "202606120700" +down_revision = "202606120001" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "vector_features", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("dataset_id", sa.UUID(as_uuid=True), sa.ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False), + sa.Column("feature_class", sa.Text(), nullable=True), + sa.Column("source_feature_id", sa.Text(), nullable=True), + sa.Column("properties_json", sa.JSON(), nullable=True), + sa.Column("geometry", Geometry("GEOMETRY", srid=4326, spatial_index=False), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()")), + ) + op.create_index("ix_vector_features_dataset_id", "vector_features", ["dataset_id"]) + op.create_index("ix_vector_features_geometry", "vector_features", ["geometry"], postgresql_using="gist") + + op.create_table( + "quality_checks", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("project_id", sa.UUID(as_uuid=True), sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=False), + sa.Column("job_id", sa.UUID(as_uuid=True), sa.ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True), + sa.Column("analysis_run_id", sa.UUID(as_uuid=True), sa.ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True), + sa.Column("candidate_dataset_id", sa.UUID(as_uuid=True), sa.ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True), + sa.Column("reference_dataset_id", sa.UUID(as_uuid=True), sa.ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False), + sa.Column("check_type", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("score", sa.Float(), nullable=True), + sa.Column("parameters_json", sa.JSON(), nullable=True), + sa.Column("findings_json", sa.JSON(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()")), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index("ix_quality_checks_project_id", "quality_checks", ["project_id"]) + op.create_index("ix_quality_checks_reference_dataset_id", "quality_checks", ["reference_dataset_id"]) + op.create_index("ix_quality_checks_candidate_dataset_id", "quality_checks", ["candidate_dataset_id"]) + op.create_index("ix_quality_checks_analysis_run_id", "quality_checks", ["analysis_run_id"]) + + op.create_table( + "metrics", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("quality_check_id", sa.UUID(as_uuid=True), sa.ForeignKey("quality_checks.id", ondelete="CASCADE"), nullable=True), + sa.Column("analysis_run_id", sa.UUID(as_uuid=True), sa.ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True), + sa.Column("metric_key", sa.Text(), nullable=False), + sa.Column("metric_value", sa.Float(), nullable=True), + sa.Column("metric_unit", sa.Text(), nullable=True), + sa.Column("label", sa.Text(), nullable=True), + sa.Column("metadata_json", sa.JSON(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()")), + ) + op.create_index("ix_metrics_quality_check_id", "metrics", ["quality_check_id"]) + op.create_index("ix_metrics_analysis_run_id", "metrics", ["analysis_run_id"]) + + +def downgrade() -> None: + op.drop_index("ix_metrics_analysis_run_id", table_name="metrics") + op.drop_index("ix_metrics_quality_check_id", table_name="metrics") + op.drop_table("metrics") + op.drop_index("ix_quality_checks_analysis_run_id", table_name="quality_checks") + op.drop_index("ix_quality_checks_candidate_dataset_id", table_name="quality_checks") + op.drop_index("ix_quality_checks_reference_dataset_id", table_name="quality_checks") + op.drop_index("ix_quality_checks_project_id", table_name="quality_checks") + op.drop_table("quality_checks") + op.drop_index("ix_vector_features_geometry", table_name="vector_features", postgresql_using="gist") + op.drop_index("ix_vector_features_dataset_id", table_name="vector_features") + op.drop_table("vector_features") diff --git a/backend/alembic/versions/202606120800_sprint8_detection_foundation.py b/backend/alembic/versions/202606120800_sprint8_detection_foundation.py new file mode 100644 index 00000000..3ee01ae1 --- /dev/null +++ b/backend/alembic/versions/202606120800_sprint8_detection_foundation.py @@ -0,0 +1,59 @@ +"""Add Sprint 8 detection foundation.""" + +from alembic import op +import sqlalchemy as sa +from geoalchemy2 import Geometry + + +revision = "202606120800" +down_revision = "202606120700" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("analysis_runs", sa.Column("dataset_id", sa.UUID(as_uuid=True), sa.ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)) + op.add_column("analysis_runs", sa.Column("job_id", sa.UUID(as_uuid=True), sa.ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)) + op.add_column("analysis_runs", sa.Column("model_name", sa.String(length=255), nullable=True)) + op.add_column("analysis_runs", sa.Column("model_version", sa.String(length=120), nullable=True)) + op.add_column("analysis_runs", sa.Column("result_json", sa.JSON(), nullable=True)) + op.add_column("analysis_runs", sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()"), nullable=False)) + + op.create_table( + "detections", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("project_id", sa.UUID(as_uuid=True), sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=False), + sa.Column("dataset_id", sa.UUID(as_uuid=True), sa.ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True), + sa.Column("analysis_run_id", sa.UUID(as_uuid=True), sa.ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True), + sa.Column("job_id", sa.UUID(as_uuid=True), sa.ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True), + sa.Column("model_name", sa.String(length=255), nullable=False), + sa.Column("model_version", sa.String(length=120), nullable=True), + sa.Column("class_name", sa.String(length=120), nullable=False), + sa.Column("confidence", sa.Float(), nullable=False), + sa.Column("geometry", Geometry("GEOMETRY", srid=4326, spatial_index=False), nullable=False), + sa.Column("bbox_json", sa.JSON(), nullable=True), + sa.Column("source_tile_path", sa.String(length=500), nullable=True), + sa.Column("properties_json", sa.JSON(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()"), nullable=False), + ) + op.create_index("ix_detections_project_id", "detections", ["project_id"]) + op.create_index("ix_detections_dataset_id", "detections", ["dataset_id"]) + op.create_index("ix_detections_analysis_run_id", "detections", ["analysis_run_id"]) + op.create_index("ix_detections_class_name", "detections", ["class_name"]) + op.create_index("ix_detections_geometry", "detections", ["geometry"], postgresql_using="gist") + + +def downgrade() -> None: + op.drop_index("ix_detections_geometry", table_name="detections", postgresql_using="gist") + op.drop_index("ix_detections_class_name", table_name="detections") + op.drop_index("ix_detections_analysis_run_id", table_name="detections") + op.drop_index("ix_detections_dataset_id", table_name="detections") + op.drop_index("ix_detections_project_id", table_name="detections") + op.drop_table("detections") + + op.drop_column("analysis_runs", "created_at") + op.drop_column("analysis_runs", "result_json") + op.drop_column("analysis_runs", "model_version") + op.drop_column("analysis_runs", "model_name") + op.drop_column("analysis_runs", "job_id") + op.drop_column("analysis_runs", "dataset_id") diff --git a/backend/alembic/versions/202606120900_sprint9_segmentation_foundation.py b/backend/alembic/versions/202606120900_sprint9_segmentation_foundation.py new file mode 100644 index 00000000..17d3dc5f --- /dev/null +++ b/backend/alembic/versions/202606120900_sprint9_segmentation_foundation.py @@ -0,0 +1,51 @@ +"""Add Sprint 9 segmentation foundation.""" + +from alembic import op +import sqlalchemy as sa +from geoalchemy2 import Geometry + + +revision = "202606120900" +down_revision = "202606120800" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "segmentations", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("project_id", sa.UUID(as_uuid=True), sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=False), + sa.Column("dataset_id", sa.UUID(as_uuid=True), sa.ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True), + sa.Column("job_id", sa.UUID(as_uuid=True), sa.ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True), + sa.Column("analysis_run_id", sa.UUID(as_uuid=True), sa.ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True), + sa.Column("model_name", sa.String(length=255), nullable=False), + sa.Column("model_version", sa.String(length=120), nullable=True), + sa.Column("class_name", sa.String(length=120), nullable=False), + sa.Column("confidence", sa.Float(), nullable=True), + sa.Column("geometry", Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False), + sa.Column("bbox_json", sa.JSON(), nullable=True), + sa.Column("area_m2", sa.Float(), nullable=True), + sa.Column("mask_path", sa.Text(), nullable=True), + sa.Column("source_tile_path", sa.String(length=500), nullable=True), + sa.Column("tile_index", sa.Integer(), nullable=True), + sa.Column("properties_json", sa.JSON(), nullable=True), + sa.Column("provenance_json", sa.JSON(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("NOW()"), nullable=False), + ) + op.create_index("ix_segmentations_project_id", "segmentations", ["project_id"]) + op.create_index("ix_segmentations_dataset_id", "segmentations", ["dataset_id"]) + op.create_index("ix_segmentations_analysis_run_id", "segmentations", ["analysis_run_id"]) + op.create_index("ix_segmentations_job_id", "segmentations", ["job_id"]) + op.create_index("ix_segmentations_class_name", "segmentations", ["class_name"]) + op.create_index("ix_segmentations_geometry", "segmentations", ["geometry"], postgresql_using="gist") + + +def downgrade() -> None: + op.drop_index("ix_segmentations_geometry", table_name="segmentations", postgresql_using="gist") + op.drop_index("ix_segmentations_class_name", table_name="segmentations") + op.drop_index("ix_segmentations_job_id", table_name="segmentations") + op.drop_index("ix_segmentations_analysis_run_id", table_name="segmentations") + op.drop_index("ix_segmentations_dataset_id", table_name="segmentations") + op.drop_index("ix_segmentations_project_id", table_name="segmentations") + op.drop_table("segmentations") diff --git a/backend/app/.gitkeep b/backend/app/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 00000000..75d83921 --- /dev/null +++ b/backend/app/__init__.py @@ -0,0 +1,3 @@ +from app.models.entities import AnalysisRun, Area, Dataset, Export, Project + +__all__ = ["AnalysisRun", "Area", "Dataset", "Export", "Project"] diff --git a/backend/app/ai/.gitkeep b/backend/app/ai/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/analysis/.gitkeep b/backend/app/analysis/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/api/.gitkeep b/backend/app/api/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/api/routes/.gitkeep b/backend/app/api/routes/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/api/routes/__init__.py b/backend/app/api/routes/__init__.py new file mode 100644 index 00000000..585499ea --- /dev/null +++ b/backend/app/api/routes/__init__.py @@ -0,0 +1 @@ +__all__ = ["areas", "datasets", "health", "projects", "exports", "jobs", "external", "qa"] diff --git a/backend/app/api/routes/areas.py b/backend/app/api/routes/areas.py new file mode 100644 index 00000000..71257610 --- /dev/null +++ b/backend/app/api/routes/areas.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from uuid import UUID + +from fastapi import APIRouter, Depends, Query +from fastapi import HTTPException +from sqlalchemy.orm import Session + +from app.db.session import get_db +from app.models import Area +from app.schemas.area import AreaCreate, AreaRead, AreaUpdate +from app.services.area_service import AreaService +from app.utils.response import envelope + +router = APIRouter(prefix="/projects/{project_id}/areas", tags=["areas"]) + + +@router.get("", response_model=dict) +def list_areas( + project_id: UUID, + limit: int = Query(default=50, ge=1, le=200), + offset: int = Query(default=0, ge=0), + db: Session = Depends(get_db), +): + areas, total = AreaService.list_areas(db, project_id=project_id, limit=limit, offset=offset) + return envelope({"items": [AreaRead.model_validate(area).model_dump() for area in areas], "total": total, "limit": limit, "offset": offset}) + + +@router.post("", status_code=201, response_model=dict) +def create_area(project_id: UUID, payload: AreaCreate, db: Session = Depends(get_db)): + area = AreaService.create_area(db, project_id, payload) + return envelope(AreaRead.model_validate(area).model_dump()) + + +@router.get("/{area_id}", response_model=dict) +def get_area( + project_id: UUID, + area_id: UUID, + db: Session = Depends(get_db), +): + area = AreaService.get_area(db, area_id) + if area.project_id != project_id: + raise HTTPException(status_code=404, detail="Area not found") + return envelope(AreaRead.model_validate(area).model_dump()) + + +@router.patch("/{area_id}", response_model=dict) +def update_area( + project_id: UUID, + area_id: UUID, + payload: AreaUpdate, + db: Session = Depends(get_db), +): + existing = db.get(Area, area_id) + if not existing or existing.project_id != project_id: + raise HTTPException(status_code=404, detail="Area not found") + area = AreaService.update_area(db, area_id, payload) + return envelope(AreaRead.model_validate(area).model_dump()) diff --git a/backend/app/api/routes/datasets.py b/backend/app/api/routes/datasets.py new file mode 100644 index 00000000..fc7b4a11 --- /dev/null +++ b/backend/app/api/routes/datasets.py @@ -0,0 +1,489 @@ +from __future__ import annotations + +import json +from typing import Any +from uuid import UUID +from uuid import UUID as _UUID + +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query +from fastapi import UploadFile +from sqlalchemy.orm import Session +from app.models import Area + +from app.core.errors import AppError +from app.db.session import get_db +from app.schemas import ( + RasterClipRequest, + RasterStatsResponse, + RasterReprojectRequest, + RasterTileRequest, + RasterNdviRequest, + RasterNdwiRequest, + RasterNdbiRequest, + VectorBBoxResponse, + VectorBufferRequest, + VectorClipRequest, + VectorIntersectRequest, +) +from app.schemas.job import JobCreate +from app.schemas.dataset import DatasetCreateResponse +from app.schemas.operations import VectorOperationResult +from app.services.job_service import JobService +from app.services.raster_operations_service import RasterOperationsService +from app.services.vector_operations_service import VectorOperationsService +from app.services.dataset_service import DatasetService +from app.utils.response import envelope + +router = APIRouter(prefix="/projects/{project_id}", tags=["datasets"]) + + +def _parse_metadata_json(raw: str | None, field_name: str) -> dict | None: + if raw is None: + return None + raw = raw.strip() + if not raw: + return None + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise AppError(code="INVALID_JSON", message=f"Invalid JSON for {field_name}", details={"field": field_name}, status_code=400) from exc + if not isinstance(value, dict): + raise AppError(code="INVALID_JSON", message=f"{field_name} must be a JSON object", details={"field": field_name}, status_code=400) + return value + + +def _run_job_sync( + db: Session, + project_id: UUID, + input_dataset_id: UUID, + job_type: str, + parameters: dict[str, Any], + operation, +) -> dict[str, Any]: + return JobService.run_sync_job( + db=db, + project_id=project_id, + job_type=job_type, + parameters=parameters, + operation=operation, + input_dataset_id=input_dataset_id, + ) + + +@router.post("/datasets/upload", status_code=201, response_model=dict) +async def upload_dataset( + project_id: UUID, + file: UploadFile = File(...), + dataset_type: str = Form(...), + source: str = Form("user_upload"), + area_id: UUID | None = Form(None), + dataset_role: str = Form("source"), + source_name: str | None = Form(None), + reference_layer_name: str | None = Form(None), + source_metadata_json: str | None = Form(None), + provenance_metadata_json: str | None = Form(None), + db: Session = Depends(get_db), +): + if area_id is not None: + area = db.get(Area, area_id) + if not area: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + if area.project_id != project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400) + + created = await DatasetService.upload_dataset( + db, + project_id=project_id, + file=file, + dataset_type=dataset_type, + source=source, + dataset_role=dataset_role, + source_name=source_name, + reference_layer_name=reference_layer_name, + source_metadata=_parse_metadata_json(source_metadata_json, "source_metadata_json"), + provenance_metadata=_parse_metadata_json(provenance_metadata_json, "provenance_metadata_json"), + area_id=area_id, + ) + return envelope(created.model_dump()) + + +@router.get("/datasets", response_model=dict) +def list_datasets( + project_id: UUID, + limit: int = Query(default=50, ge=1, le=200), + offset: int = Query(default=0, ge=0), + db: Session = Depends(get_db), +): + datasets, total = DatasetService.list_datasets(db, project_id, limit=limit, offset=offset) + return envelope({"items": [item.model_dump() for item in datasets], "total": total, "limit": limit, "offset": offset}) + + +@router.get("/datasets/{dataset_id}", response_model=dict) +def get_dataset( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + return envelope(DatasetCreateResponse.model_validate(dataset).model_dump()) + + +@router.post("/datasets/{dataset_id}/metadata/refresh", response_model=dict) +def refresh_dataset_metadata( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + refreshed = DatasetService.refresh_metadata(db, dataset_id) + return envelope(refreshed.model_dump()) + + +@router.get("/datasets/{dataset_id}/vector/inspect", response_model=dict) +def inspect_vector_dataset( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + return envelope(VectorOperationsService.inspect(db, dataset_id).model_dump()) + + +@router.get("/datasets/{dataset_id}/vector/bbox", response_model=dict) +def vector_bbox( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + payload = VectorOperationsService.bbox(db, dataset_id) + return envelope(VectorBBoxResponse(**payload).model_dump()) + + +@router.get("/datasets/{dataset_id}/vector/stats", response_model=dict) +def vector_stats( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + return envelope(VectorOperationsService.stats(db, dataset_id)) + + +@router.post("/datasets/{dataset_id}/vector/clip", status_code=201, response_model=dict) +def clip_vector_dataset( + project_id: UUID, + dataset_id: UUID, + payload: VectorClipRequest, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + job = _run_job_sync( + db=db, + project_id=project_id, + input_dataset_id=dataset_id, + job_type="vector.clip", + parameters=payload.model_dump(), + operation=lambda: VectorOperationsService.clip_by_area( + db, + dataset_id=dataset_id, + area_id=payload.area_id, + output_name=payload.output_name, + ), + ) + return envelope(job) + + +@router.post("/datasets/{dataset_id}/vector/buffer", status_code=201, response_model=dict) +def buffer_vector_dataset( + project_id: UUID, + dataset_id: UUID, + payload: VectorBufferRequest, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + job = _run_job_sync( + db=db, + project_id=project_id, + input_dataset_id=dataset_id, + job_type="vector.buffer", + parameters=payload.model_dump(), + operation=lambda: VectorOperationsService.buffer( + db, + dataset_id=dataset_id, + distance_m=payload.distance_m, + dissolve=payload.dissolve, + output_name=payload.output_name, + ), + ) + return envelope(job) + + +@router.post("/datasets/{dataset_id}/vector/intersect", status_code=201, response_model=dict) +def intersect_vector_dataset( + project_id: UUID, + dataset_id: UUID, + payload: VectorIntersectRequest, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + job = _run_job_sync( + db=db, + project_id=project_id, + input_dataset_id=dataset_id, + job_type="vector.intersect", + parameters=payload.model_dump(), + operation=lambda: VectorOperationsService.intersect( + db, + source_dataset_id=dataset_id, + target_dataset_id=UUID(payload.other_dataset_id), + output_name=payload.output_name, + ), + ) + return envelope(job) + + +@router.get("/datasets/{dataset_id}/vector/summary", response_model=dict) +def vector_dataset_summary( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + return envelope(DatasetService.vector_summary(db, dataset_id)) + + +@router.get("/datasets/{dataset_id}/raster/inspect", response_model=dict) +def raster_dataset_inspect( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + payload = RasterOperationsService.inspect(db, dataset_id) + return envelope(payload) + + +@router.get("/datasets/{dataset_id}/raster/preview", response_model=dict) +def raster_preview_readiness( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + return envelope(RasterOperationsService.preview(db, dataset_id)) + + +@router.get("/datasets/{dataset_id}/raster/stats", response_model=dict) +def raster_stats( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + payload = RasterOperationsService.stats(db, dataset_id) + return envelope(RasterStatsResponse(**payload).model_dump()) + + +@router.post("/datasets/{dataset_id}/raster/reproject", status_code=201, response_model=dict) +def raster_reproject_dataset( + project_id: UUID, + dataset_id: UUID, + payload: RasterReprojectRequest, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + job = _run_job_sync( + db=db, + project_id=project_id, + input_dataset_id=dataset_id, + job_type="raster.reproject", + parameters=payload.model_dump(), + operation=lambda: RasterOperationsService.reproject( + db, + dataset_id, + target_crs=payload.target_crs, + output_name=payload.output_name, + resampling=payload.resampling, + ), + ) + return envelope(job) + + +@router.post("/datasets/{dataset_id}/raster/clip", status_code=201, response_model=dict) +def raster_clip_dataset( + project_id: UUID, + dataset_id: UUID, + payload: RasterClipRequest, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + job = _run_job_sync( + db=db, + project_id=project_id, + input_dataset_id=dataset_id, + job_type="raster.clip", + parameters=payload.model_dump(), + operation=lambda: RasterOperationsService.clip(db, dataset_id, UUID(payload.area_id), payload.output_name), + ) + return envelope(job) + + +@router.post("/datasets/{dataset_id}/raster/tile", status_code=201, response_model=dict) +def raster_tile_dataset( + project_id: UUID, + dataset_id: UUID, + payload: RasterTileRequest, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + job = _run_job_sync( + db=db, + project_id=project_id, + input_dataset_id=dataset_id, + job_type="raster.tile", + parameters=payload.model_dump(), + operation=lambda: RasterOperationsService.tile( + db, + dataset_id, + tile_size=payload.tile_size, + overlap=payload.overlap, + output_name=payload.output_name, + ), + ) + return envelope(job) + + +@router.post("/datasets/{dataset_id}/raster/indices/ndvi", status_code=201, response_model=dict) +def raster_ndvi_dataset( + project_id: UUID, + dataset_id: UUID, + payload: RasterNdviRequest, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + job = _run_job_sync( + db=db, + project_id=project_id, + input_dataset_id=dataset_id, + job_type="raster.ndvi", + parameters=payload.model_dump(), + operation=lambda: RasterOperationsService.ndvi( + db, + dataset_id=dataset_id, + nir_band=payload.nir_band, + red_band=payload.red_band, + output_name=payload.output_name, + ), + ) + return envelope(job) + + +@router.post("/datasets/{dataset_id}/raster/indices/ndwi", status_code=201, response_model=dict) +def raster_ndwi_dataset( + project_id: UUID, + dataset_id: UUID, + payload: RasterNdwiRequest, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + job = _run_job_sync( + db=db, + project_id=project_id, + input_dataset_id=dataset_id, + job_type="raster.ndwi", + parameters=payload.model_dump(), + operation=lambda: RasterOperationsService.ndwi( + db, + dataset_id=dataset_id, + green_band=payload.green_band, + nir_band=payload.nir_band, + output_name=payload.output_name, + ), + ) + return envelope(job) + + +@router.post("/datasets/{dataset_id}/raster/indices/ndbi", status_code=201, response_model=dict) +def raster_ndbi_dataset( + project_id: UUID, + dataset_id: UUID, + payload: RasterNdbiRequest, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + job = _run_job_sync( + db=db, + project_id=project_id, + input_dataset_id=dataset_id, + job_type="raster.ndbi", + parameters=payload.model_dump(), + operation=lambda: RasterOperationsService.ndbi( + db, + dataset_id=dataset_id, + swir_band=payload.swir_band, + nir_band=payload.nir_band, + output_name=payload.output_name, + ), + ) + return envelope(job) + + +@router.get("/datasets/{dataset_id}/raster/metadata", response_model=dict) +def raster_dataset_metadata( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + return envelope(RasterOperationsService.metadata(db, dataset_id)) + + +@router.get("/datasets/{dataset_id}/content", response_model=dict) +def dataset_content( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + dataset = DatasetService.get_dataset(db, dataset_id) + if dataset.project_id != project_id: + raise HTTPException(status_code=404, detail="Dataset not found") + return envelope(DatasetService.get_dataset_geojson(db, dataset_id)) diff --git a/backend/app/api/routes/demo.py b/backend/app/api/routes/demo.py new file mode 100644 index 00000000..a0666507 --- /dev/null +++ b/backend/app/api/routes/demo.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, status +from sqlalchemy.orm import Session + +from app.db.session import get_db +from app.schemas.demo import DemoWorkflowResponse +from app.services.demo_workflow_service import DemoWorkflowService +from app.utils.response import envelope + +router = APIRouter(prefix="/demo", tags=["demo"]) + + +@router.post("/workflow", status_code=status.HTTP_201_CREATED, response_model=dict) +def seed_demo_workflow(db: Session = Depends(get_db)) -> dict: + result: DemoWorkflowResponse = DemoWorkflowService.seed(db) + return envelope(result.model_dump()) diff --git a/backend/app/api/routes/detection.py b/backend/app/api/routes/detection.py new file mode 100644 index 00000000..8aadac1b --- /dev/null +++ b/backend/app/api/routes/detection.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from uuid import UUID + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from app.db.session import get_db +from app.schemas import DetectionQaRequest, DetectionRunRequest +from app.services.detection_service import DetectionService +from app.services.model_registry_service import ModelRegistryService +from app.utils.response import envelope + +router = APIRouter(prefix="/detection", tags=["detection"]) + + +@router.get("/models", response_model=dict) +def list_detection_models() -> dict: + return envelope({"models": [model.model_dump() for model in ModelRegistryService.list_model_capabilities()]}) + + +@router.post("/run", response_model=dict) +def run_detection(payload: DetectionRunRequest, db: Session = Depends(get_db)) -> dict: + result = DetectionService.run_detection( + db=db, + project_id=payload.project_id, + dataset_id=payload.dataset_id, + model_id=payload.model_id, + confidence_threshold=payload.confidence_threshold, + class_filter=payload.class_filter, + tile_manifest_path=payload.tile_manifest_path, + parameters_json=payload.parameters_json, + ) + return envelope(result.model_dump()) + + +@router.get("/runs", response_model=dict) +def list_detection_runs( + project_id: UUID | None = None, + dataset_id: UUID | None = None, + db: Session = Depends(get_db), +) -> dict: + return envelope(DetectionService.list_runs(db, project_id=project_id, dataset_id=dataset_id).model_dump()) + + +@router.get("/runs/{analysis_run_id}", response_model=dict) +def get_detection_run(analysis_run_id: UUID, db: Session = Depends(get_db)) -> dict: + return envelope(DetectionService.get_run(db, analysis_run_id).model_dump()) + + +@router.get("/runs/{analysis_run_id}/detections", response_model=dict) +def list_detection_run_detections( + analysis_run_id: UUID, + dataset_id: UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + db: Session = Depends(get_db), +) -> dict: + return envelope( + DetectionService.list_detections( + db, + analysis_run_id=analysis_run_id, + dataset_id=dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ).model_dump() + ) + + +@router.get("/datasets/{dataset_id}/detections", response_model=dict) +def list_dataset_detections( + dataset_id: UUID, + analysis_run_id: UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + db: Session = Depends(get_db), +) -> dict: + return envelope( + DetectionService.list_detections( + db, + analysis_run_id=analysis_run_id, + dataset_id=dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ).model_dump() + ) + + +@router.get("/detections/{detection_id}", response_model=dict) +def get_detection(detection_id: UUID, db: Session = Depends(get_db)) -> dict: + return envelope(DetectionService.get_detection(db, detection_id).model_dump()) + + +@router.get("/runs/{analysis_run_id}/geojson", response_model=dict) +def get_detection_run_geojson( + analysis_run_id: UUID, + class_name: str | None = None, + min_confidence: float | None = None, + db: Session = Depends(get_db), +) -> dict: + return envelope( + DetectionService.detections_to_geojson( + db, + analysis_run_id=analysis_run_id, + class_name=class_name, + min_confidence=min_confidence, + ) + ) + + +@router.get("/datasets/{dataset_id}/geojson", response_model=dict) +def get_dataset_detection_geojson( + dataset_id: UUID, + analysis_run_id: UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + db: Session = Depends(get_db), +) -> dict: + return envelope( + DetectionService.detections_to_geojson( + db, + analysis_run_id=analysis_run_id, + dataset_id=dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ) + ) + + +@router.post("/runs/{analysis_run_id}/qa/reference", response_model=dict) +def compare_detection_run_with_reference( + analysis_run_id: UUID, + payload: DetectionQaRequest, + db: Session = Depends(get_db), +) -> dict: + return envelope( + DetectionService.compare_detections_with_reference( + db=db, + analysis_run_id=analysis_run_id, + reference_dataset_id=payload.reference_dataset_id, + iou_threshold=payload.iou_threshold, + class_name=payload.class_name, + min_confidence=payload.min_confidence, + ) + ) diff --git a/backend/app/api/routes/exports.py b/backend/app/api/routes/exports.py new file mode 100644 index 00000000..a4d9d994 --- /dev/null +++ b/backend/app/api/routes/exports.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from uuid import UUID + +from fastapi import APIRouter, Depends, Query +from fastapi.responses import FileResponse +from sqlalchemy.orm import Session + +from app.db.session import get_db +from app.schemas.export import GeoJsonExportRequest, MetadataExportRequest, ReportExportRequest +from app.services.export_service import ExportService +from app.utils.response import envelope + +router = APIRouter(prefix="/exports", tags=["exports"]) + + +@router.post("/geojson", response_model=dict) +def export_geojson(payload: GeoJsonExportRequest, db: Session = Depends(get_db)): + if payload.export_kind == "detection_run" and payload.analysis_run_id is not None: + return envelope( + ExportService.export_detection_run_geojson(db, payload.analysis_run_id, payload.name).model_dump(mode="json") + ) + if payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None: + return envelope( + ExportService.export_segmentation_run_geojson(db, payload.analysis_run_id, payload.name).model_dump(mode="json") + ) + if payload.dataset_id is not None: + return envelope(ExportService.export_dataset_geojson(db, payload.dataset_id, payload.name).model_dump(mode="json")) + return envelope({}) + + +@router.post("/metadata", response_model=dict) +def export_project_metadata(payload: MetadataExportRequest, db: Session = Depends(get_db)): + return envelope(ExportService.export_project_metadata(db, payload.project_id, payload.name).model_dump(mode="json")) + + +@router.post("/report", response_model=dict) +def export_project_report(payload: ReportExportRequest, db: Session = Depends(get_db)): + return envelope(ExportService.export_project_report(db, payload.project_id, payload.name).model_dump(mode="json")) + + +@router.get("/projects/{project_id}/exports", response_model=dict) +def list_project_exports( + project_id: UUID, + limit: int = Query(default=50, ge=1, le=100), + offset: int = Query(default=0, ge=0), + db: Session = Depends(get_db), +): + return envelope(ExportService.list_project_exports(db, project_id, limit=limit, offset=offset).model_dump(mode="json")) + + +@router.get("/{export_id}", response_model=dict) +def get_export(export_id: UUID, db: Session = Depends(get_db)): + return envelope(ExportService.get_export(db, export_id).model_dump(mode="json")) + + +@router.get("/{export_id}/download") +def download_export(export_id: UUID, db: Session = Depends(get_db)): + path = ExportService.get_export_download_path(db, export_id) + media_type = "text/html" if path.suffix.lower() in {".html", ".htm"} else "application/json" + return FileResponse(path, filename=path.name, media_type=media_type) + + +@router.get("/{export_id}/content", response_model=dict) +def get_export_content(export_id: UUID, db: Session = Depends(get_db)): + return envelope(ExportService.get_export_content(db, export_id).model_dump(mode="json")) diff --git a/backend/app/api/routes/external.py b/backend/app/api/routes/external.py new file mode 100644 index 00000000..8f9d9ff6 --- /dev/null +++ b/backend/app/api/routes/external.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from app.core.errors import AppError +from app.db.session import get_db +from app.models import Area, Project +from app.providers.registry import fetch_provider_data, get_provider, import_provider_dataset, list_provider_capabilities +from app.schemas import ExternalFetchRequest, ExternalFetchResponse, ProviderImportRequest +from app.utils.response import envelope + +router = APIRouter(prefix="/external", tags=["external"]) + + +def _validate_area_in_project(db: Session, project_id, area_id: str | None) -> None: + if area_id is None: + return + area = db.get(Area, area_id) + if not area: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + if area.project_id != project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400) + + +def _assert_project_exists(db: Session, project_id): + project = db.get(Project, project_id) + if not project: + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + + +def _normalize_layer_input(layers: list[str] | None) -> list[str]: + return [layer.strip() for layer in (layers or []) if isinstance(layer, str) and layer.strip()] + + + + +def _provider_payload(provider_name: str) -> dict: + return get_provider(provider_name).capability.to_dict() + + +@router.get("/providers") +def list_external_providers() -> dict: + return envelope({ + "providers": [provider.to_dict() for provider in list_provider_capabilities()], + }) + + +@router.get("/providers/capabilities") +def get_external_provider_capabilities() -> dict: + return envelope({ + "providers": [provider.to_dict() for provider in list_provider_capabilities()], + }) + + +@router.get("/providers/{provider_name}") +def get_external_provider(provider_name: str) -> dict: + return envelope(_provider_payload(provider_name)) + + +@router.get("/providers/{provider_name}/layers") +def get_external_provider_layers(provider_name: str) -> dict: + provider = get_provider(provider_name) + return envelope({ + "provider_name": provider.provider_name, + "layers": provider.supported_layers, + }) + + +@router.get("/providers/{provider_name}/status") +def get_external_provider_status(provider_name: str) -> dict: + provider = get_provider(provider_name) + return envelope({ + "provider_name": provider.provider_name, + "configured": provider.is_configured, + "status": provider.capability.status, + "limitation_message": provider.limitation_message, + }) + + +@router.post("/providers/{provider_name}/import") +def import_external_provider_dataset(provider_name: str, payload: ProviderImportRequest) -> dict: + result = import_provider_dataset( + provider_name=provider_name, + project_id=payload.project_id, + area_id=payload.area_id, + layers=_normalize_layer_input(payload.layers), + requested_dataset_role=payload.dataset_role, + ) + return envelope(result.model_dump()) + + +def _run_fetch(payload: ExternalFetchRequest, provider_name: str) -> ExternalFetchResponse: + area_id_str = str(payload.area_id) if payload.area_id else None + response = fetch_provider_data( + provider_name=provider_name, + project_id=str(payload.project_id), + area_id=area_id_str, + layers=_normalize_layer_input(payload.layers), + ) + return ExternalFetchResponse( + provider=provider_name, + status=response.get("status", "not_configured"), + message=response.get("message", "Provider fetch executed."), + requested_layers=_normalize_layer_input(payload.layers), + project_id=payload.project_id, + area_id=payload.area_id, + ) + + +@router.post("/osm/fetch") +def fetch_osm(payload: ExternalFetchRequest, db: Session = Depends(get_db)) -> dict: + _assert_project_exists(db, payload.project_id) + _validate_area_in_project(db, payload.project_id, payload.area_id) + return envelope(_run_fetch(payload, "osm").model_dump()) + + +@router.post("/grb/fetch") +def fetch_grb(payload: ExternalFetchRequest, db: Session = Depends(get_db)) -> dict: + _assert_project_exists(db, payload.project_id) + _validate_area_in_project(db, payload.project_id, payload.area_id) + return envelope(_run_fetch(payload, "grb").model_dump()) diff --git a/backend/app/api/routes/health.py b/backend/app/api/routes/health.py new file mode 100644 index 00000000..f9bb6ef7 --- /dev/null +++ b/backend/app/api/routes/health.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from importlib import import_module +from sqlalchemy import text +from fastapi import APIRouter + +from app.schemas.health import HealthResponse, SystemCapabilities +from app.providers.registry import list_provider_capabilities +from app.db.session import get_engine + +router = APIRouter() + + +def _dependency_enabled(module: str) -> bool: + try: + import_module(module) + return True + except Exception: + return False + + +@router.get("/health") +def readiness() -> HealthResponse: + db_status = "ok" + try: + with get_engine().connect() as connection: + connection.execute(text("SELECT 1")) + except Exception: + db_status = "degraded" + return HealthResponse(status="ok", service="geointel-backend", version="0.1.0", database=db_status) + + +@router.get("/api/v1/system/capabilities") +def capabilities() -> dict: + providers = [item.to_dict() for item in list_provider_capabilities()] + return {"data": SystemCapabilities( + postgis=True, + rasterio=_dependency_enabled("rasterio"), + geopandas=_dependency_enabled("geopandas"), + yolo=False, + sam=False, + grb="planned", + sentinel="planned", + providers=providers, + ).model_dump()} diff --git a/backend/app/api/routes/jobs.py b/backend/app/api/routes/jobs.py new file mode 100644 index 00000000..f1ae7a96 --- /dev/null +++ b/backend/app/api/routes/jobs.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy.orm import Session + +from app.db.session import get_db +from app.schemas import JobCreate, JobList, JobRead, JobStatus +from app.services.job_service import JobService +from app.utils.response import envelope + + +router = APIRouter(prefix="/projects/{project_id}", tags=["jobs"]) + + +@router.post("/jobs", status_code=201, response_model=dict) +def create_job( + project_id: UUID, + payload: JobCreate, + db: Session = Depends(get_db), +): + if payload.project_id != project_id: + raise HTTPException(status_code=400, detail="project_id mismatch") + return envelope(JobService.create_job(db, payload).model_dump()) + + +@router.get("/jobs", response_model=dict) +def list_jobs( + project_id: UUID, + dataset_id: UUID | None = Query(default=None), + limit: int = Query(default=50, ge=1, le=200), + offset: int = Query(default=0, ge=0), + db: Session = Depends(get_db), +): + items, total = JobService.list_jobs( + db, + project_id=project_id, + dataset_id=dataset_id, + limit=limit, + offset=offset, + ) + return envelope(JobList(items=items, total=total, limit=limit, offset=offset).model_dump()) + + +@router.get("/jobs/{job_id}", response_model=dict) +def read_job( + project_id: UUID, + job_id: UUID, + db: Session = Depends(get_db), +): + job = JobService.get_job(db, job_id) + if job.project_id != project_id: + raise HTTPException(status_code=404, detail="Job not found") + return envelope(job.model_dump()) + + +@router.get("/jobs/{job_id}/status", response_model=dict) +def read_job_status( + project_id: UUID, + job_id: UUID, + db: Session = Depends(get_db), +): + status_row = JobService.get_job_status(db, job_id) + if status_row["project_id"] != str(project_id): + raise HTTPException(status_code=404, detail="Job not found") + return envelope(JobStatus(**status_row).model_dump()) diff --git a/backend/app/api/routes/projects.py b/backend/app/api/routes/projects.py new file mode 100644 index 00000000..b61c3bb6 --- /dev/null +++ b/backend/app/api/routes/projects.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from sqlalchemy.orm import Session + +from app.db.session import get_db +from app.schemas.project import ProjectCreate, ProjectRead, ProjectUpdate +from app.services.project_service import ProjectService +from app.utils.response import envelope + +router = APIRouter(prefix="/projects", tags=["projects"]) + + +@router.get("", response_model=dict) +def list_projects( + limit: int = Query(default=50, ge=1, le=200), + offset: int = Query(default=0, ge=0), + db: Session = Depends(get_db), +): + projects, total = ProjectService.list_projects(db, limit=limit, offset=offset) + return envelope({"items": [ProjectRead.model_validate(item).model_dump() for item in projects], "total": total, "limit": limit, "offset": offset}) + + +@router.post("", status_code=status.HTTP_201_CREATED, response_model=dict) +def create_project(payload: ProjectCreate, db: Session = Depends(get_db)): + project = ProjectService.create_project(db, payload) + return envelope(ProjectRead.model_validate(project).model_dump()) + + +@router.get("/{project_id}", response_model=dict) +def get_project(project_id: UUID, db: Session = Depends(get_db)): + project = ProjectService.get_project(db, project_id) + if not project: + raise HTTPException(status_code=404, detail="Project not found") + return envelope(ProjectRead.model_validate(project).model_dump()) + + +@router.patch("/{project_id}", response_model=dict) +def update_project(project_id: UUID, payload: ProjectUpdate, db: Session = Depends(get_db)): + project = ProjectService.update_project(db, project_id, payload) + return envelope(ProjectRead.model_validate(project).model_dump()) + + +@router.delete("/{project_id}", status_code=status.HTTP_200_OK, response_model=dict) +def delete_project(project_id: UUID, db: Session = Depends(get_db)): + if not ProjectService.delete_project(db, project_id): + raise HTTPException(status_code=404, detail="Project not found") + return envelope({"deleted": True}) diff --git a/backend/app/api/routes/qa.py b/backend/app/api/routes/qa.py new file mode 100644 index 00000000..47d62b64 --- /dev/null +++ b/backend/app/api/routes/qa.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from app.db.session import get_db +from app.core.errors import AppError +from app.models import Dataset, Job +from app.schemas import QaProviderComparisonRequest +from app.services.qa_service import QaService +from app.services.job_service import JobService +from app.services.quality_service import QualityService +from app.utils.response import envelope + +router = APIRouter(prefix="/qa", tags=["qa"]) + + +@router.post("/detections-vs-reference") +def compare_candidate_with_reference( + payload: QaProviderComparisonRequest, + db: Session = Depends(get_db), +) -> dict: + candidate_dataset = db.get(Dataset, payload.candidate_dataset_id) + if not candidate_dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Candidate dataset not found", status_code=404) + job = JobService.run_sync_job( + db=db, + project_id=candidate_dataset.project_id, + job_type="qa.compare-candidate-with-reference", + parameters=payload.model_dump(mode="json"), + input_dataset_id=candidate_dataset.id, + operation=lambda: QaService.compare_candidate_with_reference( + db=db, + project_id=candidate_dataset.project_id, + candidate_dataset_id=payload.candidate_dataset_id, + reference_dataset_id=payload.reference_dataset_id, + iou_threshold=payload.iou_threshold, + area_id=payload.area_id, + ).model_dump(mode="json"), + ) + result_json = job.get("result_json") if isinstance(job, dict) else None + if isinstance(result_json, dict) and job.get("status") == "success": + quality_check = QualityService.persist_quality_check( + db=db, + project_id=candidate_dataset.project_id, + job_id=uuid.UUID(str(job["id"])), + candidate_dataset_id=payload.candidate_dataset_id, + reference_dataset_id=payload.reference_dataset_id, + check_type="candidate_vs_reference", + status=str(result_json.get("status", "ok")), + score=result_json.get("f1_score"), + parameters=payload.model_dump(mode="json"), + findings={ + "matches": result_json.get("matches"), + "false_positives": result_json.get("false_positives"), + "false_negatives": result_json.get("false_negatives"), + "warnings": result_json.get("warnings", []), + "unsupported_geometry": result_json.get("unsupported_geometry", False), + "unsupported_geometries": result_json.get("unsupported_geometries", []), + }, + metrics={ + "precision": result_json.get("precision"), + "recall": result_json.get("recall"), + "f1": result_json.get("f1_score"), + "mean_iou": result_json.get("mean_iou"), + "false_positive_count": result_json.get("false_positives"), + "false_negative_count": result_json.get("false_negatives"), + }, + ) + result_json["quality_check_id"] = str(quality_check.id) + + job_record = db.get(Job, uuid.UUID(str(job["id"]))) + if job_record: + job_record.result_json = result_json + db.add(job_record) + db.commit() + + return envelope(job) diff --git a/backend/app/api/routes/quality_checks.py b/backend/app/api/routes/quality_checks.py new file mode 100644 index 00000000..28e7506e --- /dev/null +++ b/backend/app/api/routes/quality_checks.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from uuid import UUID + +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session + +from app.db.session import get_db +from app.schemas.qa import QualityCheckList +from app.services.quality_check_service import QualityCheckService +from app.utils.response import envelope + +router = APIRouter(prefix="/projects/{project_id}", tags=["quality-checks"]) + + +@router.get("/quality-checks", response_model=dict) +def list_quality_checks( + project_id: UUID, + limit: int = Query(default=50, ge=1, le=200), + offset: int = Query(default=0, ge=0), + db: Session = Depends(get_db), +) -> dict: + items, total = QualityCheckService.list_quality_checks( + db, + project_id=project_id, + limit=limit, + offset=offset, + ) + return envelope(QualityCheckList(items=items, total=total, limit=limit, offset=offset).model_dump()) diff --git a/backend/app/api/routes/segmentation.py b/backend/app/api/routes/segmentation.py new file mode 100644 index 00000000..bfb4fe96 --- /dev/null +++ b/backend/app/api/routes/segmentation.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from uuid import UUID + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from app.db.session import get_db +from app.schemas import SegmentationQaRequest, SegmentationRunRequest +from app.services.model_registry_service import ModelRegistryService +from app.services.segmentation_service import SegmentationService +from app.utils.response import envelope + +router = APIRouter(prefix="/segmentation", tags=["segmentation"]) + + +@router.get("/models", response_model=dict) +def list_segmentation_models() -> dict: + return envelope({"models": [model.model_dump() for model in ModelRegistryService.list_model_capabilities(task_type="segmentation")]}) + + +@router.post("/run", response_model=dict) +def run_segmentation(payload: SegmentationRunRequest, db: Session = Depends(get_db)) -> dict: + result = SegmentationService.run_segmentation( + db=db, + project_id=payload.project_id, + dataset_id=payload.dataset_id, + model_id=payload.model_id, + confidence_threshold=payload.confidence_threshold, + class_filter=payload.class_filter, + tile_manifest_path=payload.tile_manifest_path, + parameters_json=payload.parameters_json, + ) + return envelope(result.model_dump()) + + +@router.get("/runs", response_model=dict) +def list_segmentation_runs( + project_id: UUID | None = None, + dataset_id: UUID | None = None, + db: Session = Depends(get_db), +) -> dict: + return envelope(SegmentationService.list_runs(db, project_id=project_id, dataset_id=dataset_id).model_dump()) + + +@router.get("/runs/{analysis_run_id}", response_model=dict) +def get_segmentation_run(analysis_run_id: UUID, db: Session = Depends(get_db)) -> dict: + return envelope(SegmentationService.get_run(db, analysis_run_id).model_dump()) + + +@router.get("/runs/{analysis_run_id}/segmentations", response_model=dict) +def list_segmentation_run_outputs( + analysis_run_id: UUID, + dataset_id: UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + db: Session = Depends(get_db), +) -> dict: + return envelope( + SegmentationService.list_segmentations( + db, + analysis_run_id=analysis_run_id, + dataset_id=dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ).model_dump() + ) + + +@router.get("/datasets/{dataset_id}/segmentations", response_model=dict) +def list_dataset_segmentations( + dataset_id: UUID, + analysis_run_id: UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + db: Session = Depends(get_db), +) -> dict: + return envelope( + SegmentationService.list_segmentations( + db, + analysis_run_id=analysis_run_id, + dataset_id=dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ).model_dump() + ) + + +@router.get("/segmentations/{segmentation_id}", response_model=dict) +def get_segmentation(segmentation_id: UUID, db: Session = Depends(get_db)) -> dict: + return envelope(SegmentationService.get_segmentation(db, segmentation_id).model_dump()) + + +@router.get("/runs/{analysis_run_id}/geojson", response_model=dict) +def get_segmentation_run_geojson( + analysis_run_id: UUID, + class_name: str | None = None, + min_confidence: float | None = None, + db: Session = Depends(get_db), +) -> dict: + return envelope( + SegmentationService.segmentations_to_geojson( + db, + analysis_run_id=analysis_run_id, + class_name=class_name, + min_confidence=min_confidence, + ) + ) + + +@router.get("/datasets/{dataset_id}/geojson", response_model=dict) +def get_dataset_segmentation_geojson( + dataset_id: UUID, + analysis_run_id: UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + db: Session = Depends(get_db), +) -> dict: + return envelope( + SegmentationService.segmentations_to_geojson( + db, + analysis_run_id=analysis_run_id, + dataset_id=dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ) + ) + + +@router.post("/runs/{analysis_run_id}/qa/reference", response_model=dict) +def compare_segmentation_run_with_reference( + analysis_run_id: UUID, + payload: SegmentationQaRequest, + db: Session = Depends(get_db), +) -> dict: + return envelope( + SegmentationService.compare_segmentations_with_reference( + db=db, + analysis_run_id=analysis_run_id, + reference_dataset_id=payload.reference_dataset_id, + iou_threshold=payload.iou_threshold, + class_name=payload.class_name, + min_confidence=payload.min_confidence, + ) + ) diff --git a/backend/app/core/.gitkeep b/backend/app/core/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/core/config.py b/backend/app/core/config.py new file mode 100644 index 00000000..e9a0a6e7 --- /dev/null +++ b/backend/app/core/config.py @@ -0,0 +1,52 @@ +from pydantic import Field, field_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + populate_by_name=True, + ) + + app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV") + app_version: str = Field(default="0.1.0") + api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX") + database_url: str = Field( + default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1", + validation_alias="DATABASE_URL", + ) + storage_root: str = Field(default="./storage", validation_alias="STORAGE_ROOT") + max_upload_mb: int = Field(default=500, validation_alias="MAX_UPLOAD_MB") + redis_url: str | None = Field(default=None, validation_alias="REDIS_URL") + log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL") + database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS") + yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED") + yolo_model_path: str | None = Field(default=None, validation_alias="YOLO_MODEL_PATH") + yolo_model_id: str = Field(default="yolo-configured", validation_alias="YOLO_MODEL_ID") + yolo_model_display_name: str = Field(default="Configured YOLO detector", validation_alias="YOLO_MODEL_DISPLAY_NAME") + yolo_model_version: str | None = Field(default=None, validation_alias="YOLO_MODEL_VERSION") + yolo_device: str = Field(default="cpu", validation_alias="YOLO_DEVICE") + yolo_image_size: int = Field(default=640, validation_alias="YOLO_IMAGE_SIZE") + yolo_max_tiles: int = Field(default=100, validation_alias="YOLO_MAX_TILES") + yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE") + cors_origins: list[str] | str = Field( + default=["http://localhost:5173", "http://127.0.0.1:5173"], + validation_alias="CORS_ORIGINS", + ) + + @field_validator("cors_origins", mode="before") + @classmethod + def parse_cors_origins(cls, value: object) -> list[str]: + if isinstance(value, str): + return [item.strip() for item in value.split(",") if item.strip()] + if isinstance(value, list): + return value + if value is None: + return ["http://localhost:5173", "http://127.0.0.1:5173"] + return [str(value)] + + +def get_settings() -> Settings: + return Settings() diff --git a/backend/app/core/errors.py b/backend/app/core/errors.py new file mode 100644 index 00000000..4cb1328f --- /dev/null +++ b/backend/app/core/errors.py @@ -0,0 +1,15 @@ +class AppError(Exception): + """Domain error used by services to return canonical API errors.""" + + def __init__( + self, + code: str, + message: str, + details: dict | list | None = None, + status_code: int = 400, + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.details = details or {} + self.status_code = status_code diff --git a/backend/app/core/logging.py b/backend/app/core/logging.py new file mode 100644 index 00000000..dcf74961 --- /dev/null +++ b/backend/app/core/logging.py @@ -0,0 +1,12 @@ +import logging +import sys + + +def configure_logging(level: str = "INFO") -> None: + logging.basicConfig( + level=level, + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + stream=sys.stdout, + ) + for name in ["uvicorn", "uvicorn.error", "uvicorn.access", "sqlalchemy.engine"]: + logging.getLogger(name).setLevel(level) diff --git a/backend/app/db/.gitkeep b/backend/app/db/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/db/__init__.py b/backend/app/db/__init__.py new file mode 100644 index 00000000..2128662c --- /dev/null +++ b/backend/app/db/__init__.py @@ -0,0 +1,4 @@ +from .base import Base +from .session import get_db, get_engine + +__all__ = ["Base", "get_db", "get_engine"] diff --git a/backend/app/db/base.py b/backend/app/db/base.py new file mode 100644 index 00000000..fa2b68a5 --- /dev/null +++ b/backend/app/db/base.py @@ -0,0 +1,5 @@ +from sqlalchemy.orm import DeclarativeBase + + +class Base(DeclarativeBase): + pass diff --git a/backend/app/db/session.py b/backend/app/db/session.py new file mode 100644 index 00000000..bcd413cc --- /dev/null +++ b/backend/app/db/session.py @@ -0,0 +1,20 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, Session + +from app.core.config import get_settings + + +engine = create_engine(get_settings().database_url, pool_pre_ping=True, future=True) +SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False, future=True) + + +def get_db(): + db: Session = SessionLocal() + try: + yield db + finally: + db.close() + + +def get_engine(): + return engine diff --git a/backend/app/geo/.gitkeep b/backend/app/geo/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 00000000..3e5ece17 --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse + +from app.api.routes import areas, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation +from app.core.config import get_settings +from app.core.errors import AppError +from app.core.logging import configure_logging + + +def _to_error_payload(code: str, message: str, details: dict | list | None = None) -> dict: + return { + "error": { + "code": code, + "message": message, + "details": details or {}, + }, + } + + +def create_app() -> FastAPI: + settings = get_settings() + configure_logging(settings.log_level) + + app = FastAPI( + title="GeoIntel Kempen", + version=settings.app_version, + docs_url="/docs", + redoc_url="/redoc", + ) + + app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins, + allow_methods=["*"], + allow_headers=["*"], + allow_credentials=True, + ) + + app.include_router(health.router) + app.include_router(projects.router, prefix=settings.api_prefix) + app.include_router(areas.router, prefix=settings.api_prefix) + app.include_router(datasets.router, prefix=settings.api_prefix) + app.include_router(jobs.router, prefix=settings.api_prefix) + app.include_router(quality_checks.router, prefix=settings.api_prefix) + app.include_router(exports.router, prefix=settings.api_prefix) + app.include_router(external.router, prefix=settings.api_prefix) + app.include_router(demo.router, prefix=settings.api_prefix) + app.include_router(qa.router, prefix=settings.api_prefix) + app.include_router(detection.router, prefix=settings.api_prefix) + app.include_router(segmentation.router, prefix=settings.api_prefix) + + @app.exception_handler(AppError) + async def app_error(request: Request, exc: AppError): # noqa: ARG001 + return JSONResponse( + status_code=exc.status_code, + content=_to_error_payload(exc.code, exc.message, exc.details), + ) + + @app.exception_handler(HTTPException) + async def http_error(request: Request, exc: HTTPException): # noqa: ARG001 + return JSONResponse( + status_code=exc.status_code, + content=_to_error_payload("HTTP_ERROR", str(exc.detail), {}), + ) + + @app.exception_handler(RequestValidationError) + async def validation_error(request: Request, exc: RequestValidationError): # noqa: ARG001 + return JSONResponse( + status_code=422, + content=_to_error_payload("VALIDATION_ERROR", "Validation failed", exc.errors()), + ) + + @app.exception_handler(Exception) + async def unexpected_error(request: Request, exc: Exception): # noqa: ARG001 + return JSONResponse( + status_code=500, + content=_to_error_payload("INTERNAL_ERROR", "Unexpected server error", {"type": exc.__class__.__name__}), + ) + + return app + + +app = create_app() + + +def main() -> None: + import uvicorn + + settings = get_settings() + uvicorn.run( + "app.main:app", + host="0.0.0.0", + port=8000, + reload=settings.app_env == "development", + ) diff --git a/backend/app/models.py b/backend/app/models.py new file mode 100644 index 00000000..5bf5a522 --- /dev/null +++ b/backend/app/models.py @@ -0,0 +1 @@ +from app.models import * diff --git a/backend/app/models/.gitkeep b/backend/app/models/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 00000000..aa87ee4d --- /dev/null +++ b/backend/app/models/__init__.py @@ -0,0 +1,16 @@ +from .entities import AnalysisRun, Area, Dataset, DatasetVersion, Detection, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature + +__all__ = [ + "AnalysisRun", + "Area", + "Dataset", + "DatasetVersion", + "Detection", + "Export", + "Job", + "Metric", + "Project", + "QualityCheck", + "Segmentation", + "VectorFeature", +] diff --git a/backend/app/models/entities.py b/backend/app/models/entities.py new file mode 100644 index 00000000..5d441731 --- /dev/null +++ b/backend/app/models/entities.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from geoalchemy2 import Geometry +from sqlalchemy import DateTime, ForeignKey, Float, Index, JSON, String, Text, func +from sqlalchemy.sql.sqltypes import Integer +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base + + +class Project(Base): + __tablename__ = "projects" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + name: Mapped[str] = mapped_column(String(255), nullable=False) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + region: Mapped[str] = mapped_column(String(120), default="Kempen") + status: Mapped[str] = mapped_column(String(32), default="active") + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + areas: Mapped[list["Area"]] = relationship("Area", back_populates="project", cascade="all, delete-orphan") + datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="project", cascade="all, delete-orphan") + + +class Area(Base): + __tablename__ = "areas" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + name: Mapped[str] = mapped_column(String(255), nullable=False) + geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326), nullable=False) + original_crs: Mapped[str | None] = mapped_column(String(64), nullable=True) + area_m2: Mapped[float | None] = mapped_column(Float, nullable=True) + bbox: Mapped[str | None] = mapped_column(Geometry("Polygon", srid=4326), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + project: Mapped[Project] = relationship("Project", back_populates="areas") + + +class Dataset(Base): + __tablename__ = "datasets" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + area_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("areas.id", ondelete="SET NULL"), nullable=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + dataset_type: Mapped[str] = mapped_column(String(64), nullable=False) + source: Mapped[str] = mapped_column(String(120), nullable=False) + storage_path: Mapped[str | None] = mapped_column(String(500), nullable=True) + original_filename: Mapped[str | None] = mapped_column(String(255), nullable=True) + stored_filename: Mapped[str | None] = mapped_column(String(255), nullable=True) + content_type: Mapped[str | None] = mapped_column(String(120), nullable=True) + size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) + derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), + ForeignKey("datasets.id", ondelete="SET NULL"), + nullable=True, + ) + crs: Mapped[str | None] = mapped_column(String(64), nullable=True) + bounds_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + resolution_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + bands_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + dataset_role: Mapped[str] = mapped_column(String(32), nullable=False, default="source", server_default="source") + source_name: Mapped[str | None] = mapped_column(String(120), nullable=True) + reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True) + source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) + provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) + imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + status: Mapped[str] = mapped_column(String(32), default="uploaded") + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + project: Mapped[Project] = relationship("Project", back_populates="datasets") + versions: Mapped[list["DatasetVersion"]] = relationship( + "DatasetVersion", + back_populates="dataset", + cascade="all, delete-orphan", + ) + vector_features: Mapped[list["VectorFeature"]] = relationship( + "VectorFeature", + back_populates="dataset", + cascade="all, delete-orphan", + ) + + +class DatasetVersion(Base): + __tablename__ = "dataset_versions" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False) + version: Mapped[int] = mapped_column(Integer, default=1) + storage_path: Mapped[str | None] = mapped_column(String(500), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions") + + +class VectorFeature(Base): + __tablename__ = "vector_features" + __table_args__ = ( + Index("ix_vector_features_dataset_id", "dataset_id"), + Index("ix_vector_features_geometry", "geometry", postgresql_using="gist"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False) + feature_class: Mapped[str | None] = mapped_column(String(120), nullable=True) + source_feature_id: Mapped[str | None] = mapped_column(String(255), nullable=True) + properties_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + geometry: Mapped[str] = mapped_column(Geometry("Geometry", srid=4326, spatial_index=False), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + dataset: Mapped[Dataset] = relationship("Dataset", back_populates="vector_features") + + +class AnalysisRun(Base): + __tablename__ = "analysis_runs" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + area_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("areas.id", ondelete="SET NULL"), nullable=True) + dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True) + job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True) + analysis_type: Mapped[str] = mapped_column(String(64), nullable=False) + status: Mapped[str] = mapped_column(String(32), nullable=False) + model_name: Mapped[str | None] = mapped_column(String(255), nullable=True) + model_version: Mapped[str | None] = mapped_column(String(120), nullable=True) + parameters_json: Mapped[dict] = mapped_column(JSON, nullable=False) + result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + + +class Detection(Base): + __tablename__ = "detections" + __table_args__ = ( + Index("ix_detections_project_id", "project_id"), + Index("ix_detections_dataset_id", "dataset_id"), + Index("ix_detections_analysis_run_id", "analysis_run_id"), + Index("ix_detections_class_name", "class_name"), + Index("ix_detections_geometry", "geometry", postgresql_using="gist"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True) + analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True) + job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True) + model_name: Mapped[str] = mapped_column(String(255), nullable=False) + model_version: Mapped[str | None] = mapped_column(String(120), nullable=True) + class_name: Mapped[str] = mapped_column(String(120), nullable=False) + confidence: Mapped[float] = mapped_column(Float, nullable=False) + geometry: Mapped[str] = mapped_column(Geometry("Geometry", srid=4326, spatial_index=False), nullable=False) + bbox_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + source_tile_path: Mapped[str | None] = mapped_column(String(500), nullable=True) + properties_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + +class Segmentation(Base): + __tablename__ = "segmentations" + __table_args__ = ( + Index("ix_segmentations_project_id", "project_id"), + Index("ix_segmentations_dataset_id", "dataset_id"), + Index("ix_segmentations_analysis_run_id", "analysis_run_id"), + Index("ix_segmentations_job_id", "job_id"), + Index("ix_segmentations_class_name", "class_name"), + Index("ix_segmentations_geometry", "geometry", postgresql_using="gist"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True) + job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True) + analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True) + model_name: Mapped[str] = mapped_column(String(255), nullable=False) + model_version: Mapped[str | None] = mapped_column(String(120), nullable=True) + class_name: Mapped[str] = mapped_column(String(120), nullable=False) + confidence: Mapped[float | None] = mapped_column(Float, nullable=True) + geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False) + bbox_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + area_m2: Mapped[float | None] = mapped_column(Float, nullable=True) + mask_path: Mapped[str | None] = mapped_column(Text, nullable=True) + source_tile_path: Mapped[str | None] = mapped_column(String(500), nullable=True) + tile_index: Mapped[int | None] = mapped_column(Integer, nullable=True) + properties_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + provenance_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + +class QualityCheck(Base): + __tablename__ = "quality_checks" + __table_args__ = ( + Index("ix_quality_checks_project_id", "project_id"), + Index("ix_quality_checks_reference_dataset_id", "reference_dataset_id"), + Index("ix_quality_checks_candidate_dataset_id", "candidate_dataset_id"), + Index("ix_quality_checks_analysis_run_id", "analysis_run_id"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True) + analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True) + candidate_dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True) + reference_dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False) + check_type: Mapped[str] = mapped_column(String(120), nullable=False) + status: Mapped[str] = mapped_column(String(32), nullable=False) + score: Mapped[float | None] = mapped_column(Float, nullable=True) + parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + findings_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class Metric(Base): + __tablename__ = "metrics" + __table_args__ = ( + Index("ix_metrics_quality_check_id", "quality_check_id"), + Index("ix_metrics_analysis_run_id", "analysis_run_id"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + quality_check_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("quality_checks.id", ondelete="CASCADE"), nullable=True) + analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True) + metric_key: Mapped[str] = mapped_column(String(120), nullable=False) + metric_value: Mapped[float | None] = mapped_column(Float, nullable=True) + metric_unit: Mapped[str | None] = mapped_column(String(64), nullable=True) + label: Mapped[str | None] = mapped_column(String(120), nullable=True) + metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + +class Export(Base): + __tablename__ = "exports" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True) + export_type: Mapped[str] = mapped_column(String(64), nullable=False) + storage_path: Mapped[str] = mapped_column(String(500), nullable=False) + metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + +class Job(Base): + __tablename__ = "jobs" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + job_type: Mapped[str] = mapped_column(String(128), nullable=False) + status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued") + project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True) + input_dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True) + output_dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True) + parameters_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) diff --git a/backend/app/providers/.gitkeep b/backend/app/providers/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/providers/__init__.py b/backend/app/providers/__init__.py new file mode 100644 index 00000000..be5589dd --- /dev/null +++ b/backend/app/providers/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from app.providers import base, fixture, grb, manual, osm, registry + +__all__ = ["base", "fixture", "grb", "manual", "osm", "registry"] diff --git a/backend/app/providers/base.py b/backend/app/providers/base.py new file mode 100644 index 00000000..da343554 --- /dev/null +++ b/backend/app/providers/base.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class ProviderCapability: + provider_name: str + display_name: str + authority_level: str + supported_layers: list[str] + supported_geometry_types: list[str] + supported_query_modes: list[str] + fetch_signature: str + configured: bool + status: str + limitation_message: str + attribution: str + license_note: str + not_configured_reason: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "provider_name": self.provider_name, + "display_name": self.display_name, + "authority_level": self.authority_level, + "supported_layers": self.supported_layers, + "supported_geometry_types": self.supported_geometry_types, + "supported_query_modes": self.supported_query_modes, + "fetch_signature": self.fetch_signature, + "configured": self.configured, + "status": self.status, + "limitation_message": self.limitation_message, + "attribution": self.attribution, + "license_note": self.license_note, + "not_configured_reason": self.not_configured_reason, + } + + +class BaseReferenceProvider: + def __init__( + self, + provider_name: str, + display_name: str, + authority_level: str, + supported_layers: list[str], + supported_geometry_types: list[str], + supported_query_modes: list[str], + fetch_signature: str, + limitation_message: str, + attribution: str, + license_note: str, + configured: bool = False, + ) -> None: + self.provider_name = provider_name + self.display_name = display_name + self.authority_level = authority_level + self.supported_layers = supported_layers + self.supported_geometry_types = supported_geometry_types + self.supported_query_modes = supported_query_modes + self.fetch_signature = fetch_signature + self.limitation_message = limitation_message + self.attribution = attribution + self.license_note = license_note + self._configured = configured + + @property + def capability(self) -> ProviderCapability: + return ProviderCapability( + provider_name=self.provider_name, + display_name=self.display_name, + authority_level=self.authority_level, + supported_layers=self.supported_layers, + supported_geometry_types=self.supported_geometry_types, + supported_query_modes=self.supported_query_modes, + fetch_signature=self.fetch_signature, + configured=self.is_configured, + status="configured" if self.is_configured else "not_configured", + limitation_message=self.limitation_message, + attribution=self.attribution, + license_note=self.license_note, + not_configured_reason=None if self.is_configured else "Provider integration is not configured yet", + ) + + @property + def is_configured(self) -> bool: + return self._configured + + def fetch(self, project_id: str, area_id: str | None, layers: list[str]) -> dict[str, Any]: + del project_id, area_id, layers + return { + "provider": self.provider_name, + "status": "not_configured", + "message": "Provider integration is not configured yet", + } diff --git a/backend/app/providers/fixture.py b/backend/app/providers/fixture.py new file mode 100644 index 00000000..bdc00c07 --- /dev/null +++ b/backend/app/providers/fixture.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from app.providers.base import BaseReferenceProvider + + +class FixtureProvider(BaseReferenceProvider): + def __init__(self) -> None: + super().__init__( + provider_name="fixture", + display_name="Fixture data", + authority_level="fixture", + supported_layers=["buildings", "roads", "water", "landuse", "custom"], + supported_geometry_types=["Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon"], + supported_query_modes=["fixture"], + fetch_signature="tests/fixtures and demo fixture upload flow", + limitation_message="Fixture provider represents local test/demo fixtures only.", + attribution="GeoIntel local fixtures", + license_note="Fixtures are for local development and tests; do not present them as official data.", + configured=True, + ) diff --git a/backend/app/providers/grb.py b/backend/app/providers/grb.py new file mode 100644 index 00000000..5773c0ef --- /dev/null +++ b/backend/app/providers/grb.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from app.providers.base import BaseReferenceProvider + + +class GRBProvider(BaseReferenceProvider): + def __init__(self) -> None: + super().__init__( + provider_name="grb", + display_name="GRB", + authority_level="authoritative", + supported_layers=["buildings", "roads", "parcels"], + supported_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"], + supported_query_modes=["area"], + fetch_signature="POST /api/v1/external/grb/fetch", + limitation_message="GRB live WFS/download integration is not configured in Sprint 7B.", + attribution="Digitaal Vlaanderen - Basiskaart Vlaanderen (GRB)", + license_note="Use must follow Digitaal Vlaanderen open data and attribution terms.", + configured=False, + ) diff --git a/backend/app/providers/manual.py b/backend/app/providers/manual.py new file mode 100644 index 00000000..3057919a --- /dev/null +++ b/backend/app/providers/manual.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from app.providers.base import BaseReferenceProvider + + +class ManualProvider(BaseReferenceProvider): + def __init__(self) -> None: + super().__init__( + provider_name="manual", + display_name="Manual upload", + authority_level="manual", + supported_layers=["buildings", "roads", "water", "landuse", "custom"], + supported_geometry_types=["Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon"], + supported_query_modes=["upload"], + fetch_signature="POST /api/v1/projects/{project_id}/datasets/upload", + limitation_message="Manual provider data is supplied through the existing dataset upload flow.", + attribution="User supplied", + license_note="License and attribution must be supplied by the uploader in source metadata.", + configured=True, + ) diff --git a/backend/app/providers/osm.py b/backend/app/providers/osm.py new file mode 100644 index 00000000..d5e91790 --- /dev/null +++ b/backend/app/providers/osm.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from app.providers.base import BaseReferenceProvider + + +class OSMProvider(BaseReferenceProvider): + def __init__(self) -> None: + super().__init__( + provider_name="osm", + display_name="OpenStreetMap", + authority_level="contextual", + supported_layers=["buildings", "roads", "water", "landuse"], + supported_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"], + supported_query_modes=["area"], + fetch_signature="POST /api/v1/external/osm/fetch", + limitation_message="OSM live Overpass/download integration is not configured in Sprint 7B.", + attribution="OpenStreetMap contributors", + license_note="OpenStreetMap data is available under ODbL; attribution is required.", + configured=False, + ) diff --git a/backend/app/providers/registry.py b/backend/app/providers/registry.py new file mode 100644 index 00000000..44d7b0cc --- /dev/null +++ b/backend/app/providers/registry.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +from pydantic import BaseModel + +from app.core.errors import AppError +from app.providers.base import ProviderCapability +from app.providers.fixture import FixtureProvider +from app.providers.grb import GRBProvider +from app.providers.manual import ManualProvider +from app.providers.osm import OSMProvider + + +class ProviderDatasetMapping(BaseModel): + provider_name: str + dataset_role: str + source_name: str + reference_required: bool + write_path: str = "DatasetService" + + +class ProviderImportResult(BaseModel): + provider_name: str + status: str + message: str + requested_layers: list[str] + dataset_id: str | None = None + dataset_role: str | None = None + source_name: str | None = None + + +class ExternalProviderRegistry: + def __init__(self) -> None: + self.providers = { + "grb": GRBProvider(), + "osm": OSMProvider(), + "manual": ManualProvider(), + "fixture": FixtureProvider(), + } + + def list_capabilities(self) -> list[ProviderCapability]: + return [provider.capability for provider in self.providers.values()] + + def get(self, provider_name: str): + normalized = provider_name.strip().lower() + if normalized not in self.providers: + raise AppError(code="PROVIDER_NOT_FOUND", message="Provider not found", status_code=404) + return self.providers[normalized] + + def fetch(self, provider_name: str, project_id: str, area_id: str | None, layers: list[str]) -> dict: + provider = self.get(provider_name) + return provider.fetch(project_id=project_id, area_id=area_id, layers=layers) + + def dataset_mapping(self, provider_name: str, requested_dataset_role: str | None = None) -> ProviderDatasetMapping: + provider = self.get(provider_name) + if provider.provider_name == "osm": + dataset_role = "reference" if requested_dataset_role == "reference" else "source" + return ProviderDatasetMapping( + provider_name="osm", + dataset_role=dataset_role, + source_name="osm", + reference_required=requested_dataset_role == "reference", + ) + return ProviderDatasetMapping( + provider_name=provider.provider_name, + dataset_role="reference", + source_name=provider.provider_name, + reference_required=True, + ) + + def import_contract( + self, + provider_name: str, + project_id: str, + area_id: str | None, + layers: list[str], + requested_dataset_role: str | None = None, + ) -> ProviderImportResult: + del project_id, area_id + provider = self.get(provider_name) + mapping = self.dataset_mapping(provider.provider_name, requested_dataset_role=requested_dataset_role) + if provider.provider_name in {"grb", "osm"}: + return ProviderImportResult( + provider_name=provider.provider_name, + status="not_configured", + message=f"No live {provider.display_name} import is configured in Sprint 7B.", + requested_layers=layers, + dataset_role=mapping.dataset_role, + source_name=mapping.source_name, + ) + if provider.provider_name == "manual": + return ProviderImportResult( + provider_name="manual", + status="upload_flow_required", + message="Manual provider data must use the existing dataset upload/reference flow.", + requested_layers=layers, + dataset_role=mapping.dataset_role, + source_name=mapping.source_name, + ) + return ProviderImportResult( + provider_name="fixture", + status="fixture_flow_required", + message="Fixture provider data must use checked-in demo/test fixture flows.", + requested_layers=layers, + dataset_role=mapping.dataset_role, + source_name=mapping.source_name, + ) + + +_registry = ExternalProviderRegistry() + + +def list_provider_capabilities() -> list[ProviderCapability]: + return _registry.list_capabilities() + + +def get_provider(provider_name: str): + return _registry.get(provider_name) + + +def fetch_provider_data(provider_name: str, project_id: str, area_id: str | None, layers: list[str]) -> dict: + return _registry.fetch(provider_name, project_id, area_id, layers) + + +def get_provider_dataset_mapping(provider_name: str, requested_dataset_role: str | None = None) -> ProviderDatasetMapping: + return _registry.dataset_mapping(provider_name, requested_dataset_role=requested_dataset_role) + + +def import_provider_dataset( + provider_name: str, + project_id: str, + area_id: str | None, + layers: list[str], + requested_dataset_role: str | None = None, +) -> ProviderImportResult: + return _registry.import_contract( + provider_name=provider_name, + project_id=project_id, + area_id=area_id, + layers=layers, + requested_dataset_role=requested_dataset_role, + ) diff --git a/backend/app/repositories/.gitkeep b/backend/app/repositories/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/schemas/.gitkeep b/backend/app/schemas/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py new file mode 100644 index 00000000..56015829 --- /dev/null +++ b/backend/app/schemas/__init__.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from .common import ApiErrorEnvelope, ApiErrorItem, Envelope, PaginationEnvelope +from .project import ProjectCreate, ProjectList, ProjectRead, ProjectUpdate +from .area import AreaCreate, AreaList, AreaRead, AreaUpdate +from .dataset import DatasetCreateResponse, DatasetList +from .detection import ( + DetectionListResponse, + DetectionModelCapability, + DetectionModelsResponse, + DetectionQaRequest, + DetectionRead, + DetectionRunListResponse, + DetectionRunRead, + DetectionRunRequest, + DetectionRunResponse, +) +from .segmentation import ( + SegmentationListResponse, + SegmentationModelCapability, + SegmentationModelsResponse, + SegmentationQaRequest, + SegmentationRead, + SegmentationRunListResponse, + SegmentationRunRead, + SegmentationRunRequest, + SegmentationRunResponse, +) +from .health import HealthResponse, SystemCapabilities +from .job import JobCreate, JobList, JobRead, JobStatus +from .external import ( + ExternalFetchRequest, + ExternalFetchResponse, + ProviderCapabilitiesResponse, + ProviderCapabilityResponse, + ProviderImportRequest, + ProviderImportResponse, + ProviderLayersResponse, + ProviderStatusResponse, +) +from .export import ( + ExportContentResponse, + ExportCreateResponse, + ExportListResponse, + ExportRead, + GeoJsonExportRequest, + MetadataExportRequest, + ReportExportRequest, +) +from .qa import QaProviderComparisonRequest, QaProviderComparisonResult +from .operations import ( + RasterClipRequest, + RasterIndexBaseRequest, + RasterMetadataResponse, + RasterNdviRequest, + RasterNdwiRequest, + RasterNdbiRequest, + RasterOperationResult, + RasterPreviewResponse, + RasterReprojectRequest, + RasterReprojectResponse, + RasterStatsResponse, + RasterTileManifest, + RasterTileManifestTile, + RasterTileRequest, + RasterTileResponse, + VectorBBoxResponse, + VectorBufferRequest, + VectorClipRequest, + VectorIntersectRequest, + VectorOperationRequest, + VectorOperationResult, + VectorStatsRequest, + VectorStatsResponse, +) + +__all__ = [ + "Envelope", + "ApiErrorEnvelope", + "ApiErrorItem", + "PaginationEnvelope", + "ProjectCreate", + "ProjectRead", + "ProjectUpdate", + "ProjectList", + "AreaCreate", + "AreaRead", + "AreaUpdate", + "AreaList", + "DatasetCreateResponse", + "DatasetList", + "DetectionListResponse", + "DetectionModelCapability", + "DetectionModelsResponse", + "DetectionQaRequest", + "DetectionRead", + "DetectionRunListResponse", + "DetectionRunRead", + "DetectionRunRequest", + "DetectionRunResponse", + "SegmentationListResponse", + "SegmentationModelCapability", + "SegmentationModelsResponse", + "SegmentationQaRequest", + "SegmentationRead", + "SegmentationRunListResponse", + "SegmentationRunRead", + "SegmentationRunRequest", + "SegmentationRunResponse", + "HealthResponse", + "SystemCapabilities", + "JobCreate", + "JobList", + "JobRead", + "JobStatus", + "VectorBBoxResponse", + "VectorClipRequest", + "VectorBufferRequest", + "VectorIntersectRequest", + "VectorOperationRequest", + "VectorOperationResult", + "RasterClipRequest", + "RasterStatsResponse", + "RasterReprojectRequest", + "RasterReprojectResponse", + "RasterTileRequest", + "RasterMetadataResponse", + "RasterOperationResult", + "RasterPreviewResponse", + "RasterTileManifestTile", + "RasterTileManifest", + "RasterTileResponse", + "RasterIndexBaseRequest", + "RasterNdviRequest", + "RasterNdwiRequest", + "RasterNdbiRequest", + "VectorStatsRequest", + "VectorStatsResponse", + "ExternalFetchRequest", + "ExternalFetchResponse", + "ProviderCapabilitiesResponse", + "ProviderCapabilityResponse", + "ProviderImportRequest", + "ProviderImportResponse", + "ProviderLayersResponse", + "ProviderStatusResponse", + "GeoJsonExportRequest", + "MetadataExportRequest", + "ReportExportRequest", + "ExportRead", + "ExportCreateResponse", + "ExportListResponse", + "ExportContentResponse", + "QaProviderComparisonRequest", + "QaProviderComparisonResult", +] diff --git a/backend/app/schemas/area.py b/backend/app/schemas/area.py new file mode 100644 index 00000000..de801218 --- /dev/null +++ b/backend/app/schemas/area.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel + + +class AreaCreate(BaseModel): + name: str + geometry: dict + crs: str | None = "EPSG:4326" + + +class AreaUpdate(BaseModel): + name: str | None = None + crs: str | None = None + + +class AreaRead(BaseModel): + id: UUID + project_id: UUID + name: str + original_crs: str | None + area_m2: float | None + created_at: datetime | None = None + geometry_type: str | None = None + + model_config = {"from_attributes": True} + + +class AreaListItem(AreaRead): + pass + + +class AreaList(BaseModel): + items: list[AreaRead] + total: int + limit: int + offset: int diff --git a/backend/app/schemas/common.py b/backend/app/schemas/common.py new file mode 100644 index 00000000..7640c741 --- /dev/null +++ b/backend/app/schemas/common.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class Envelope(BaseModel): + data: object + + +class PaginatedEnvelope(BaseModel): + items: list + total: int + limit: int + offset: int + + +class PaginationEnvelope(BaseModel): + items: list + total: int + limit: int = Field(default=50) + offset: int = Field(default=0) + + +class ApiErrorItem(BaseModel): + code: str + message: str + details: dict = Field(default_factory=dict) + + +class ApiErrorEnvelope(BaseModel): + error: ApiErrorItem diff --git a/backend/app/schemas/dataset.py b/backend/app/schemas/dataset.py new file mode 100644 index 00000000..c52d5ed1 --- /dev/null +++ b/backend/app/schemas/dataset.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel + + +class DatasetStorageResponse(BaseModel): + original_filename: str | None = None + stored_filename: str | None = None + content_type: str | None = None + size_bytes: int | None = None + checksum_sha256: str | None = None + + +class DatasetVectorSummary(BaseModel): + feature_count: int | None = None + geometry_types: list[str] | None = None + bounds_json: dict | None = None + approximate_area_m2: float | None = None + crs: str | None = None + feature_geometry_count: int | None = None + invalid_features: int | None = None + crs_assumed: bool | None = None + + +class DatasetCreateResponse(BaseModel): + id: UUID + name: str + dataset_type: str + source: str + dataset_role: str = "source" + source_name: str | None = None + reference_layer_name: str | None = None + source_metadata: dict | None = None + provenance_metadata: dict | None = None + imported_at: datetime | None = None + project_id: UUID + area_id: UUID | None = None + storage_path: str | None = None + original_filename: str | None = None + stored_filename: str | None = None + content_type: str | None = None + size_bytes: int | None = None + checksum_sha256: str | None = None + crs: str | None = None + bounds_json: dict | None = None + metadata_json: dict | None = None + vector_summary: DatasetVectorSummary | None = None + status: str + derived_from_dataset_id: UUID | None = None + created_at: datetime | None = None + feature_count: int | None = None + + model_config = {"from_attributes": True} + + +class DatasetList(BaseModel): + items: list[DatasetCreateResponse] + total: int + limit: int + offset: int + + +class DatasetMetadataRefresh(BaseModel): + feature_count: int | None = None + geometry_types: list[str] | None = None + bounds_json: dict | None = None + crs: str | None = None + + +class ExportRequest(BaseModel): + dataset_id: UUID + name: str | None = None + + +class ExportRead(BaseModel): + export_id: UUID + path: str + status: str diff --git a/backend/app/schemas/demo.py b/backend/app/schemas/demo.py new file mode 100644 index 00000000..494ef38d --- /dev/null +++ b/backend/app/schemas/demo.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from uuid import UUID + +from pydantic import BaseModel + + +class DemoWorkflowResponse(BaseModel): + project_id: UUID + area_id: UUID + reference_dataset_id: UUID + candidate_dataset_id: UUID + quality_check_id: UUID + metric_count: int + status: str + message: str + created: bool diff --git a/backend/app/schemas/detection.py b/backend/app/schemas/detection.py new file mode 100644 index 00000000..736c5540 --- /dev/null +++ b/backend/app/schemas/detection.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, Field + + +class DetectionModelCapability(BaseModel): + model_id: str + display_name: str + framework: str + task_type: str + supported_classes: list[str] + configured: bool + status: str + limitation_message: str + version: str | None = None + + +class DetectionModelsResponse(BaseModel): + models: list[DetectionModelCapability] + + +class DetectionRunRequest(BaseModel): + project_id: UUID + dataset_id: UUID + model_id: str + confidence_threshold: float = Field(default=0.5, ge=0.0, le=1.0) + class_filter: list[str] | None = None + tile_manifest_path: str | None = None + parameters_json: dict = Field(default_factory=dict) + + +class DetectionQaRequest(BaseModel): + reference_dataset_id: UUID + iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0) + class_name: str | None = None + min_confidence: float | None = Field(default=None, ge=0.0, le=1.0) + + +class DetectionRunResponse(BaseModel): + analysis_run_id: UUID + job_id: UUID + project_id: UUID + dataset_id: UUID + model_id: str + status: str + detection_count: int + error_code: str | None = None + message: str + + +class DetectionRunRead(BaseModel): + id: UUID + project_id: UUID + dataset_id: UUID | None = None + job_id: UUID | None = None + analysis_type: str + status: str + model_name: str | None = None + model_version: str | None = None + parameters_json: dict + result_json: dict | None = None + error_message: str | None = None + created_at: datetime | None = None + started_at: datetime | None = None + finished_at: datetime | None = None + + model_config = {"from_attributes": True} + + +class DetectionRunListResponse(BaseModel): + items: list[DetectionRunRead] + total: int + + +class DetectionRead(BaseModel): + id: UUID + project_id: UUID + dataset_id: UUID | None = None + analysis_run_id: UUID | None = None + job_id: UUID | None = None + model_name: str + model_version: str | None = None + class_name: str + confidence: float + bbox_json: dict | None = None + source_tile_path: str | None = None + properties_json: dict | None = None + created_at: datetime | None = None + + model_config = {"from_attributes": True} + + +class DetectionListResponse(BaseModel): + items: list[DetectionRead] + total: int diff --git a/backend/app/schemas/export.py b/backend/app/schemas/export.py new file mode 100644 index 00000000..a32dea66 --- /dev/null +++ b/backend/app/schemas/export.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, model_validator + + +ExportKind = Literal["dataset", "detection_run", "segmentation_run"] + + +class GeoJsonExportRequest(BaseModel): + dataset_id: UUID | None = None + analysis_run_id: UUID | None = None + export_kind: ExportKind = "dataset" + name: str | None = None + + @model_validator(mode="after") + def validate_target(self) -> "GeoJsonExportRequest": + if self.export_kind == "dataset" and self.dataset_id is None: + raise ValueError("dataset_id is required for dataset GeoJSON exports") + if self.export_kind in {"detection_run", "segmentation_run"} and self.analysis_run_id is None: + raise ValueError("analysis_run_id is required for run GeoJSON exports") + return self + + +class MetadataExportRequest(BaseModel): + project_id: UUID + name: str | None = None + + +class ReportExportRequest(BaseModel): + project_id: UUID + name: str | None = None + + +class ExportRead(BaseModel): + id: UUID + project_id: UUID + analysis_run_id: UUID | None = None + export_type: str + storage_path: str + metadata_json: dict | None = None + created_at: datetime | None = None + status: str = "ready" + + model_config = {"from_attributes": True} + + +class ExportCreateResponse(BaseModel): + export_id: UUID + path: str + status: str + export_type: str + metadata_json: dict | None = None + + +class ExportListResponse(BaseModel): + items: list[ExportRead] + total: int + limit: int + offset: int + + +class ExportContentResponse(BaseModel): + export_id: UUID + export_type: str + content: dict diff --git a/backend/app/schemas/external.py b/backend/app/schemas/external.py new file mode 100644 index 00000000..dfed323e --- /dev/null +++ b/backend/app/schemas/external.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from uuid import UUID +from pydantic import BaseModel + + +class ProviderCapabilityResponse(BaseModel): + provider_name: str + display_name: str + authority_level: str + supported_layers: list[str] + supported_geometry_types: list[str] + supported_query_modes: list[str] + fetch_signature: str + configured: bool + status: str + limitation_message: str + attribution: str + license_note: str + not_configured_reason: str | None = None + + +class ProviderCapabilitiesResponse(BaseModel): + providers: list[ProviderCapabilityResponse] + + +class ProviderLayersResponse(BaseModel): + provider_name: str + layers: list[str] + + +class ProviderStatusResponse(BaseModel): + provider_name: str + configured: bool + status: str + limitation_message: str + + +class ExternalFetchRequest(BaseModel): + project_id: UUID + area_id: UUID | None = None + layers: list[str] = [] + + +class ExternalFetchResponse(BaseModel): + provider: str + status: str + message: str + requested_layers: list[str] + project_id: UUID + area_id: UUID | None = None + + +class ProviderImportRequest(BaseModel): + project_id: str + area_id: str | None = None + layers: list[str] = [] + dataset_role: str | None = None + + +class ProviderImportResponse(BaseModel): + provider_name: str + status: str + message: str + requested_layers: list[str] + dataset_id: str | None = None + dataset_role: str | None = None + source_name: str | None = None diff --git a/backend/app/schemas/health.py b/backend/app/schemas/health.py new file mode 100644 index 00000000..67924441 --- /dev/null +++ b/backend/app/schemas/health.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class ProviderCapability(BaseModel): + provider_name: str + display_name: str + authority_level: str + supported_layers: list[str] + supported_geometry_types: list[str] + supported_query_modes: list[str] + fetch_signature: str + configured: bool + status: str + limitation_message: str + attribution: str + license_note: str + not_configured_reason: str | None = None + + +class HealthResponse(BaseModel): + status: str + service: str + version: str + database: str | None = None + + +class SystemCapabilities(BaseModel): + postgis: bool + rasterio: bool + geopandas: bool + yolo: bool | str + sam: bool | str + grb: str + sentinel: str + providers: list[ProviderCapability] = Field(default_factory=list) diff --git a/backend/app/schemas/job.py b/backend/app/schemas/job.py new file mode 100644 index 00000000..2d1a2bfc --- /dev/null +++ b/backend/app/schemas/job.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, Field + + +class JobCreate(BaseModel): + job_type: str + project_id: UUID + dataset_id: UUID | None = None + input_dataset_id: UUID | None = None + output_dataset_id: UUID | None = None + parameters_json: dict = Field(default_factory=dict) + + +class JobRead(BaseModel): + id: UUID + job_type: str + status: str + project_id: UUID + dataset_id: UUID | None = None + input_dataset_id: UUID | None = None + output_dataset_id: UUID | None = None + parameters_json: dict + result_json: dict | None = None + error_message: str | None = None + created_at: datetime | None = None + started_at: datetime | None = None + finished_at: datetime | None = None + + model_config = {"from_attributes": True} + + +class JobStatus(BaseModel): + id: UUID + status: str + error_message: str | None = None + started_at: datetime | None = None + finished_at: datetime | None = None + result_json: dict | None = None + + model_config = {"from_attributes": True} + + +class JobList(BaseModel): + items: list[JobRead] + total: int + limit: int + offset: int diff --git a/backend/app/schemas/operations.py b/backend/app/schemas/operations.py new file mode 100644 index 00000000..7d329dd5 --- /dev/null +++ b/backend/app/schemas/operations.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +from pydantic import BaseModel + + +class VectorOperationResult(BaseModel): + feature_count: int + geometry_type_summary: dict[str, int] + bounds_json: dict | None = None + crs: str | None = None + source_dataset_id: str + + +class VectorOperationRequest(BaseModel): + output_name: str | None = None + + +class VectorClipRequest(VectorOperationRequest): + area_id: str + + +class VectorBufferRequest(VectorOperationRequest): + distance_m: float + dissolve: bool = False + + +class VectorIntersectRequest(VectorOperationRequest): + other_dataset_id: str + + +class VectorStatsRequest(BaseModel): + pass + + +class RasterReadyResponse(BaseModel): + dataset_id: str + ready: bool + message: str | None = None + + +class RasterOperationResult(BaseModel): + dataset_id: str + ready: bool + metadata: dict | None = None + output_dataset_id: str | None = None + operation: str | None = None + + +class RasterMetadataResponse(BaseModel): + dataset_id: str + driver: str | None = None + width: int | None = None + height: int | None = None + band_count: int | None = None + crs: str | None = None + bounds: list[float] | None = None + resolution: list[float] | None = None + dtype: list[str] | None = None + nodata: list[float] | float | None = None + transform: list[float] | None = None + size_bytes: int | None = None + checksum_sha256: str | None = None + path: str | None = None + + +class RasterPreviewResponse(BaseModel): + dataset_id: str + ready: bool + preview: dict + metadata: dict | None = None + + +class RasterBandStats(BaseModel): + band_index: int + dtype: str | None = None + min: float | None = None + max: float | None = None + mean: float | None = None + std: float | None = None + nodata_count: int + nodata_ratio: float + valid_pixel_count: int + histogram: list[int] | None = None + histogram_bins: list[float] | None = None + + +class RasterStatsResponse(BaseModel): + dataset_id: str + source_dataset_id: str | None = None + bands: list[RasterBandStats] + generated_at: str | None = None + metadata: dict | None = None + + +class RasterReprojectRequest(BaseModel): + target_crs: str | None = "EPSG:31370" + resampling: str = "nearest" + output_name: str | None = None + + +class RasterClipRequest(BaseModel): + area_id: str + output_name: str | None = None + + +class RasterTileRequest(BaseModel): + tile_size: int = 512 + overlap: int = 64 + output_name: str | None = None + + +class RasterIndexBaseRequest(BaseModel): + output_name: str | None = None + + +class RasterNdviRequest(RasterIndexBaseRequest): + nir_band: int + red_band: int + + +class RasterNdwiRequest(RasterIndexBaseRequest): + green_band: int + nir_band: int + + +class RasterNdbiRequest(RasterIndexBaseRequest): + swir_band: int + nir_band: int + + +class RasterTileManifestTile(BaseModel): + path: str + pixel_window: list[int] + bounds: list[float] + transform: list[float] + index: int + + +class RasterTileManifest(BaseModel): + tile_set_id: str + source_dataset_id: str + source_raster_id: str + bounds: list[float] + tile_size: int + overlap: int + parameters: dict[str, str | int | float | bool | None] + created_at: str + tile_paths: list[str] + count: int + tiles: list[RasterTileManifestTile] + ai_inference: bool = False + tile_server: str | None = None + + +class RasterTileResponse(BaseModel): + dataset_id: str + ready: bool + operation: str + tile_set_id: str + tile_size: int + overlap: int + manifest_path: str + count: int + manifest: RasterTileManifest + + +class RasterReprojectResponse(BaseModel): + dataset_id: str + ready: bool + operation: str + output_dataset_id: str + source_dataset_id: str + target_dataset_id: str | None = None + + +class RasterOperationUnavailable(BaseModel): + code: str + message: str + + +class VectorBBoxResponse(BaseModel): + dataset_id: str + bounds_json: dict | None + feature_count: int + crs: str | None = None + + +class VectorStatsResponse(BaseModel): + dataset_id: str + feature_count: int + geometry_type_summary: dict[str, int] + bounds_json: dict | None + crs: str | None = None diff --git a/backend/app/schemas/project.py b/backend/app/schemas/project.py new file mode 100644 index 00000000..4ec7d082 --- /dev/null +++ b/backend/app/schemas/project.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel + + +class ProjectCreate(BaseModel): + name: str + description: str | None = None + region: str | None = "Kempen" + + +class ProjectUpdate(BaseModel): + name: str | None = None + description: str | None = None + region: str | None = None + + +class ProjectRead(BaseModel): + id: UUID + name: str + description: str | None = None + region: str + status: str + created_at: datetime | None = None + updated_at: datetime | None = None + + model_config = {"from_attributes": True} + + +class ProjectListItem(ProjectRead): + pass + + +class ProjectList(BaseModel): + items: list[ProjectRead] + total: int + limit: int + offset: int diff --git a/backend/app/schemas/qa.py b/backend/app/schemas/qa.py new file mode 100644 index 00000000..17ce192e --- /dev/null +++ b/backend/app/schemas/qa.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, Field + + +class QaProviderComparisonRequest(BaseModel): + candidate_dataset_id: UUID + reference_dataset_id: UUID + iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0) + area_id: UUID | None = None + + +class QaProviderComparisonResult(BaseModel): + status: str + warnings: list[str] = Field(default_factory=list) + candidate_feature_count: int + reference_feature_count: int + matches: int + false_positives: int + false_negatives: int + precision: float | None + recall: float | None + f1_score: float | None + mean_iou: float | None + iou_threshold: float + unsupported_geometry: bool = False + unsupported_geometries: list[str] = Field(default_factory=list) + generated_at: datetime + + +class MetricRead(BaseModel): + id: UUID + quality_check_id: UUID | None = None + analysis_run_id: UUID | None = None + metric_key: str + metric_value: float | None = None + metric_unit: str | None = None + label: str | None = None + metadata_json: dict | None = None + created_at: datetime | None = None + + model_config = {"from_attributes": True} + + +class QualityCheckRead(BaseModel): + id: UUID + project_id: UUID + job_id: UUID | None = None + analysis_run_id: UUID | None = None + candidate_dataset_id: UUID | None = None + reference_dataset_id: UUID + check_type: str + status: str + score: float | None = None + parameters_json: dict | None = None + findings_json: dict | None = None + created_at: datetime | None = None + completed_at: datetime | None = None + metrics: list[MetricRead] = Field(default_factory=list) + + model_config = {"from_attributes": True} + + +class QualityCheckList(BaseModel): + items: list[QualityCheckRead] + total: int + limit: int + offset: int diff --git a/backend/app/schemas/segmentation.py b/backend/app/schemas/segmentation.py new file mode 100644 index 00000000..8f66a6e2 --- /dev/null +++ b/backend/app/schemas/segmentation.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, Field + +from app.schemas.detection import DetectionModelCapability + + +SegmentationModelCapability = DetectionModelCapability + + +class SegmentationModelsResponse(BaseModel): + models: list[SegmentationModelCapability] + + +class SegmentationRunRequest(BaseModel): + project_id: UUID + dataset_id: UUID + model_id: str + confidence_threshold: float = Field(default=0.5, ge=0.0, le=1.0) + class_filter: list[str] | None = None + tile_manifest_path: str | None = None + parameters_json: dict = Field(default_factory=dict) + + +class SegmentationQaRequest(BaseModel): + reference_dataset_id: UUID + iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0) + class_name: str | None = None + min_confidence: float | None = Field(default=None, ge=0.0, le=1.0) + + +class SegmentationRunResponse(BaseModel): + analysis_run_id: UUID + job_id: UUID + project_id: UUID + dataset_id: UUID + model_id: str + status: str + segmentation_count: int + error_code: str | None = None + message: str + + +class SegmentationRunRead(BaseModel): + id: UUID + project_id: UUID + dataset_id: UUID | None = None + job_id: UUID | None = None + analysis_type: str + status: str + model_name: str | None = None + model_version: str | None = None + parameters_json: dict + result_json: dict | None = None + error_message: str | None = None + created_at: datetime | None = None + started_at: datetime | None = None + finished_at: datetime | None = None + + model_config = {"from_attributes": True} + + +class SegmentationRunListResponse(BaseModel): + items: list[SegmentationRunRead] + total: int + + +class SegmentationRead(BaseModel): + id: UUID + project_id: UUID + dataset_id: UUID | None = None + analysis_run_id: UUID | None = None + job_id: UUID | None = None + model_name: str + model_version: str | None = None + class_name: str + confidence: float | None = None + bbox_json: dict | None = None + area_m2: float | None = None + mask_path: str | None = None + source_tile_path: str | None = None + tile_index: int | None = None + properties_json: dict | None = None + provenance_json: dict | None = None + created_at: datetime | None = None + + model_config = {"from_attributes": True} + + +class SegmentationListResponse(BaseModel): + items: list[SegmentationRead] + total: int diff --git a/backend/app/services/.gitkeep b/backend/app/services/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/services/area_service.py b/backend/app/services/area_service.py new file mode 100644 index 00000000..fada557f --- /dev/null +++ b/backend/app/services/area_service.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import uuid + +from sqlalchemy.orm import Session +from geoalchemy2.shape import from_shape + +from app.core.errors import AppError +from app.models import Area, Project +from app.schemas.area import AreaCreate, AreaRead, AreaUpdate +from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon + + +class AreaService: + @staticmethod + def list_areas(db: Session, project_id: uuid.UUID, limit: int = 50, offset: int = 0) -> tuple[list[AreaRead], int]: + total = db.query(Area).filter(Area.project_id == project_id).count() + areas = ( + db.query(Area) + .filter(Area.project_id == project_id) + .order_by(Area.created_at.desc()) + .offset(offset) + .limit(limit) + .all() + ) + return [AreaRead.model_validate(area) for area in areas], total + + @staticmethod + def create_area(db: Session, project_id: uuid.UUID, payload: AreaCreate) -> AreaRead: + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + + try: + multipolygon = normalize_to_multipolygon(payload.geometry) + except ValueError as exc: + raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc + + area = Area( + project_id=project_id, + name=payload.name.strip() or "Unnamed area", + geometry=from_shape(multipolygon, srid=4326), + original_crs=payload.crs or "EPSG:4326", + area_m2=area_m2(multipolygon), + bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326), + ) + db.add(area) + db.commit() + db.refresh(area) + return AreaRead.model_validate(area) + + @staticmethod + def get_area(db: Session, area_id: uuid.UUID) -> AreaRead: + area = db.get(Area, area_id) + if not area: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + return AreaRead.model_validate(area) + + @staticmethod + def update_area(db: Session, area_id: uuid.UUID, payload: AreaUpdate) -> AreaRead: + area = db.get(Area, area_id) + if not area: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + + changed = False + if payload.name: + area.name = payload.name.strip() or area.name + changed = True + if payload.crs: + area.original_crs = payload.crs + changed = True + if not changed: + raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422) + + db.add(area) + db.commit() + db.refresh(area) + return AreaRead.model_validate(area) diff --git a/backend/app/services/dataset_service.py b/backend/app/services/dataset_service.py new file mode 100644 index 00000000..3ad47138 --- /dev/null +++ b/backend/app/services/dataset_service.py @@ -0,0 +1,452 @@ +from __future__ import annotations + +import json +import pathlib +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from uuid import UUID +import uuid + +from fastapi import UploadFile +from sqlalchemy.orm import Session + +from app.core.errors import AppError +from app.models import Dataset, Project +from app.schemas.dataset import DatasetCreateResponse, DatasetStorageResponse, DatasetVectorSummary +from app.services.geojson_service import parse_geojson_payload, load_dataset_text +from app.services.raster_service import extract_raster_metadata +from app.services.storage_service import StorageService +from app.services.vector_feature_service import VectorFeatureService + + +class DatasetService: + VECTOR_EXTENSIONS = {".geojson", ".json"} + RASTER_EXTENSIONS = {".tif", ".tiff", ".geotiff"} + VECTOR_TYPES = {"vector", "geojson"} + RASTER_TYPES = {"raster", "tif", "tiff", "geotiff"} + VALID_DATASET_ROLES = {"source", "derived", "reference"} + + @staticmethod + def _canonical_dataset_type(dataset_type: str) -> str: + normalized = (dataset_type or "").strip().lower() + if normalized in DatasetService.VECTOR_TYPES: + return "vector" + if normalized in DatasetService.RASTER_TYPES: + return "raster" + raise AppError( + code="INVALID_DATASET_TYPE", + message="dataset_type must be 'vector' or 'raster' (or legacy 'geojson')", + status_code=400, + ) + + @staticmethod + def _normalize_stored_dataset_type(dataset_type: str) -> str: + normalized = (dataset_type or "").strip().lower() + if normalized in DatasetService.VECTOR_TYPES: + return "vector" + if normalized in DatasetService.RASTER_TYPES: + return "raster" + return normalized + + @staticmethod + def _is_vector_type(dataset_type: str) -> bool: + return DatasetService._normalize_stored_dataset_type(dataset_type) == "vector" + + @staticmethod + def _is_raster_type(dataset_type: str) -> bool: + return DatasetService._normalize_stored_dataset_type(dataset_type) == "raster" + + @staticmethod + def _normalize_dataset_role(dataset_role: str | None) -> str: + normalized = (dataset_role or "").strip().lower() or "source" + if normalized not in DatasetService.VALID_DATASET_ROLES: + raise AppError( + code="INVALID_DATASET_ROLE", + message="dataset_role must be one of: source, derived, reference", + status_code=400, + ) + return normalized + + @staticmethod + def _extension_for_path(filename: str) -> str: + return Path(filename).suffix.lower() + + @staticmethod + def _validate_upload_filename(filename: str | None) -> str: + if not filename: + raise AppError(code="INVALID_UPLOAD", message="Missing file name", status_code=400) + return filename + + @staticmethod + def list_datasets(db: Session, project_id: UUID, limit: int = 50, offset: int = 0) -> tuple[list[DatasetCreateResponse], int]: + total = db.query(Dataset).filter(Dataset.project_id == project_id).count() + rows = ( + db.query(Dataset) + .filter(Dataset.project_id == project_id) + .order_by(Dataset.created_at.desc()) + .offset(offset) + .limit(limit) + .all() + ) + response_items = [] + for row in rows: + feature_count = None + metadata_json = row.metadata_json or {} + vector_summary = DatasetService._extract_vector_summary(row.dataset_type, metadata_json) + if isinstance(metadata_json, dict): + feature_count = metadata_json.get("feature_count") + response_items.append( + DatasetCreateResponse( + id=row.id, + name=row.name, + dataset_type=row.dataset_type, + source=row.source, + dataset_role=row.dataset_role, + source_name=row.source_name, + reference_layer_name=row.reference_layer_name, + source_metadata=row.source_metadata, + provenance_metadata=row.provenance_metadata, + imported_at=row.imported_at, + project_id=row.project_id, + area_id=row.area_id, + storage_path=row.storage_path, + original_filename=row.original_filename, + stored_filename=row.stored_filename, + content_type=row.content_type, + size_bytes=row.size_bytes, + checksum_sha256=row.checksum_sha256, + crs=row.crs, + bounds_json=row.bounds_json, + metadata_json=row.metadata_json, + vector_summary=vector_summary, + status=row.status, + derived_from_dataset_id=row.derived_from_dataset_id, + created_at=row.created_at, + feature_count=feature_count, + ) + ) + return response_items, total + + @staticmethod + def _extract_vector_summary(dataset_type: str, metadata_json: dict) -> DatasetVectorSummary | None: + if not DatasetService._is_vector_type(dataset_type): + return None + if not isinstance(metadata_json, dict): + return None + return DatasetVectorSummary( + feature_count=metadata_json.get("feature_count"), + geometry_types=metadata_json.get("geometry_types"), + bounds_json=metadata_json.get("bounds_json"), + approximate_area_m2=metadata_json.get("approximate_area_m2"), + crs=metadata_json.get("crs"), + feature_geometry_count=metadata_json.get("feature_geometry_count"), + invalid_features=metadata_json.get("invalid_features"), + crs_assumed=metadata_json.get("crs_assumed"), + ) + + @staticmethod + async def upload_dataset( + db: Session, + project_id: UUID, + file: UploadFile, + dataset_type: str, + source: str, + dataset_role: str = "source", + source_name: str | None = None, + reference_layer_name: str | None = None, + source_metadata: dict | None = None, + provenance_metadata: dict | None = None, + area_id: UUID | None = None, + ) -> DatasetCreateResponse: + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + + filename = DatasetService._validate_upload_filename(file.filename) + canonical_type = DatasetService._canonical_dataset_type(dataset_type) + normalized_role = DatasetService._normalize_dataset_role(dataset_role) + normalized_source_name = source_name + if normalized_role == "reference" and not normalized_source_name: + normalized_source_name = "manual" + if normalized_role == "reference" and canonical_type == "raster": + raise AppError(code="INVALID_DATASET_ROLE", message="reference role requires vector dataset type", status_code=400) + extension = DatasetService._extension_for_path(filename) + + if canonical_type == "vector" and extension not in DatasetService.VECTOR_EXTENSIONS: + raise AppError(code="INVALID_UPLOAD", message="Vector uploads require .geojson or .json files", status_code=415) + if canonical_type == "raster" and extension not in DatasetService.RASTER_EXTENSIONS: + raise AppError( + code="INVALID_UPLOAD", + message="Raster uploads require .tif, .tiff or .geotiff files", + status_code=415, + ) + + raw = await file.read() + storage_info = StorageService.persist_dataset_file( + project_id=str(project_id), + dataset_id=str(dataset_id := uuid.uuid4()), + dataset_type=canonical_type, + original_filename=filename, + content=raw, + content_type=file.content_type, + ) + + metadata: dict[str, Any] = {} + vector_payload: dict[str, Any] | None = None + status = "uploaded" + try: + status = "validating" + if canonical_type == "vector": + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise AppError(code="INVALID_UPLOAD", message="Upload must be UTF-8 encoded", status_code=400) from exc + metadata = parse_geojson_payload(text) + vector_payload = json.loads(text) + status = "ready" + else: + metadata = extract_raster_metadata(storage_info["storage_path"]) + status = "ready" + except ValueError as exc: + status = "failed" + StorageService.remove_dataset_file(storage_info["storage_path"]) + raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc + except AppError as exc: + if canonical_type == "raster" and exc.code == "RASTER_PROCESSING_UNAVAILABLE": + status = "failed" + metadata = { + "processing_error": exc.message, + "processing_code": exc.code, + } + else: + StorageService.remove_dataset_file(storage_info["storage_path"]) + raise + + dataset = Dataset( + id=dataset_id, + project_id=project_id, + area_id=area_id, + name=filename, + dataset_type=canonical_type, + source=source, + dataset_role=normalized_role, + source_name=normalized_source_name, + reference_layer_name=reference_layer_name if normalized_role == "reference" else None, + source_metadata=source_metadata, + provenance_metadata=provenance_metadata, + imported_at=datetime.now(timezone.utc), + storage_path=storage_info["storage_path"], + original_filename=storage_info["original_filename"], + stored_filename=storage_info["stored_filename"], + content_type=storage_info["content_type"], + size_bytes=storage_info["size_bytes"], + checksum_sha256=storage_info["checksum_sha256"], + crs=metadata.get("crs") if isinstance(metadata, dict) else None, + bounds_json=metadata.get("bounds_json") if isinstance(metadata, dict) else None, + resolution_json=metadata.get("resolution_json") if isinstance(metadata, dict) else None, + bands_json=metadata.get("bands_json") if isinstance(metadata, dict) else None, + metadata_json=metadata, + status=status, + ) + db.add(dataset) + db.commit() + db.refresh(dataset) + + if canonical_type == "vector" and vector_payload is not None and status == "ready": + feature_class = reference_layer_name if normalized_role == "reference" else None + VectorFeatureService.persist_geojson_features( + db=db, + dataset_id=dataset.id, + payload=vector_payload, + feature_class=feature_class, + ) + + return DatasetCreateResponse( + id=dataset.id, + name=dataset.name, + dataset_type=dataset.dataset_type, + source=dataset.source, + dataset_role=dataset.dataset_role, + source_name=dataset.source_name, + reference_layer_name=dataset.reference_layer_name, + source_metadata=dataset.source_metadata, + provenance_metadata=dataset.provenance_metadata, + imported_at=dataset.imported_at, + project_id=dataset.project_id, + area_id=dataset.area_id, + storage_path=dataset.storage_path, + original_filename=dataset.original_filename, + stored_filename=dataset.stored_filename, + content_type=dataset.content_type, + size_bytes=dataset.size_bytes, + checksum_sha256=dataset.checksum_sha256, + crs=dataset.crs, + derived_from_dataset_id=dataset.derived_from_dataset_id, + bounds_json=dataset.bounds_json, + metadata_json=dataset.metadata_json, + vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}), + status=dataset.status, + created_at=dataset.created_at, + feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None, + ) + + @staticmethod + def refresh_metadata(db: Session, dataset_id: UUID) -> DatasetCreateResponse: + dataset = DatasetService._get_dataset(db, dataset_id) + if not dataset.storage_path: + raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) + if not Path(dataset.storage_path).exists(): + raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) + + try: + if DatasetService._is_vector_type(dataset.dataset_type): + metadata = parse_geojson_payload(load_dataset_text(dataset.storage_path)) + elif DatasetService._is_raster_type(dataset.dataset_type): + metadata = extract_raster_metadata(dataset.storage_path) + else: + raise AppError(code="INVALID_DATASET_TYPE", message="Cannot refresh metadata for this dataset type", status_code=400) + dataset.status = "ready" + except ValueError as exc: + dataset.status = "failed" + raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc + except AppError as exc: + if DatasetService._is_raster_type(dataset.dataset_type) and exc.code == "RASTER_PROCESSING_UNAVAILABLE": + dataset.status = "failed" + metadata = {"processing_error": exc.message, "processing_code": exc.code} + else: + dataset.status = "failed" + raise + + dataset.crs = metadata.get("crs") if isinstance(metadata, dict) else dataset.crs + dataset.bounds_json = metadata.get("bounds_json") if isinstance(metadata, dict) else dataset.bounds_json + dataset.metadata_json = metadata + dataset.resolution_json = metadata.get("resolution_json") if isinstance(metadata, dict) else dataset.resolution_json + dataset.bands_json = metadata.get("bands_json") if isinstance(metadata, dict) else dataset.bands_json + + db.add(dataset) + db.commit() + db.refresh(dataset) + + return DatasetCreateResponse( + id=dataset.id, + name=dataset.name, + dataset_type=dataset.dataset_type, + source=dataset.source, + dataset_role=dataset.dataset_role, + source_name=dataset.source_name, + reference_layer_name=dataset.reference_layer_name, + source_metadata=dataset.source_metadata, + provenance_metadata=dataset.provenance_metadata, + imported_at=dataset.imported_at, + project_id=dataset.project_id, + area_id=dataset.area_id, + storage_path=dataset.storage_path, + original_filename=dataset.original_filename, + stored_filename=dataset.stored_filename, + content_type=dataset.content_type, + size_bytes=dataset.size_bytes, + checksum_sha256=dataset.checksum_sha256, + crs=dataset.crs, + derived_from_dataset_id=dataset.derived_from_dataset_id, + bounds_json=dataset.bounds_json, + metadata_json=dataset.metadata_json, + vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}), + status=dataset.status, + created_at=dataset.created_at, + feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None, + ) + + @staticmethod + def get_dataset(db: Session, dataset_id: UUID) -> Dataset: + dataset = db.get(Dataset, dataset_id) + if not dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + return dataset + + @staticmethod + def _get_dataset(db: Session, dataset_id: UUID) -> Dataset: + return DatasetService.get_dataset(db, dataset_id) + + @staticmethod + def get_dataset_geojson(db: Session, dataset_id: UUID) -> dict: + dataset = DatasetService._get_dataset(db, dataset_id) + if not DatasetService._is_vector_type(dataset.dataset_type): + raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400) + if not dataset.storage_path: + raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) + if not pathlib.Path(dataset.storage_path).exists(): + raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) + + raw = load_dataset_text(dataset.storage_path) + try: + return json.loads(raw) + except Exception as exc: + raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=500) from exc + + @staticmethod + def inspect_vector_dataset(db: Session, dataset_id: UUID) -> dict[str, Any]: + dataset = DatasetService._get_dataset(db, dataset_id) + if not DatasetService._is_vector_type(dataset.dataset_type): + raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400) + if not dataset.storage_path or not Path(dataset.storage_path).exists(): + raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) + metadata = dataset.metadata_json or {} + if not isinstance(metadata, dict): + metadata = {} + summary = DatasetService._extract_vector_summary(dataset.dataset_type, metadata) + return { + "dataset": { + "id": str(dataset.id), + "name": dataset.name, + "dataset_type": dataset.dataset_type, + "status": dataset.status, + "source": dataset.source, + "storage": DatasetStorageResponse( + original_filename=dataset.original_filename, + stored_filename=dataset.stored_filename, + content_type=dataset.content_type, + size_bytes=dataset.size_bytes, + checksum_sha256=dataset.checksum_sha256, + ).model_dump(), + "feature_count": metadata.get("feature_count"), + "crs": metadata.get("crs"), + }, + "summary": summary.model_dump() if summary else None, + "metadata": metadata, + } + + @staticmethod + def vector_summary(db: Session, dataset_id: UUID) -> dict[str, Any]: + dataset = DatasetService._get_dataset(db, dataset_id) + if not DatasetService._is_vector_type(dataset.dataset_type): + raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400) + + metadata = dataset.metadata_json or {} + if not isinstance(metadata, dict): + metadata = {} + summary = DatasetService._extract_vector_summary(dataset.dataset_type, metadata) + if not summary: + raise AppError(code="INVALID_GEOJSON", message="Vector summary unavailable", status_code=422) + return summary.model_dump() + + @staticmethod + def raster_metadata(db: Session, dataset_id: UUID) -> dict[str, Any]: + dataset = DatasetService._get_dataset(db, dataset_id) + if not DatasetService._is_raster_type(dataset.dataset_type): + raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a raster dataset", status_code=400) + if not dataset.storage_path: + raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) + if not Path(dataset.storage_path).exists(): + raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) + + if isinstance(dataset.metadata_json, dict) and dataset.metadata_json.get("driver"): + return dataset.metadata_json + + metadata = extract_raster_metadata(dataset.storage_path) + dataset.metadata_json = dict(dataset.metadata_json or {}) + dataset.metadata_json.update(metadata) + dataset.status = "ready" + db.add(dataset) + db.commit() + db.refresh(dataset) + return metadata diff --git a/backend/app/services/demo_workflow_service.py b/backend/app/services/demo_workflow_service.py new file mode 100644 index 00000000..bf4ec923 --- /dev/null +++ b/backend/app/services/demo_workflow_service.py @@ -0,0 +1,288 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID, uuid4 + +from geoalchemy2.shape import from_shape +from sqlalchemy.orm import Session + +from app.models import Area, Dataset, Metric, Project, QualityCheck +from app.schemas.demo import DemoWorkflowResponse +from app.services.geojson_service import parse_geojson_payload +from app.services.qa_service import QaService +from app.services.quality_service import QualityService +from app.services.storage_service import StorageService +from app.services.vector_feature_service import VectorFeatureService +from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon + + +class DemoWorkflowService: + PROJECT_NAME = "GeoIntel Demo - Building QA" + AREA_NAME = "Demo AOI - Geel buildings" + REFERENCE_FILENAME = "demo_reference_buildings.geojson" + CANDIDATE_FILENAME = "demo_predicted_buildings.geojson" + + @staticmethod + def _repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + @staticmethod + def _fixture_path(filename: str) -> Path: + return DemoWorkflowService._repo_root() / "fixtures" / "golden" / filename + + @staticmethod + def _load_fixture(filename: str) -> tuple[dict, bytes]: + path = DemoWorkflowService._fixture_path(filename) + raw = path.read_bytes() + return json.loads(raw.decode("utf-8")), raw + + @staticmethod + def _find_existing_project(db: Session) -> Project | None: + return ( + db.query(Project) + .filter(Project.name == DemoWorkflowService.PROJECT_NAME) + .filter(Project.status != "deleted") + .first() + ) + + @staticmethod + def _create_area(db: Session, project_id: UUID) -> Area: + geometry = { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [4.30, 51.18], + [4.45, 51.18], + [4.45, 51.33], + [4.30, 51.33], + [4.30, 51.18], + ] + ] + ], + } + multipolygon = normalize_to_multipolygon(geometry) + area = Area( + id=uuid4(), + project_id=project_id, + name=DemoWorkflowService.AREA_NAME, + geometry=from_shape(multipolygon, srid=4326), + original_crs="EPSG:4326", + area_m2=area_m2(multipolygon), + bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326), + ) + db.add(area) + db.commit() + db.refresh(area) + return area + + @staticmethod + def _create_dataset( + db: Session, + *, + project_id: UUID, + area_id: UUID, + filename: str, + payload: dict, + raw: bytes, + role: str, + source_name: str, + reference_layer_name: str | None, + ) -> Dataset: + dataset_id = uuid4() + storage_info = StorageService.persist_dataset_file( + project_id=str(project_id), + dataset_id=str(dataset_id), + dataset_type="vector", + original_filename=filename, + content=raw, + content_type="application/geo+json", + ) + metadata = parse_geojson_payload(payload) + dataset = Dataset( + id=dataset_id, + project_id=project_id, + area_id=area_id, + name=filename, + dataset_type="vector", + source="fixture", + dataset_role=role, + source_name=source_name, + reference_layer_name=reference_layer_name, + source_metadata={ + "fixture": True, + "fixture_name": filename, + "usage": "offline demo workflow only", + }, + provenance_metadata={ + "created_by": "demo_workflow", + "source_path": str(DemoWorkflowService._fixture_path(filename)), + }, + imported_at=datetime.now(timezone.utc), + storage_path=storage_info["storage_path"], + original_filename=storage_info["original_filename"], + stored_filename=storage_info["stored_filename"], + content_type=storage_info["content_type"], + size_bytes=storage_info["size_bytes"], + checksum_sha256=storage_info["checksum_sha256"], + crs=metadata.get("crs"), + bounds_json=metadata.get("bounds_json"), + metadata_json=metadata, + status="ready", + ) + db.add(dataset) + db.commit() + db.refresh(dataset) + VectorFeatureService.persist_geojson_features( + db=db, + dataset_id=dataset.id, + payload=payload, + feature_class=reference_layer_name or "building", + ) + return dataset + + @staticmethod + def _persist_qa( + db: Session, + *, + project_id: UUID, + candidate_dataset_id: UUID, + reference_dataset_id: UUID, + area_id: UUID, + ) -> QualityCheck: + result = QaService.compare_candidate_with_reference( + db=db, + project_id=project_id, + candidate_dataset_id=candidate_dataset_id, + reference_dataset_id=reference_dataset_id, + iou_threshold=0.5, + area_id=area_id, + ) + return QualityService.persist_quality_check( + db=db, + project_id=project_id, + candidate_dataset_id=candidate_dataset_id, + reference_dataset_id=reference_dataset_id, + check_type="demo_candidate_vs_reference", + status=result.status, + score=result.f1_score, + parameters={ + "iou_threshold": result.iou_threshold, + "area_id": str(area_id), + "fixture_workflow": True, + }, + findings={ + "matches": result.matches, + "false_positives": result.false_positives, + "false_negatives": result.false_negatives, + "warnings": result.warnings, + "unsupported_geometry": result.unsupported_geometry, + "unsupported_geometries": result.unsupported_geometries, + }, + metrics={ + "precision": result.precision, + "recall": result.recall, + "f1": result.f1_score, + "mean_iou": result.mean_iou, + "false_positive_count": result.false_positives, + "false_negative_count": result.false_negatives, + }, + ) + + @staticmethod + def seed(db: Session) -> DemoWorkflowResponse: + existing = DemoWorkflowService._find_existing_project(db) + if existing: + area = db.query(Area).filter(Area.project_id == existing.id).order_by(Area.created_at.asc()).first() + reference = ( + db.query(Dataset) + .filter(Dataset.project_id == existing.id) + .filter(Dataset.dataset_role == "reference") + .filter(Dataset.source_name == "fixture") + .first() + ) + candidate = ( + db.query(Dataset) + .filter(Dataset.project_id == existing.id) + .filter(Dataset.dataset_role == "source") + .filter(Dataset.source_name == "fixture") + .first() + ) + quality_check = ( + db.query(QualityCheck) + .filter(QualityCheck.project_id == existing.id) + .filter(QualityCheck.check_type == "demo_candidate_vs_reference") + .order_by(QualityCheck.created_at.desc()) + .first() + ) + if area and reference and candidate and quality_check: + return DemoWorkflowResponse( + project_id=existing.id, + area_id=area.id, + reference_dataset_id=reference.id, + candidate_dataset_id=candidate.id, + quality_check_id=quality_check.id, + metric_count=db.query(Metric).filter(Metric.quality_check_id == quality_check.id).count(), + status="ready", + message="Demo workflow already exists.", + created=False, + ) + + project = Project( + id=uuid4(), + name=DemoWorkflowService.PROJECT_NAME, + description="Offline fixture workflow: reference buildings, predicted buildings and persisted QA metrics.", + region="Kempen", + status="active", + ) + db.add(project) + db.commit() + db.refresh(project) + + area = DemoWorkflowService._create_area(db, project.id) + reference_payload, reference_raw = DemoWorkflowService._load_fixture("reference_buildings.geojson") + candidate_payload, candidate_raw = DemoWorkflowService._load_fixture("predicted_buildings.geojson") + + reference = DemoWorkflowService._create_dataset( + db=db, + project_id=project.id, + area_id=area.id, + filename=DemoWorkflowService.REFERENCE_FILENAME, + payload=reference_payload, + raw=reference_raw, + role="reference", + source_name="fixture", + reference_layer_name="buildings", + ) + candidate = DemoWorkflowService._create_dataset( + db=db, + project_id=project.id, + area_id=area.id, + filename=DemoWorkflowService.CANDIDATE_FILENAME, + payload=candidate_payload, + raw=candidate_raw, + role="source", + source_name="fixture", + reference_layer_name=None, + ) + quality_check = DemoWorkflowService._persist_qa( + db=db, + project_id=project.id, + candidate_dataset_id=candidate.id, + reference_dataset_id=reference.id, + area_id=area.id, + ) + + return DemoWorkflowResponse( + project_id=project.id, + area_id=area.id, + reference_dataset_id=reference.id, + candidate_dataset_id=candidate.id, + quality_check_id=quality_check.id, + metric_count=6, + status="ready", + message="Demo workflow seeded from explicit local fixtures.", + created=True, + ) diff --git a/backend/app/services/detection_georeferencing.py b/backend/app/services/detection_georeferencing.py new file mode 100644 index 00000000..b80a627c --- /dev/null +++ b/backend/app/services/detection_georeferencing.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from typing import Any + +from pyproj import Transformer +from shapely.geometry import Polygon + +from app.core.errors import AppError + + +def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs: str | None = None) -> Polygon: + if len(bbox) != 4: + raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO detection bbox must contain four pixel coordinates", status_code=422) + + x_min, y_min, x_max, y_max = [float(value) for value in bbox] + if x_max <= x_min or y_max <= y_min: + raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO detection bbox must have positive width and height", status_code=422) + + transform = tile.get("transform") + if isinstance(transform, list) and len(transform) >= 6: + corners = [ + _apply_gdal_transform(transform, x_min, y_min), + _apply_gdal_transform(transform, x_max, y_min), + _apply_gdal_transform(transform, x_max, y_max), + _apply_gdal_transform(transform, x_min, y_max), + _apply_gdal_transform(transform, x_min, y_min), + ] + else: + corners = _corners_from_bounds(bbox=[x_min, y_min, x_max, y_max], tile=tile) + + source_crs = crs or tile.get("crs") or tile.get("source_crs") or "EPSG:4326" + if str(source_crs).upper() not in {"EPSG:4326", "4326"}: + transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True) + corners = [transformer.transform(x, y) for x, y in corners] + + polygon = Polygon(corners) + if polygon.is_empty or not polygon.is_valid: + raise AppError(code="DETECTION_INVALID_GEOMETRY", message="Georeferenced detection geometry is invalid", status_code=422) + return polygon + + +def _apply_gdal_transform(transform: list[float], x: float, y: float) -> tuple[float, float]: + c, a, b, f, d, e = [float(value) for value in transform[:6]] + return (a * x + b * y + c, d * x + e * y + f) + + +def _corners_from_bounds(bbox: list[float], tile: dict[str, Any]) -> list[tuple[float, float]]: + bounds = tile.get("bounds") + pixel_window = tile.get("pixel_window") + if not (isinstance(bounds, list) and len(bounds) == 4 and isinstance(pixel_window, list) and len(pixel_window) == 4): + raise AppError( + code="DETECTION_TILE_MANIFEST_INVALID", + message="Tile manifest entries require transform or bounds plus pixel_window for georeferencing", + status_code=422, + ) + x_min, y_min, x_max, y_max = bbox + left, bottom, right, top = [float(value) for value in bounds] + _, _, width, height = [float(value) for value in pixel_window] + if width <= 0 or height <= 0: + raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile pixel_window must have positive size", status_code=422) + + def project(px: float, py: float) -> tuple[float, float]: + x = left + (px / width) * (right - left) + y = top - (py / height) * (top - bottom) + return (x, y) + + return [ + project(x_min, y_min), + project(x_max, y_min), + project(x_max, y_max), + project(x_min, y_max), + project(x_min, y_min), + ] diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py new file mode 100644 index 00000000..1d9c06b0 --- /dev/null +++ b/backend/app/services/detection_service.py @@ -0,0 +1,618 @@ +from __future__ import annotations + +import uuid +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from typing import Type + +from geoalchemy2.shape import from_shape, to_shape +from shapely.geometry import mapping, shape + +from app.core.config import Settings, get_settings +from app.core.errors import AppError +from app.models import AnalysisRun, Dataset, Detection, Job, Project, VectorFeature +from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse +from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon +from app.services.model_registry_service import ModelRegistryService +from app.services.qa_service import QaService +from app.services.quality_service import QualityService +from app.services.yolo_adapter import YoloDetectionAdapter + + +class DetectionService: + @staticmethod + def _now() -> datetime: + return datetime.now(UTC) + + @staticmethod + def run_detection( + db, + project_id: uuid.UUID, + dataset_id: uuid.UUID, + model_id: str, + confidence_threshold: float, + class_filter: list[str] | None = None, + tile_manifest_path: str | None = None, + parameters_json: dict[str, Any] | None = None, + settings: Settings | None = None, + yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter, + ) -> DetectionRunResponse: + parameters = dict(parameters_json or {}) + resolved_settings = settings or get_settings() + project = db.get(Project, project_id) + if not project: + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + dataset = db.get(Dataset, dataset_id) + if not dataset or dataset.project_id != project_id: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + if dataset.dataset_type != "raster": + raise AppError( + code="INVALID_DATASET_TYPE", + message="Detection requires a raster dataset", + details={"dataset_type": dataset.dataset_type}, + status_code=400, + ) + + model = ModelRegistryService.get_model_capability( + model_id, + settings=resolved_settings, + yolo_adapter_class=yolo_adapter_class, + ) + if model is None: + raise AppError(code="DETECTION_MODEL_NOT_FOUND", message="Detection model not found", status_code=404) + if model.model_id == "manual-fixture-detector" and parameters.get("fixture_mode") is not True: + raise AppError( + code="FIXTURE_MODE_REQUIRED", + message="Fixture detector requires explicit fixture_mode=true", + status_code=400, + ) + if model.model_id == resolved_settings.yolo_model_id and not tile_manifest_path: + raise AppError( + code="DETECTION_TILE_MANIFEST_REQUIRED", + message="Configured YOLO inference requires an existing raster tile manifest path", + status_code=400, + ) + + run_parameters = { + "model_id": model.model_id, + "confidence_threshold": confidence_threshold, + "class_filter": class_filter or [], + "tile_manifest_path": tile_manifest_path, + "parameters_json": parameters, + } + job = DetectionService._create_job(db, project_id, dataset_id, run_parameters) + analysis_run = DetectionService._create_analysis_run(db, project_id, dataset_id, job.id, model, run_parameters) + + if not model.configured: + message = model.limitation_message + code = "DETECTION_DEPENDENCY_UNAVAILABLE" if model.status == "dependency_unavailable" else "DETECTION_MODEL_UNAVAILABLE" + DetectionService._mark_failed(db, analysis_run, job, code=code, message=message) + return DetectionRunResponse( + analysis_run_id=analysis_run.id, + job_id=job.id, + project_id=project_id, + dataset_id=dataset_id, + model_id=model.model_id, + status="failed", + detection_count=0, + error_code=code, + message=message, + ) + + if model.model_id == "manual-fixture-detector": + detections = DetectionService._persist_fixture_detections( + db=db, + project_id=project_id, + dataset_id=dataset_id, + analysis_run=analysis_run, + job=job, + model_name=model.model_id, + model_version=model.version, + raw_detections=parameters.get("fixture_detections"), + confidence_threshold=confidence_threshold, + class_filter=class_filter or [], + ) + DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections)) + return DetectionRunResponse( + analysis_run_id=analysis_run.id, + job_id=job.id, + project_id=project_id, + dataset_id=dataset_id, + model_id=model.model_id, + status="success", + detection_count=len(detections), + message="Fixture detections persisted.", + ) + + if model.model_id == resolved_settings.yolo_model_id: + try: + detections = DetectionService._run_configured_yolo( + db=db, + project_id=project_id, + dataset_id=dataset_id, + analysis_run=analysis_run, + job=job, + model_name=model.model_id, + model_version=model.version, + tile_manifest_path=tile_manifest_path, + confidence_threshold=confidence_threshold, + class_filter=class_filter or [], + settings=resolved_settings, + yolo_adapter_class=yolo_adapter_class, + ) + except AppError as exc: + DetectionService._mark_failed(db, analysis_run, job, code=exc.code, message=exc.message) + return DetectionRunResponse( + analysis_run_id=analysis_run.id, + job_id=job.id, + project_id=project_id, + dataset_id=dataset_id, + model_id=model.model_id, + status="failed", + detection_count=0, + error_code=exc.code, + message=exc.message, + ) + DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections)) + return DetectionRunResponse( + analysis_run_id=analysis_run.id, + job_id=job.id, + project_id=project_id, + dataset_id=dataset_id, + model_id=model.model_id, + status="success", + detection_count=len(detections), + message="YOLO detections persisted.", + ) + + raise AppError(code="DETECTION_MODEL_UNAVAILABLE", message="Detection model is unavailable", status_code=503) + + @staticmethod + def get_run(db, analysis_run_id: uuid.UUID) -> DetectionRunRead: + run = db.get(AnalysisRun, analysis_run_id) + if not run or run.analysis_type != "detection": + raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404) + return DetectionRunRead.model_validate(run) + + @staticmethod + def list_runs( + db, + *, + project_id: uuid.UUID | None = None, + dataset_id: uuid.UUID | None = None, + ) -> DetectionRunListResponse: + query = db.query(AnalysisRun).filter(AnalysisRun.analysis_type == "detection") + if project_id is not None: + query = query.filter(AnalysisRun.project_id == project_id) + if dataset_id is not None: + query = query.filter(AnalysisRun.dataset_id == dataset_id) + rows = query.order_by(AnalysisRun.created_at.desc()).all() + return DetectionRunListResponse(items=[DetectionRunRead.model_validate(row) for row in rows], total=len(rows)) + + @staticmethod + def list_detections( + db, + analysis_run_id: uuid.UUID | None = None, + *, + dataset_id: uuid.UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + ) -> DetectionListResponse: + if analysis_run_id is not None: + run = db.get(AnalysisRun, analysis_run_id) + if not run or run.analysis_type != "detection": + raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404) + rows = DetectionService._query_detection_rows( + db, + analysis_run_id=analysis_run_id, + dataset_id=dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ) + items = [DetectionRead.model_validate(row) for row in rows] + return DetectionListResponse(items=items, total=len(items)) + + @staticmethod + def get_detection(db, detection_id: uuid.UUID) -> DetectionRead: + detection = db.get(Detection, detection_id) + if not detection: + raise AppError(code="DETECTION_NOT_FOUND", message="Detection not found", status_code=404) + return DetectionRead.model_validate(detection) + + @staticmethod + def detections_to_geojson( + db, + *, + analysis_run_id: uuid.UUID | None = None, + dataset_id: uuid.UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + ) -> dict[str, Any]: + detections = DetectionService._query_detection_rows( + db, + analysis_run_id=analysis_run_id, + dataset_id=dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ) + return { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": str(detection.id), + "properties": DetectionService._detection_properties(detection), + "geometry": mapping(to_shape(detection.geometry)), + } + for detection in detections + ], + } + + @staticmethod + def compare_detections_with_reference( + db, + analysis_run_id: uuid.UUID, + reference_dataset_id: uuid.UUID, + iou_threshold: float = 0.5, + class_name: str | None = None, + min_confidence: float | None = None, + ) -> dict[str, Any]: + run = db.get(AnalysisRun, analysis_run_id) + if not run or run.analysis_type != "detection": + raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404) + reference_dataset = db.get(Dataset, reference_dataset_id) + if not reference_dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Reference dataset not found", status_code=404) + if reference_dataset.project_id != run.project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Reference dataset does not belong to detection project", status_code=400) + if reference_dataset.dataset_type not in {"vector", "geojson"}: + raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400) + + detections = DetectionService._query_detection_rows( + db, + analysis_run_id=analysis_run_id, + dataset_id=run.dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ) + references = db.query(VectorFeature).filter(VectorFeature.dataset_id == reference_dataset_id).all() + if not references: + raise AppError( + code="REFERENCE_FEATURES_NOT_FOUND", + message="Reference dataset has no persisted vector features for QA", + status_code=422, + ) + + candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections] + reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references] + matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics( + candidate_geometries, + reference_geometries, + iou_threshold, + ) + mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values) + precision = matches / (matches + false_positives) if matches + false_positives > 0 else None + recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None + f1_score = None + if precision is not None and recall is not None: + f1_score = (2 * precision * recall) / (precision + recall) if precision + recall > 0 else 0.0 + status = "unsupported" if unsupported else "ok" + quality_check = QualityService.persist_quality_check( + db=db, + project_id=run.project_id, + analysis_run_id=analysis_run_id, + candidate_dataset_id=run.dataset_id, + reference_dataset_id=reference_dataset_id, + check_type="detections_vs_reference", + status=status, + score=f1_score, + parameters={ + "analysis_run_id": str(analysis_run_id), + "reference_dataset_id": str(reference_dataset_id), + "iou_threshold": iou_threshold, + "class_name": class_name, + "min_confidence": min_confidence, + }, + findings={ + "matches": matches, + "false_positives": false_positives, + "false_negatives": false_negatives, + "warnings": warnings, + "unsupported_geometry": unsupported, + }, + metrics={ + "precision": precision, + "recall": recall, + "f1": f1_score, + "mean_iou": mean_iou, + "false_positive_count": false_positives, + "false_negative_count": false_negatives, + }, + ) + return { + "status": status, + "quality_check_id": str(quality_check.id), + "analysis_run_id": str(analysis_run_id), + "reference_dataset_id": str(reference_dataset_id), + "candidate_feature_count": len(candidate_geometries), + "reference_feature_count": len(reference_geometries), + "matches": matches, + "false_positives": false_positives, + "false_negatives": false_negatives, + "precision": precision, + "recall": recall, + "f1_score": f1_score, + "mean_iou": mean_iou, + "iou_threshold": iou_threshold, + "warnings": warnings, + } + + @staticmethod + def _create_job(db, project_id: uuid.UUID, dataset_id: uuid.UUID, parameters: dict[str, Any]) -> Job: + job = Job( + id=uuid.uuid4(), + job_type="detection.run", + status="running", + project_id=project_id, + dataset_id=dataset_id, + input_dataset_id=dataset_id, + parameters_json=parameters, + started_at=DetectionService._now(), + ) + db.add(job) + db.commit() + db.refresh(job) + return job + + @staticmethod + def _query_detection_rows( + db, + *, + analysis_run_id: uuid.UUID | None = None, + dataset_id: uuid.UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + ) -> list[Detection]: + query = db.query(Detection) + if analysis_run_id is not None: + query = query.filter(Detection.analysis_run_id == analysis_run_id) + if dataset_id is not None: + query = query.filter(Detection.dataset_id == dataset_id) + if class_name: + query = query.filter(Detection.class_name == class_name) + if min_confidence is not None: + query = query.filter(Detection.confidence >= min_confidence) + return query.order_by(Detection.created_at.desc()).all() + + @staticmethod + def _detection_properties(detection: Detection) -> dict[str, Any]: + return { + "detection_id": str(detection.id), + "class_name": detection.class_name, + "confidence": detection.confidence, + "model_name": detection.model_name, + "model_version": detection.model_version, + "analysis_run_id": str(detection.analysis_run_id) if detection.analysis_run_id else None, + "dataset_id": str(detection.dataset_id) if detection.dataset_id else None, + "job_id": str(detection.job_id) if detection.job_id else None, + "source_tile_path": detection.source_tile_path, + "bbox_json": detection.bbox_json, + } + + @staticmethod + def _create_analysis_run(db, project_id, dataset_id, job_id, model, parameters: dict[str, Any]) -> AnalysisRun: + analysis_run = AnalysisRun( + id=uuid.uuid4(), + project_id=project_id, + dataset_id=dataset_id, + job_id=job_id, + analysis_type="detection", + status="running", + model_name=model.model_id, + model_version=model.version, + parameters_json=parameters, + started_at=DetectionService._now(), + ) + db.add(analysis_run) + db.commit() + db.refresh(analysis_run) + return analysis_run + + @staticmethod + def _mark_failed(db, analysis_run: AnalysisRun, job: Job, code: str, message: str) -> None: + result = {"error_code": code, "message": message, "detection_count": 0} + analysis_run.status = "failed" + analysis_run.finished_at = DetectionService._now() + analysis_run.error_message = message + analysis_run.result_json = result + job.status = "failed" + job.finished_at = analysis_run.finished_at + job.error_message = message + job.result_json = result + db.add(analysis_run) + db.add(job) + db.commit() + db.refresh(analysis_run) + db.refresh(job) + + @staticmethod + def _mark_success(db, analysis_run: AnalysisRun, job: Job, detection_count: int) -> None: + result = {"detection_count": detection_count} + analysis_run.status = "success" + analysis_run.finished_at = DetectionService._now() + analysis_run.result_json = result + job.status = "success" + job.finished_at = analysis_run.finished_at + job.result_json = result + db.add(analysis_run) + db.add(job) + db.commit() + db.refresh(analysis_run) + db.refresh(job) + + @staticmethod + def _persist_fixture_detections( + db, + project_id: uuid.UUID, + dataset_id: uuid.UUID, + analysis_run: AnalysisRun, + job: Job, + model_name: str, + model_version: str | None, + raw_detections: Any, + confidence_threshold: float, + class_filter: list[str], + ) -> list[Detection]: + if not isinstance(raw_detections, list): + raise AppError(code="INVALID_FIXTURE_DETECTIONS", message="fixture_detections must be a list", status_code=400) + persisted: list[Detection] = [] + allowed_classes = set(class_filter) + for raw in raw_detections: + if not isinstance(raw, dict): + raise AppError(code="INVALID_FIXTURE_DETECTION", message="Each fixture detection must be an object", status_code=400) + class_name = str(raw.get("class_name") or "") + confidence = float(raw.get("confidence", 0.0)) + if allowed_classes and class_name not in allowed_classes: + continue + if confidence < confidence_threshold: + continue + geometry_payload = raw.get("geometry") + if not isinstance(geometry_payload, dict): + raise AppError(code="INVALID_FIXTURE_DETECTION", message="Fixture detection geometry is required", status_code=400) + geometry = shape(geometry_payload) + if geometry.is_empty or not geometry.is_valid: + raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture detection geometry must be valid", status_code=400) + detection = Detection( + id=uuid.uuid4(), + project_id=project_id, + dataset_id=dataset_id, + analysis_run_id=analysis_run.id, + job_id=job.id, + model_name=model_name, + model_version=model_version, + class_name=class_name, + confidence=confidence, + geometry=from_shape(geometry, srid=4326), + bbox_json=raw.get("bbox_json"), + source_tile_path=raw.get("source_tile_path"), + properties_json=raw.get("properties_json"), + ) + db.add(detection) + persisted.append(detection) + db.commit() + for detection in persisted: + db.refresh(detection) + return persisted + + @staticmethod + def _run_configured_yolo( + db, + project_id: uuid.UUID, + dataset_id: uuid.UUID, + analysis_run: AnalysisRun, + job: Job, + model_name: str, + model_version: str | None, + tile_manifest_path: str | None, + confidence_threshold: float, + class_filter: list[str], + settings: Settings, + yolo_adapter_class: Type[YoloDetectionAdapter], + ) -> list[Detection]: + manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles) + model_path = Path(settings.yolo_model_path or "").expanduser() + adapter = yolo_adapter_class(settings) + model = adapter.load_model(model_path) + allowed_classes = set(class_filter) + persisted: list[Detection] = [] + manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") or "EPSG:4326" + for tile in manifest["tiles"]: + tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser()) + for raw in adapter.predict_tile(model, tile_path, confidence_threshold): + class_name = str(raw.get("class_name") or "") + confidence = float(raw.get("confidence", 0.0)) + if allowed_classes and class_name not in allowed_classes: + continue + if confidence < confidence_threshold: + continue + bbox = raw.get("bbox") + if not isinstance(bbox, list): + raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO adapter returned a detection without bbox", status_code=422) + geometry = pixel_bbox_to_epsg4326_polygon(bbox=bbox, tile=tile, crs=tile.get("crs") or manifest_crs) + detection = Detection( + id=uuid.uuid4(), + project_id=project_id, + dataset_id=dataset_id, + analysis_run_id=analysis_run.id, + job_id=job.id, + model_name=model_name, + model_version=model_version, + class_name=class_name, + confidence=confidence, + geometry=from_shape(geometry, srid=4326), + bbox_json={ + "x_min": float(bbox[0]), + "y_min": float(bbox[1]), + "x_max": float(bbox[2]), + "y_max": float(bbox[3]), + }, + source_tile_path=str(tile_path), + properties_json={**dict(raw.get("properties") or {}), "tile_index": tile.get("index")}, + ) + db.add(detection) + persisted.append(detection) + db.commit() + for detection in persisted: + db.refresh(detection) + return persisted + + @staticmethod + def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int) -> dict[str, Any]: + if not tile_manifest_path: + raise AppError( + code="DETECTION_TILE_MANIFEST_REQUIRED", + message="Configured YOLO inference requires an existing raster tile manifest path", + status_code=400, + ) + manifest_path = Path(tile_manifest_path).expanduser() + if not manifest_path.exists() or not manifest_path.is_file(): + raise AppError( + code="DETECTION_TILE_MANIFEST_NOT_FOUND", + message="Raster tile manifest path does not exist", + details={"tile_manifest_path": str(manifest_path)}, + status_code=422, + ) + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Raster tile manifest must be valid JSON", status_code=422) from exc + tiles = manifest.get("tiles") + if not isinstance(tiles, list) or not tiles: + raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Raster tile manifest must contain tiles", status_code=422) + if len(tiles) > max_tiles: + raise AppError( + code="DETECTION_TILE_LIMIT_EXCEEDED", + message="Raster tile manifest exceeds configured YOLO tile limit", + details={"tile_count": len(tiles), "max_tiles": max_tiles}, + status_code=422, + ) + return manifest + + @staticmethod + def _resolve_tile_path(tile: dict[str, Any], manifest_path: Path) -> Path: + raw_path = tile.get("path") + if not isinstance(raw_path, str) or not raw_path: + raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile manifest entries require a path", status_code=422) + tile_path = Path(raw_path).expanduser() + if not tile_path.is_absolute(): + tile_path = manifest_path.parent / tile_path + if not tile_path.exists() or not tile_path.is_file(): + raise AppError( + code="DETECTION_TILE_NOT_FOUND", + message="Tile referenced by manifest does not exist", + details={"tile_path": str(tile_path)}, + status_code=422, + ) + return tile_path diff --git a/backend/app/services/export_service.py b/backend/app/services/export_service.py new file mode 100644 index 00000000..1af38204 --- /dev/null +++ b/backend/app/services/export_service.py @@ -0,0 +1,431 @@ +from __future__ import annotations + +import json +import re +import uuid +from html import escape +from pathlib import Path +from typing import Any + +from sqlalchemy.orm import Session + +from app.core.errors import AppError +from app.models import AnalysisRun, Dataset, Export, Project, QualityCheck +from app.schemas.export import ExportContentResponse, ExportCreateResponse, ExportListResponse, ExportRead +from app.services.dataset_service import DatasetService +from app.services.detection_service import DetectionService +from app.services.segmentation_service import SegmentationService +from app.services.storage_service import StorageService + + +class ExportService: + @staticmethod + def export_dataset_geojson(db: Session, dataset_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse: + dataset = db.get(Dataset, dataset_id) + if not dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + if dataset.dataset_type not in DatasetService.VECTOR_TYPES: + raise AppError( + code="INVALID_DATASET_TYPE", + message="GeoJSON dataset export requires a vector dataset", + details={"dataset_type": dataset.dataset_type}, + status_code=400, + ) + + feature_collection = DatasetService.get_dataset_geojson(db, dataset_id) + filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson") + export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename) + metadata = { + "source": "dataset", + "dataset_id": str(dataset.id), + "project_id": str(dataset.project_id), + "dataset_type": dataset.dataset_type, + "feature_count": len(feature_collection.get("features", [])), + } + export = ExportService._write_json_export( + db, + project_id=dataset.project_id, + analysis_run_id=None, + export_type="dataset_geojson", + storage_path=export_path, + content=feature_collection, + metadata=metadata, + ) + return ExportService._create_response(export) + + @staticmethod + def export_detection_run_geojson(db: Session, analysis_run_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse: + run = db.get(AnalysisRun, analysis_run_id) + if not run or run.analysis_type != "detection": + raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404) + + feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id) + filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson") + export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename) + metadata = { + "source": "detection_run", + "analysis_run_id": str(run.id), + "project_id": str(run.project_id), + "dataset_id": str(run.dataset_id) if run.dataset_id else None, + "feature_count": len(feature_collection.get("features", [])), + } + export = ExportService._write_json_export( + db, + project_id=run.project_id, + analysis_run_id=run.id, + export_type="detection_geojson", + storage_path=export_path, + content=feature_collection, + metadata=metadata, + ) + return ExportService._create_response(export) + + @staticmethod + def export_segmentation_run_geojson(db: Session, analysis_run_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse: + run = db.get(AnalysisRun, analysis_run_id) + if not run or run.analysis_type != "segmentation": + raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404) + + feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id) + filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson") + export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename) + metadata = { + "source": "segmentation_run", + "analysis_run_id": str(run.id), + "project_id": str(run.project_id), + "dataset_id": str(run.dataset_id) if run.dataset_id else None, + "feature_count": len(feature_collection.get("features", [])), + } + export = ExportService._write_json_export( + db, + project_id=run.project_id, + analysis_run_id=run.id, + export_type="segmentation_geojson", + storage_path=export_path, + content=feature_collection, + metadata=metadata, + ) + return ExportService._create_response(export) + + @staticmethod + def export_project_metadata(db: Session, project_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse: + project = db.get(Project, project_id) + if not project: + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + + content = ExportService._project_summary(db, project) + filename = ExportService._filename(name, f"{project.id}-metadata.json", ".json") + export_path = StorageService.dataset_export_path(str(project.id), "project", filename) + metadata = { + "source": "project_metadata", + "project_id": str(project.id), + "dataset_count": len(content["datasets"]), + "quality_check_count": len(content["quality_checks"]), + "export_count": len(content["exports"]), + } + export = ExportService._write_json_export( + db, + project_id=project.id, + analysis_run_id=None, + export_type="project_metadata_json", + storage_path=export_path, + content=content, + metadata=metadata, + ) + return ExportService._create_response(export) + + @staticmethod + def export_project_report(db: Session, project_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse: + project = db.get(Project, project_id) + if not project: + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + + summary = ExportService._project_summary(db, project) + html = ExportService._render_project_report_html(summary) + filename = ExportService._filename(name, f"{project.id}-report.html", ".html") + export_path = StorageService.dataset_export_path(str(project.id), "project", filename) + metadata = { + "source": "project_report", + "project_id": str(project.id), + "dataset_count": len(summary["datasets"]), + "quality_check_count": len(summary["quality_checks"]), + "export_count": len(summary["exports"]), + "format": "html", + } + export = ExportService._write_text_export( + db, + project_id=project.id, + analysis_run_id=None, + export_type="project_report_html", + storage_path=export_path, + content=html, + metadata=metadata, + ) + return ExportService._create_response(export) + + @staticmethod + def list_project_exports(db: Session, project_id: uuid.UUID, limit: int = 50, offset: int = 0) -> ExportListResponse: + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + query = db.query(Export).filter(Export.project_id == project_id).order_by(Export.created_at.desc()) + rows = query.offset(offset).limit(limit).all() + total = query.count() + return ExportListResponse( + items=[ExportRead.model_validate(row) for row in rows], + total=total, + limit=limit, + offset=offset, + ) + + @staticmethod + def get_export(db: Session, export_id: uuid.UUID) -> ExportRead: + export = db.get(Export, export_id) + if not export: + raise AppError(code="EXPORT_NOT_FOUND", message="Export not found", status_code=404) + return ExportRead.model_validate(export) + + @staticmethod + def get_export_content(db: Session, export_id: uuid.UUID) -> ExportContentResponse: + export = db.get(Export, export_id) + if not export: + raise AppError(code="EXPORT_NOT_FOUND", message="Export not found", status_code=404) + path = ExportService.get_export_download_path(db, export_id) + try: + content = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise AppError(code="EXPORT_CONTENT_INVALID", message="Export artifact is not valid JSON", status_code=422) from exc + return ExportContentResponse(export_id=export.id, export_type=export.export_type, content=content) + + @staticmethod + def get_export_download_path(db: Session, export_id: uuid.UUID) -> Path: + export = db.get(Export, export_id) + if not export: + raise AppError(code="EXPORT_NOT_FOUND", message="Export not found", status_code=404) + path = Path(export.storage_path) + if not path.exists() or not path.is_file(): + raise AppError( + code="EXPORT_CONTENT_NOT_FOUND", + message="Export artifact is missing from storage", + details={"storage_path": export.storage_path}, + status_code=404, + ) + return path + + @staticmethod + def _write_json_export( + db: Session, + *, + project_id: uuid.UUID, + analysis_run_id: uuid.UUID | None, + export_type: str, + storage_path: str, + content: dict[str, Any], + metadata: dict[str, Any], + ) -> Export: + path = Path(storage_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(content, ensure_ascii=False, indent=2), encoding="utf-8") + return ExportService._persist_export( + db, + project_id=project_id, + analysis_run_id=analysis_run_id, + export_type=export_type, + storage_path=str(path), + metadata=metadata, + ) + + @staticmethod + def _write_text_export( + db: Session, + *, + project_id: uuid.UUID, + analysis_run_id: uuid.UUID | None, + export_type: str, + storage_path: str, + content: str, + metadata: dict[str, Any], + ) -> Export: + path = Path(storage_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return ExportService._persist_export( + db, + project_id=project_id, + analysis_run_id=analysis_run_id, + export_type=export_type, + storage_path=str(path), + metadata=metadata, + ) + + @staticmethod + def _persist_export( + db: Session, + *, + project_id: uuid.UUID, + analysis_run_id: uuid.UUID | None, + export_type: str, + storage_path: str, + metadata: dict[str, Any], + ) -> Export: + export = Export( + id=uuid.uuid4(), + project_id=project_id, + analysis_run_id=analysis_run_id, + export_type=export_type, + storage_path=storage_path, + metadata_json=metadata, + ) + db.add(export) + db.commit() + db.refresh(export) + return export + + @staticmethod + def _project_summary(db: Session, project: Project) -> dict[str, Any]: + datasets = db.query(Dataset).filter(Dataset.project_id == project.id).order_by(Dataset.created_at.desc()).all() + quality_checks = ( + db.query(QualityCheck) + .filter(QualityCheck.project_id == project.id) + .order_by(QualityCheck.created_at.desc()) + .all() + ) + exports = db.query(Export).filter(Export.project_id == project.id).order_by(Export.created_at.desc()).all() + return { + "project": { + "id": str(project.id), + "name": project.name, + "description": project.description, + "region": project.region, + "status": project.status, + }, + "datasets": [ + { + "id": str(dataset.id), + "name": dataset.name, + "dataset_type": dataset.dataset_type, + "dataset_role": dataset.dataset_role, + "source_name": dataset.source_name, + "reference_layer_name": dataset.reference_layer_name, + "status": dataset.status, + "crs": dataset.crs, + "bounds_json": dataset.bounds_json, + "feature_count": (dataset.metadata_json or {}).get("feature_count"), + } + for dataset in datasets + ], + "quality_checks": [ + { + "id": str(check.id), + "analysis_run_id": str(check.analysis_run_id) if check.analysis_run_id else None, + "candidate_dataset_id": str(check.candidate_dataset_id) if check.candidate_dataset_id else None, + "reference_dataset_id": str(check.reference_dataset_id), + "check_type": check.check_type, + "status": check.status, + "score": check.score, + } + for check in quality_checks + ], + "exports": [ + { + "id": str(export.id), + "analysis_run_id": str(export.analysis_run_id) if export.analysis_run_id else None, + "export_type": export.export_type, + "storage_path": export.storage_path, + "metadata_json": export.metadata_json, + "created_at": export.created_at.isoformat() if export.created_at else None, + } + for export in exports + ], + } + + @staticmethod + def _render_project_report_html(summary: dict[str, Any]) -> str: + project = summary["project"] + datasets = summary["datasets"] + quality_checks = summary["quality_checks"] + exports = summary["exports"] + dataset_rows = "\n".join( + "" + f"{escape(str(item['name']))}" + f"{escape(str(item['dataset_type']))}" + f"{escape(str(item['dataset_role']))}" + f"{escape(str(item['status']))}" + f"{escape(str(item['feature_count'] if item['feature_count'] is not None else 'n/a'))}" + "" + for item in datasets + ) + quality_rows = "\n".join( + "" + f"{escape(str(item['check_type']))}" + f"{escape(str(item['status']))}" + f"{escape(str(item['score'] if item['score'] is not None else 'n/a'))}" + f"{escape(str(item['reference_dataset_id']))}" + "" + for item in quality_checks + ) + export_rows = "\n".join( + "" + f"{escape(str(item['export_type']))}" + f"{escape(str(item['storage_path']))}" + f"{escape(str(item['created_at'] or 'n/a'))}" + "" + for item in exports + ) + return f""" + + + + GeoIntel Project Report - {escape(str(project["name"]))} + + + +

{escape(str(project["name"]))}

+

GeoIntel project report artifact

+

Region: {escape(str(project["region"]))}

+

Status: {escape(str(project["status"]))}

+

Description: {escape(str(project["description"] or "n/a"))}

+

Datasets ({len(datasets)})

+ + + {dataset_rows or ''} +
NameTypeRoleStatusFeatures
No datasets
+

QA/QC Results ({len(quality_checks)})

+ + + {quality_rows or ''} +
CheckStatusScoreReference dataset
No QA/QC results
+

Export History ({len(exports)})

+ + + {export_rows or ''} +
TypeStorage pathCreated
No exports
+ + +""" + + @staticmethod + def _create_response(export: Export) -> ExportCreateResponse: + return ExportCreateResponse( + export_id=export.id, + path=export.storage_path, + status="ready", + export_type=export.export_type, + metadata_json=export.metadata_json, + ) + + @staticmethod + def _filename(name: str | None, fallback: str, suffix: str) -> str: + raw_name = name or fallback + cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "_", raw_name).strip("._") + if not cleaned: + cleaned = fallback + if not cleaned.lower().endswith(suffix): + cleaned = f"{cleaned}{suffix}" + return cleaned diff --git a/backend/app/services/geojson_service.py b/backend/app/services/geojson_service.py new file mode 100644 index 00000000..dc14c749 --- /dev/null +++ b/backend/app/services/geojson_service.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import json +from pyproj import Transformer, CRS +from shapely.geometry import shape +from shapely.geometry.base import BaseGeometry +from shapely.ops import unary_union +from shapely.ops import transform as _transform_geometry +from shapely.validation import make_valid + + +def parse_geojson_payload(raw_text: str | dict[str, Any]) -> dict[str, Any]: + if isinstance(raw_text, dict): + payload = raw_text + else: + try: + payload = json.loads(raw_text) + except Exception as exc: + raise ValueError("Uploaded dataset is not valid JSON") from exc + + if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection": + raise ValueError("Upload must be a GeoJSON FeatureCollection") + + features = payload.get("features") or [] + if not isinstance(features, list): + raise ValueError("FeatureCollection features is invalid") + + geometry_types: set[str] = set() + geometries = [] + invalid_features = 0 + polygon_area_m2: float | None = None + crs_assumed = None + for feature in features: + if not isinstance(feature, dict): + continue + geometry = feature.get("geometry") + if not geometry: + continue + try: + geom = shape(geometry) + except Exception as exc: + raise ValueError("Invalid feature geometry") from exc + if not geom.is_valid: + geom = make_valid(geom) + if not geom.is_valid: + invalid_features += 1 + raise ValueError("Invalid geometry remains after repair") + geometry_types.add(str(geom.geom_type)) + geometries.append(geom) + + if geometries: + unioned = unary_union(geometries) + bounds = unioned.bounds + bounds_json = { + "min_x": float(bounds[0]), + "min_y": float(bounds[1]), + "max_x": float(bounds[2]), + "max_y": float(bounds[3]), + } + else: + bounds_json = None + + crs = None + crs_assumed = False + raw_crs = payload.get("crs") + if isinstance(raw_crs, dict): + raw_name = raw_crs.get("properties", {}).get("name") + if isinstance(raw_name, str): + crs = raw_name + elif isinstance(raw_crs, str): + crs = raw_crs + if not crs: + crs = "EPSG:4326" + crs_assumed = True + + polygon_area_m2 = _approximate_polygon_area_m2(geometries, crs) + + return { + "feature_count": len(features), + "geometry_types": sorted(geometry_types), + "bounds_json": bounds_json, + "approximate_area_m2": polygon_area_m2, + "invalid_features": invalid_features, + "crs": crs, + "crs_assumed": crs_assumed, + "extracted_at": datetime.now(timezone.utc).isoformat(), + "feature_geometry_count": len(geometries), + } + + +def load_dataset_text(file_path: str) -> str: + return Path(file_path).read_text(encoding="utf-8") + + +def _approximate_polygon_area_m2(geometries: list[BaseGeometry], crs: str | None) -> float | None: + if not geometries: + return 0.0 + try: + polygons = [geometry for geometry in geometries if geometry.geom_type.lower() in {"polygon", "multipolygon"}] + if not polygons: + return None + target_crs = CRS.from_epsg(31370) + source_crs = _crs_to_epsg(crs) + transformer = Transformer.from_crs(source_crs, target_crs, always_xy=True) + projected = [_transform_polygon_for_area(geometry, transformer) for geometry in polygons] + area = sum(item.area for item in projected) + if area < 0: + area = 0.0 + return float(area) + except Exception: + return None + + +def _crs_to_epsg(value: str | None) -> str: + if not value: + return "EPSG:4326" + normalized = value.upper().strip().replace(" ", "") + if normalized.startswith("EPSG:"): + return normalized + if normalized.replace("-", "").isdigit(): + return f"EPSG:{normalized}" + return "EPSG:4326" + + +def _transform_polygon_for_area(geometry: BaseGeometry, transformer: Transformer): + if geometry.is_empty: + return geometry + if geometry.geom_type.lower() in {"polygon", "multipolygon"}: + return _transform_geometry(transformer.transform, geometry) + return geometry diff --git a/backend/app/services/job_service.py b/backend/app/services/job_service.py new file mode 100644 index 00000000..c7cf9d2f --- /dev/null +++ b/backend/app/services/job_service.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from collections.abc import Callable +from typing import Any + +from app.core.errors import AppError +from app.models import Job +from app.schemas.job import JobCreate, JobRead + + +class JobService: + VALID_STATUSES = {"queued", "running", "success", "failed"} + + @staticmethod + def run_sync_job( + db, + project_id: uuid.UUID, + job_type: str, + parameters: dict[str, Any] | None, + operation: Callable[[], Any], + input_dataset_id: uuid.UUID | None = None, + ) -> dict[str, Any]: + created = JobService.create_job( + db, + JobCreate( + job_type=job_type, + project_id=project_id, + input_dataset_id=input_dataset_id, + parameters_json=JobService._coerce_payload(parameters), + ), + ) + try: + JobService.mark_running(db, created.id) + result = operation() + output_dataset_id = None + if isinstance(result, uuid.UUID): + output_dataset_id = result + result = {"output_dataset_id": str(result)} + if isinstance(result, dict): + candidate_output_dataset_id = result.get("output_dataset_id") + if isinstance(candidate_output_dataset_id, str): + try: + output_dataset_id = uuid.UUID(candidate_output_dataset_id) + except ValueError: + output_dataset_id = None + elif isinstance(candidate_output_dataset_id, uuid.UUID): + output_dataset_id = candidate_output_dataset_id + if isinstance(result, dict): + job = JobService.mark_success(db, created.id, result=result, output_dataset_id=output_dataset_id) + else: + job = JobService.mark_success(db, created.id, result={"result": result}, output_dataset_id=output_dataset_id) + job_payload = job.model_dump() + if isinstance(job_payload.get("output_dataset_id"), uuid.UUID): + job_payload["output_dataset_id"] = str(job_payload["output_dataset_id"]) + result_json = job_payload.get("result_json") + if isinstance(result_json, dict): + if isinstance(result_json.get("output_dataset_id"), uuid.UUID): + result_json["output_dataset_id"] = str(result_json["output_dataset_id"]) + job_payload["result_json"] = result_json + return job_payload + except AppError as exc: + failed = JobService.mark_failed( + db, + created.id, + error_message=exc.message, + details={"code": exc.code, "details": exc.details}, + ) + payload = failed.model_dump() + if isinstance(payload.get("output_dataset_id"), uuid.UUID): + payload["output_dataset_id"] = str(payload["output_dataset_id"]) + result_json = payload.get("result_json") + if isinstance(result_json, dict): + if isinstance(result_json.get("output_dataset_id"), uuid.UUID): + result_json["output_dataset_id"] = str(result_json["output_dataset_id"]) + payload["result_json"] = result_json + raise + + @staticmethod + def _coerce_payload(payload: dict[str, Any] | None) -> dict[str, Any]: + return dict(payload or {}) + + @staticmethod + def create_job(db, payload: JobCreate) -> JobRead: + job = Job( + id=uuid.uuid4(), + job_type=payload.job_type, + status="queued", + project_id=payload.project_id, + dataset_id=payload.dataset_id, + input_dataset_id=payload.input_dataset_id, + output_dataset_id=payload.output_dataset_id, + parameters_json=JobService._coerce_payload(payload.parameters_json), + result_json=None, + error_message=None, + ) + db.add(job) + db.commit() + db.refresh(job) + return JobRead.model_validate(job) + + @staticmethod + def mark_running(db, job_id: uuid.UUID) -> JobRead: + job = JobService._get_job(db, job_id) + job.status = "running" + job.started_at = datetime.now(timezone.utc) + job.error_message = None + db.add(job) + db.commit() + db.refresh(job) + return JobRead.model_validate(job) + + @staticmethod + def mark_success( + db, + job_id: uuid.UUID, + result: dict[str, Any] | None = None, + output_dataset_id: uuid.UUID | None = None, + ) -> JobRead: + job = JobService._get_job(db, job_id) + job.status = "success" + job.finished_at = datetime.now(timezone.utc) + if output_dataset_id is not None: + job.output_dataset_id = output_dataset_id + job.result_json = result + job.error_message = None + db.add(job) + db.commit() + db.refresh(job) + return JobRead.model_validate(job) + + @staticmethod + def mark_failed(db, job_id: uuid.UUID, error_message: str, details: dict[str, Any] | None = None) -> JobRead: + job = JobService._get_job(db, job_id) + job.status = "failed" + job.finished_at = datetime.now(timezone.utc) + if details: + job.result_json = details + job.error_message = error_message + db.add(job) + db.commit() + db.refresh(job) + return JobRead.model_validate(job) + + @staticmethod + def get_job(db, job_id: uuid.UUID) -> JobRead: + return JobRead.model_validate(JobService._get_job(db, job_id)) + + @staticmethod + def get_job_status(db, job_id: uuid.UUID) -> dict: + job = JobService._get_job(db, job_id) + return { + "id": job.id, + "project_id": str(job.project_id), + "status": job.status, + "error_message": job.error_message, + "started_at": job.started_at, + "finished_at": job.finished_at, + "result_json": job.result_json, + } + + @staticmethod + def list_jobs( + db, + project_id: uuid.UUID | None = None, + dataset_id: uuid.UUID | None = None, + limit: int = 50, + offset: int = 0, + ) -> tuple[list[JobRead], int]: + query = db.query(Job) + if project_id is not None: + query = query.filter(Job.project_id == project_id) + if dataset_id is not None: + query = query.filter((Job.dataset_id == dataset_id) | (Job.input_dataset_id == dataset_id) | (Job.output_dataset_id == dataset_id)) + total = query.count() + rows = query.order_by(Job.created_at.desc()).offset(offset).limit(limit).all() + return [JobRead.model_validate(row) for row in rows], total + + @staticmethod + def _get_job(db, job_id: uuid.UUID) -> Job: + job = db.get(Job, job_id) + if not job: + raise AppError(code="JOB_NOT_FOUND", message="Job not found", status_code=404) + return job + + @staticmethod + def validate_status(status: str) -> None: + if status not in JobService.VALID_STATUSES: + raise AppError(code="INVALID_JOB_STATUS", message="Invalid job status", status_code=400) diff --git a/backend/app/services/model_registry_service.py b/backend/app/services/model_registry_service.py new file mode 100644 index 00000000..3334a65b --- /dev/null +++ b/backend/app/services/model_registry_service.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Type + +from app.core.config import Settings, get_settings +from app.schemas.detection import DetectionModelCapability +from app.services.yolo_adapter import YoloDetectionAdapter + + +class ModelRegistryService: + @staticmethod + def list_model_capabilities( + settings: Settings | None = None, + yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter, + task_type: str = "object_detection", + ) -> list[DetectionModelCapability]: + resolved_settings = settings or get_settings() + if task_type == "segmentation": + return ModelRegistryService.list_segmentation_model_capabilities() + if task_type != "object_detection": + return [] + return [ + DetectionModelCapability( + model_id="yolo-placeholder", + display_name="YOLO detector placeholder", + framework="ultralytics/pytorch", + task_type="object_detection", + supported_classes=["building", "road", "water", "landuse"], + configured=False, + status="not_configured", + limitation_message="YOLO/PyTorch inference is not configured in Sprint 8; no model is downloaded or executed.", + version=None, + ), + ModelRegistryService._configured_yolo_capability(resolved_settings, yolo_adapter_class), + DetectionModelCapability( + model_id="manual-fixture-detector", + display_name="Manual fixture detector", + framework="fixture", + task_type="object_detection", + supported_classes=["building"], + configured=True, + status="configured", + limitation_message="Fixture detector is for explicit tests/demo fixtures only and is not production inference.", + version="fixture-v1", + ), + ] + + @staticmethod + def get_model_capability( + model_id: str, + settings: Settings | None = None, + yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter, + task_type: str = "object_detection", + ) -> DetectionModelCapability | None: + normalized = model_id.strip() + for model in ModelRegistryService.list_model_capabilities(settings=settings, yolo_adapter_class=yolo_adapter_class, task_type=task_type): + if model.model_id == normalized: + return model + return None + + @staticmethod + def list_segmentation_model_capabilities() -> list[DetectionModelCapability]: + return [ + DetectionModelCapability( + model_id="segmentation-placeholder", + display_name="Segmentation placeholder", + framework="placeholder", + task_type="segmentation", + supported_classes=["building", "vegetation", "water", "landuse"], + configured=False, + status="not_configured", + limitation_message="Segmentation inference is not configured in Sprint 9; no SAM/YOLO-seg model is downloaded or executed.", + version=None, + ), + DetectionModelCapability( + model_id="fixture-segmenter", + display_name="Fixture segmenter", + framework="fixture", + task_type="segmentation", + supported_classes=["building", "vegetation", "water", "landuse"], + configured=True, + status="configured", + limitation_message="Fixture segmenter is for explicit tests/demo fixtures only and is not production inference.", + version="fixture-v1", + ), + DetectionModelCapability( + model_id="yolo-seg-configured", + display_name="Configured YOLO segmentation", + framework="ultralytics/pytorch", + task_type="segmentation", + supported_classes=["building", "vegetation", "water", "landuse"], + configured=False, + status="not_configured", + limitation_message="YOLO-seg is not configured in Sprint 9. GeoIntel will not download segmentation model weights automatically.", + version=None, + ), + DetectionModelCapability( + model_id="sam-configured", + display_name="Configured SAM segmentation", + framework="sam", + task_type="segmentation", + supported_classes=["building", "vegetation", "water", "landuse"], + configured=False, + status="not_configured", + limitation_message="SAM is not configured in Sprint 9 and is not installed as a backend dependency.", + version=None, + ), + ] + + @staticmethod + def _configured_yolo_capability( + settings: Settings, + yolo_adapter_class: Type[YoloDetectionAdapter], + ) -> DetectionModelCapability: + configured = False + status = "not_configured" + limitation = "YOLO is disabled. Set YOLO_ENABLED=true and YOLO_MODEL_PATH to a local model file to enable inference." + model_path = Path(settings.yolo_model_path).expanduser() if settings.yolo_model_path else None + + if settings.yolo_enabled: + if not yolo_adapter_class.dependencies_available(): + status = "dependency_unavailable" + limitation = "YOLO dependencies are not installed. Install backend optional extras with geointel-backend[ai]." + elif model_path is None: + limitation = "YOLO_MODEL_PATH is not set. GeoIntel will not download model weights automatically." + elif not model_path.exists() or not model_path.is_file(): + limitation = "YOLO_MODEL_PATH does not point to an existing local model file. GeoIntel will not download model weights automatically." + else: + configured = True + status = "configured" + limitation = "Configured for local YOLO inference over an existing raster tile manifest." + + return DetectionModelCapability( + model_id=settings.yolo_model_id, + display_name=settings.yolo_model_display_name, + framework="ultralytics/pytorch", + task_type="object_detection", + supported_classes=["building", "road", "water", "landuse"], + configured=configured, + status=status, + limitation_message=limitation, + version=settings.yolo_model_version, + ) diff --git a/backend/app/services/project_service.py b/backend/app/services/project_service.py new file mode 100644 index 00000000..177641f3 --- /dev/null +++ b/backend/app/services/project_service.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import uuid + +from sqlalchemy.orm import Session + +from app.core.errors import AppError +from app.models import Project +from app.schemas.project import ProjectCreate, ProjectRead, ProjectUpdate + + +class ProjectService: + @staticmethod + def list_projects(db: Session, limit: int = 50, offset: int = 0) -> tuple[list[ProjectRead], int]: + query = db.query(Project).filter(Project.status != "deleted").order_by(Project.created_at.desc()) + total = query.count() + items = query.offset(offset).limit(limit).all() + return [ProjectRead.model_validate(item) for item in items], total + + @staticmethod + def create_project(db: Session, payload: ProjectCreate) -> ProjectRead: + project = Project(name=payload.name.strip(), description=(payload.description or "").strip() or None, region=payload.region or "Kempen") + db.add(project) + db.commit() + db.refresh(project) + return ProjectRead.model_validate(project) + + @staticmethod + def get_project(db: Session, project_id: uuid.UUID) -> ProjectRead | None: + project = db.get(Project, project_id) + if not project or project.status == "deleted": + return None + return ProjectRead.model_validate(project) + + @staticmethod + def update_project(db: Session, project_id: uuid.UUID, payload: ProjectUpdate) -> ProjectRead | None: + project = db.get(Project, project_id) + if not project or project.status == "deleted": + return None + + payload_data = payload.model_dump(exclude_unset=True) + changed = False + for key, value in payload_data.items(): + if value is None: + continue + setattr(project, key, value) + changed = True + if not changed: + raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422) + + db.add(project) + db.commit() + db.refresh(project) + return ProjectRead.model_validate(project) + + @staticmethod + def delete_project(db: Session, project_id: uuid.UUID) -> bool: + project = db.get(Project, project_id) + if not project or project.status == "deleted": + return False + project.status = "deleted" + db.add(project) + db.commit() + return True diff --git a/backend/app/services/qa_service.py b/backend/app/services/qa_service.py new file mode 100644 index 00000000..d6a5112f --- /dev/null +++ b/backend/app/services/qa_service.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any +from uuid import UUID + +from geoalchemy2.shape import to_shape +from shapely.geometry import GeometryCollection +from shapely.geometry.base import BaseGeometry +from shapely.ops import unary_union +from shapely.validation import make_valid +from shapely.geometry import shape +from app.core.errors import AppError +from app.models import Area, Dataset +from app.schemas.qa import QaProviderComparisonResult +from app.services.vector_operations_service import VectorOperationsService + + +def _extract_crs_warnings(source_dataset: Dataset, reference_dataset: Dataset) -> list[str]: + warnings: list[str] = [] + for dataset, label in ((source_dataset, "candidate"), (reference_dataset, "reference")): + metadata = dataset.metadata_json + crs_assumed = None + if isinstance(metadata, dict): + crs_assumed = metadata.get("crs_assumed") + if crs_assumed: + warnings.append(f"CRS assumption is weak for {label} dataset ({dataset.id}); geometry metrics are approximate") + if dataset.crs is None: + warnings.append(f"Missing CRS on {label} dataset ({dataset.id})") + return warnings + + +class QaService: + SUPPORTED_GEOMETRY_TYPES = {"Polygon", "MultiPolygon"} + + @staticmethod + def _load_dataset_payload(db, dataset_id: UUID, *, expected_project_id: UUID | None = None) -> tuple[Dataset, dict[str, Any], list[tuple[dict[str, Any], BaseGeometry]]]: + dataset = db.get(Dataset, dataset_id) + if not dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + if expected_project_id is not None and dataset.project_id != expected_project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Dataset does not belong to this project", status_code=400) + if dataset.dataset_type not in {"vector", "geojson"}: + raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400) + + payload, raw_features = VectorOperationsService._load_dataset_payload(dataset) + geometries = VectorOperationsService._extract_geometries(raw_features) + return dataset, payload, geometries + + @staticmethod + def _apply_area_filter( + geometries: list[tuple[dict[str, Any], BaseGeometry]], + area_geometry: BaseGeometry, + *, + dataset_id: UUID, + ) -> list[tuple[dict[str, Any], BaseGeometry]]: + area_geom = area_geometry + if isinstance(area_geom, GeometryCollection): + area_geom = unary_union(area_geom.geoms) + + filtered: list[tuple[dict[str, Any], BaseGeometry]] = [] + for feature, feature_geometry in geometries: + clipped = feature_geometry.intersection(area_geom) + if clipped.is_empty: + continue + if not clipped.is_valid: + clipped = make_valid(clipped) + if not clipped.is_valid: + raise AppError( + code="INVALID_GEOMETRY", + message=f"Area filtering produced invalid geometry for feature in dataset {dataset_id}", + status_code=400, + ) + filtered.append((feature, clipped)) + return filtered + + @staticmethod + def _validate_area(db, area_id: UUID | None, project_id: UUID, *, dataset_ids: tuple[UUID, UUID]) -> BaseGeometry | None: + if not area_id: + return None + area = db.get(Area, area_id) + if not area: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + if area.project_id != project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400) + if area.id in dataset_ids: + raise AppError(code="INVALID_PARAMETERS", message="area_id must reference an area, not a dataset", status_code=400) + + area_geometry = to_shape(area.geometry) + if area_geometry.is_empty: + raise AppError(code="INVALID_GEOMETRY", message="Area geometry is empty", status_code=400) + return area_geometry + + @staticmethod + def _match_io_u_metrics( + source_geometries: list[tuple[dict[str, Any], BaseGeometry]], + reference_geometries: list[tuple[dict[str, Any], BaseGeometry]], + iou_threshold: float, + ) -> tuple[int, int, int, list[float], list[str], bool]: + source_supported = [ + (feature, geom) for feature, geom in source_geometries if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES + ] + reference_supported = [ + (feature, geom) + for feature, geom in reference_geometries + if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES + ] + + unsupported = sorted( + { + geom.geom_type + for _, geom in source_geometries + reference_geometries + if geom.geom_type not in QaService.SUPPORTED_GEOMETRY_TYPES + } + ) + if not source_supported or not reference_supported: + return ( + 0, + len(source_supported), + len(reference_supported), + [], + [f"Unsupported geometry types: {unsupported}"] if unsupported else [], + True, + ) + + unmatched_reference_indices = set(range(len(reference_supported))) + matches = 0 + match_iou_values: list[float] = [] + false_positives = 0 + + for _, source_geom in source_supported: + if source_geom.area <= 0: + false_positives += 1 + continue + + best_iou = 0.0 + best_index = None + for reference_index in list(unmatched_reference_indices): + _, reference_geom = reference_supported[reference_index] + if reference_geom.area <= 0: + unmatched_reference_indices.discard(reference_index) + continue + try: + intersection = source_geom.intersection(reference_geom) + except Exception as exc: # pragma: no cover - robustness path + raise AppError(code="GEOMETRY_OPERATION_UNSUPPORTED", message="Geometry operations failed", details={"reason": str(exc)}, status_code=422) + + if intersection.is_empty: + continue + + intersection_area = intersection.area + if intersection_area < 0: + intersection_area = 0.0 + union_area = source_geom.area + reference_geom.area - intersection_area + if union_area <= 0: + continue + + candidate_iou = intersection_area / union_area + if candidate_iou > best_iou: + best_iou = candidate_iou + best_index = reference_index + + if best_index is not None and best_iou >= iou_threshold: + matches += 1 + match_iou_values.append(best_iou) + unmatched_reference_indices.discard(best_index) + else: + false_positives += 1 + + false_negatives = len(unmatched_reference_indices) + warnings: list[str] = [f"Unsupported geometry types: {unsupported}"] if unsupported else [] + + return matches, false_positives, false_negatives, match_iou_values, warnings, bool(unsupported) + + @staticmethod + def compare_candidate_with_reference( + db, + project_id: UUID, + candidate_dataset_id: UUID, + reference_dataset_id: UUID, + iou_threshold: float = 0.5, + area_id: UUID | None = None, + ) -> QaProviderComparisonResult: + if candidate_dataset_id == reference_dataset_id: + raise AppError(code="INVALID_PARAMETERS", message="Candidate and reference dataset must differ", status_code=400) + + candidate_dataset, candidate_payload, candidate_geometries = QaService._load_dataset_payload( + db, + candidate_dataset_id, + expected_project_id=project_id, + ) + reference_dataset, reference_payload, reference_geometries = QaService._load_dataset_payload( + db, + reference_dataset_id, + expected_project_id=project_id, + ) + + area_geometry = QaService._validate_area( + db, + area_id=area_id, + project_id=project_id, + dataset_ids=(candidate_dataset_id, reference_dataset_id), + ) + + if area_geometry is not None: + candidate_geometries = QaService._apply_area_filter(candidate_geometries, area_geometry, dataset_id=candidate_dataset.id) + reference_geometries = QaService._apply_area_filter(reference_geometries, area_geometry, dataset_id=reference_dataset.id) + + matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics( + candidate_geometries, + reference_geometries, + iou_threshold, + ) + + candidate_feature_count = len(candidate_payload.get("features", [])) if isinstance(candidate_payload, dict) else 0 + reference_feature_count = len(reference_payload.get("features", [])) if isinstance(reference_payload, dict) else 0 + mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values) + + precision = None + if matches + false_positives > 0: + precision = matches / (matches + false_positives) + + recall = None + if matches + false_negatives > 0: + recall = matches / (matches + false_negatives) + + f1_score = None + if precision is not None and recall is not None and precision + recall > 0: + f1_score = (2 * precision * recall) / (precision + recall) + + status = "unsupported" if unsupported else "ok" + return QaProviderComparisonResult( + status=status, + warnings=_extract_crs_warnings(candidate_dataset, reference_dataset) + warnings, + candidate_feature_count=candidate_feature_count, + reference_feature_count=reference_feature_count, + matches=matches, + false_positives=false_positives, + false_negatives=false_negatives, + precision=precision, + recall=recall, + f1_score=f1_score, + mean_iou=mean_iou, + iou_threshold=iou_threshold, + unsupported_geometry=unsupported, + unsupported_geometries=warnings, + generated_at=datetime.now(timezone.utc), + ) diff --git a/backend/app/services/quality_check_service.py b/backend/app/services/quality_check_service.py new file mode 100644 index 00000000..c1e242dc --- /dev/null +++ b/backend/app/services/quality_check_service.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models import Metric, QualityCheck +from app.schemas.qa import MetricRead, QualityCheckRead + + +class QualityCheckService: + @staticmethod + def list_quality_checks( + db: Session, + *, + project_id: UUID, + limit: int = 50, + offset: int = 0, + ) -> tuple[list[QualityCheckRead], int]: + query = ( + db.query(QualityCheck) + .filter(QualityCheck.project_id == project_id) + .order_by(QualityCheck.created_at.desc()) + ) + total = query.count() + rows = query.offset(offset).limit(limit).all() + if not rows: + return [], total + + quality_check_ids = [row.id for row in rows] + metrics_by_quality_check: dict[UUID, list[MetricRead]] = {row.id: [] for row in rows} + metrics = ( + db.query(Metric) + .filter(Metric.quality_check_id.in_(quality_check_ids)) + .order_by(Metric.created_at.asc()) + .all() + ) + for metric in metrics: + if metric.quality_check_id in metrics_by_quality_check: + metrics_by_quality_check[metric.quality_check_id].append(MetricRead.model_validate(metric)) + + return [ + QualityCheckRead.model_validate(row).model_copy( + update={"metrics": metrics_by_quality_check.get(row.id, [])} + ) + for row in rows + ], total diff --git a/backend/app/services/quality_service.py b/backend/app/services/quality_service.py new file mode 100644 index 00000000..cfd01444 --- /dev/null +++ b/backend/app/services/quality_service.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from app.models import Metric, QualityCheck + + +class QualityService: + @staticmethod + def persist_quality_check( + db, + project_id: UUID, + reference_dataset_id: UUID, + check_type: str, + status: str, + score: float | None, + parameters: dict | None, + findings: dict | None, + *, + job_id: UUID | None = None, + analysis_run_id: UUID | None = None, + candidate_dataset_id: UUID | None = None, + metrics: dict[str, float | int | None] | None = None, + commit: bool = True, + ) -> QualityCheck: + quality_check = QualityCheck( + id=uuid4(), + project_id=project_id, + job_id=job_id, + analysis_run_id=analysis_run_id, + candidate_dataset_id=candidate_dataset_id, + reference_dataset_id=reference_dataset_id, + check_type=check_type, + status=status, + score=score, + parameters_json=parameters or {}, + findings_json=findings or {}, + completed_at=datetime.now(timezone.utc), + ) + db.add(quality_check) + + for key, value in (metrics or {}).items(): + db.add( + Metric( + id=uuid4(), + quality_check_id=quality_check.id, + analysis_run_id=analysis_run_id, + metric_key=key, + metric_value=float(value) if value is not None else None, + metadata_json={}, + ) + ) + + if commit: + db.commit() + db.refresh(quality_check) + return quality_check diff --git a/backend/app/services/raster_operations_service.py b/backend/app/services/raster_operations_service.py new file mode 100644 index 00000000..aabd03c6 --- /dev/null +++ b/backend/app/services/raster_operations_service.py @@ -0,0 +1,1022 @@ +from __future__ import annotations + +import json +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from geoalchemy2.shape import to_shape +from shapely.geometry import mapping +from shapely.ops import transform as shapely_transform +from shapely.validation import make_valid + +from app.core.errors import AppError +from app.models import Area, Dataset +from app.services.raster_service import extract_raster_metadata +from app.services.storage_service import StorageService + + +def _import_rasterio(): + import importlib + + rasterio = importlib.import_module("rasterio") + errors = importlib.import_module("rasterio.errors") + return rasterio, errors + + +def _import_numpy(): + import importlib + + return importlib.import_module("numpy") + + +def _import_pillow(): + import importlib + + return importlib.import_module("PIL") + + +class RasterOperationsService: + RASTER_UNAVAILABLE_MESSAGE = ( + "Raster processing unavailable. Install rasterio and GDAL-compatible drivers to enable raster processing operations." + ) + RASTER_STATS_UNAVAILABLE_MESSAGE = ( + "Raster statistics unavailable. Install rasterio and numpy to enable raster band statistics." + ) + RASTER_INDEX_UNAVAILABLE_MESSAGE = ( + "Raster processing unavailable. Install rasterio and numpy to enable raster index operations." + ) + PREVIEW_UNAVAILABLE_MESSAGE = "Raster preview unavailable. Install rasterio, numpy and pillow to enable preview generation." + DEFAULT_REPROJECT_CRS = "EPSG:31370" + DEFAULT_STATS_HISTOGRAM_BINS = 16 + + @staticmethod + def _require_raster_dataset(dataset: Dataset) -> None: + if dataset.dataset_type != "raster": + raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a raster dataset", status_code=400) + if not dataset.storage_path: + raise AppError(code="DATASET_FILE_MISSING", message="Stored raster file is missing", status_code=404) + + @staticmethod + def _load_dataset(db, dataset_id: uuid.UUID) -> Dataset: + dataset = db.get(Dataset, dataset_id) + if not dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + RasterOperationsService._require_raster_dataset(dataset) + source_path = Path(dataset.storage_path) + if not source_path.exists(): + raise AppError(code="DATASET_FILE_MISSING", message="Stored raster file missing", status_code=404) + return dataset + + @staticmethod + def _raster_dependencies() -> tuple[Any, Any]: + try: + return _import_rasterio() + except Exception as exc: + raise AppError( + code="RASTER_PROCESSING_UNAVAILABLE", + message=RasterOperationsService.RASTER_UNAVAILABLE_MESSAGE, + status_code=503, + ) from exc + + @staticmethod + def _stats_dependencies() -> tuple[Any, Any]: + rasterio, _ = RasterOperationsService._raster_dependencies() + try: + numpy = _import_numpy() + except Exception as exc: + raise AppError( + code="RASTER_PROCESSING_UNAVAILABLE", + message=RasterOperationsService.RASTER_STATS_UNAVAILABLE_MESSAGE, + status_code=503, + ) from exc + return rasterio, numpy + + @staticmethod + def _index_dependencies() -> tuple[Any, Any]: + rasterio, _ = RasterOperationsService._raster_dependencies() + try: + numpy = _import_numpy() + except Exception as exc: + raise AppError( + code="RASTER_PROCESSING_UNAVAILABLE", + message=RasterOperationsService.RASTER_INDEX_UNAVAILABLE_MESSAGE, + status_code=503, + ) from exc + return rasterio, numpy + + @staticmethod + def _validate_positive_band_index(value: int, label: str) -> int: + if not isinstance(value, int): + raise AppError(code="INVALID_PARAMETERS", message=f"{label} must be a positive integer", status_code=400) + if value <= 0: + raise AppError(code="INVALID_PARAMETERS", message=f"{label} must be greater than 0", status_code=400) + return value + + @staticmethod + def _normalize_nodata(value: Any) -> float | int | None: + if value is None: + return None + if isinstance(value, (list, tuple)): + if not value: + return None + value = value[0] + if value == "": + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + @staticmethod + def _normalize_nodata_for_band(nodata: Any, band_index: int) -> float | int | None: + if isinstance(nodata, (list, tuple)): + if band_index <= 0 or band_index > len(nodata): + return None + return RasterOperationsService._normalize_nodata(nodata[band_index - 1]) + return RasterOperationsService._normalize_nodata(nodata) + + @staticmethod + def _validate_tile_request(tile_size: int, overlap: int) -> None: + if tile_size <= 0: + raise AppError(code="INVALID_PARAMETERS", message="tile_size must be greater than 0", status_code=400) + if overlap < 0: + raise AppError(code="INVALID_PARAMETERS", message="overlap must be greater or equal to 0", status_code=400) + if overlap >= tile_size: + raise AppError(code="INVALID_PARAMETERS", message="overlap must be smaller than tile_size", status_code=400) + + @staticmethod + def _dataset_metadata(dataset_id: uuid.UUID, storage: dict[str, Any], extra: dict[str, Any] | None = None) -> dict[str, Any]: + metadata = { + "dataset_id": str(dataset_id), + "size_bytes": storage.get("size_bytes"), + "checksum_sha256": storage.get("checksum_sha256"), + "path": storage.get("storage_path"), + } + if extra: + metadata.update(extra) + return metadata + + @staticmethod + def _validate_band_mapping(dataset: Dataset, source_band_count: int, mapping: dict[str, int]) -> dict[str, int]: + if source_band_count <= 0: + raise AppError(code="INVALID_DATASET", message="Source raster has no bands", status_code=400) + validated: dict[str, int] = {} + for name, value in mapping.items(): + band_index = RasterOperationsService._validate_positive_band_index(value, name) + if band_index > source_band_count: + raise AppError( + code="INVALID_PARAMETERS", + message=f"{name} exceeds available band count ({band_index} > {source_band_count})", + status_code=400, + ) + validated[name] = band_index + return validated + + @staticmethod + def _coerce_rasterio_crs(rasterio: Any, value: str | None) -> Any: + if not value: + raise ValueError("CRS value is missing") + crs_namespace = getattr(rasterio, "crs", rasterio) + crs_class = getattr(crs_namespace, "CRS", crs_namespace) + if hasattr(crs_class, "from_user_input"): + return crs_class.from_user_input(value) + raise AttributeError("rasterio CRS converter unavailable") + + @staticmethod + def _preview_dimensions(source_width: int, source_height: int, max_dimension: int = 2048) -> tuple[int, int]: + width = max(1, int(source_width)) + height = max(1, int(source_height)) + preview_width = min(width, max_dimension) + preview_height = int(height * (preview_width / width)) + if preview_height <= 0: + preview_height = 1 + if preview_height > max_dimension: + preview_height = max_dimension + preview_width = int(width * (preview_height / height)) + if preview_width <= 0: + preview_width = 1 + return preview_width, preview_height + + @staticmethod + def _normalize_preview_data(data: Any) -> Any: + try: + if isinstance(data, (list, tuple)) and len(data) > 0: + return data[0] + except Exception: + pass + return data + + @staticmethod + def _write_preview_image( + data: Any, + preview_path: Path, + preview_width: int | None = None, + preview_height: int | None = None, + ) -> tuple[int, int]: + _import_pillow() + numpy = _import_numpy() + image_data = numpy.asarray(RasterOperationsService._normalize_preview_data(data)) + + if image_data.size == 0: + raise AppError(code="RASTER_PREVIEW_ERROR", message="Cannot generate preview for empty raster", status_code=422) + + if image_data.ndim > 2: + image_data = image_data[0] + if image_data.ndim != 2: + raise AppError(code="RASTER_PREVIEW_ERROR", message="Cannot generate preview for raster shape", status_code=500) + + valid = numpy.isfinite(image_data) + if valid.any(): + valid_values = image_data.astype("float64")[valid] + minimum = float(valid_values.min()) + maximum = float(valid_values.max()) + scale = maximum - minimum + if scale == 0: + scale = 1.0 + normalized = ((image_data.astype("float64") - minimum) / scale * 255).clip(0, 255) + normalized = normalized.astype("uint8") + else: + normalized = numpy.zeros(image_data.shape, dtype="uint8") + + from PIL import Image + + image = Image.fromarray(normalized, mode="L") + if preview_width is not None and preview_height is not None and ( + preview_width != image.width or preview_height != image.height + ): + image = image.resize( + (int(preview_width), int(preview_height)), + resample=getattr(Image.Resampling, "LANCZOS", Image.BICUBIC), + ) + + preview_path.parent.mkdir(parents=True, exist_ok=True) + image.save(preview_path) + return int(image.width), int(image.height) + + @staticmethod + def _transform_area_area_geometry(area_geom, area: Area, source_crs_str: str) -> Any: + if not area.original_crs: + raise AppError( + code="INVALID_CRS", + message="Area CRS is required to align clipping geometry with raster CRS.", + status_code=400, + ) + if area.original_crs == source_crs_str: + return area_geom + + try: + import pyproj + except Exception as exc: + raise AppError(code="INVALID_CRS", message="pyproj is required to reproject clip area", status_code=400) from exc + + try: + transformer = pyproj.Transformer.from_crs(area.original_crs, source_crs_str, always_xy=True) + return shapely_transform(transformer.transform, area_geom) + except Exception as exc: + raise AppError(code="INVALID_CRS", message="Unable to align area CRS to raster CRS", status_code=400) from exc + + @staticmethod + def _persist_derived_dataset( + db, + source_dataset: Dataset, + source_dataset_id: uuid.UUID, + operation: str, + output_path: str, + output_name: str, + metadata: dict[str, Any], + ) -> uuid.UUID: + derived_id = uuid.uuid4() + output_file = Path(output_path) + if output_file.suffix.lower() not in {".tif", ".tiff", ".geotiff"}: + output_file = output_file.with_suffix(".tif") + + storage_metadata: dict[str, Any] = {} + if output_file.exists(): + storage_metadata = { + "size_bytes": output_file.stat().st_size, + "checksum_sha256": StorageService.calculate_checksum_sha256(output_file.read_bytes()), + } + storage_metadata.update( + { + "original_filename": output_file.name, + "stored_filename": output_file.name, + "content_type": "image/tiff", + "storage_path": str(output_file), + }, + ) + + metadata_payload = dict(metadata or {}) + operation_name = operation if operation.startswith("raster.") else f"raster.{operation}" + provenance = { + "operation": operation_name, + "source_dataset_id": str(source_dataset_id), + "input_dataset_id": str(source_dataset_id), + "operation_parameters": metadata_payload.get("operation_parameters", {}), + } + metadata_payload.setdefault("operation", operation_name) + metadata_payload.update(provenance) + metadata_payload.setdefault("output_dataset_id", str(derived_id)) + + derived_dataset = Dataset( + id=derived_id, + project_id=source_dataset.project_id, + area_id=source_dataset.area_id, + name=output_name, + dataset_type="raster", + source=f"operation:{operation_name}", + storage_path=str(output_file), + original_filename=storage_metadata["original_filename"], + stored_filename=storage_metadata["stored_filename"], + content_type=storage_metadata["content_type"], + size_bytes=storage_metadata.get("size_bytes"), + checksum_sha256=storage_metadata.get("checksum_sha256"), + derived_from_dataset_id=source_dataset_id, + crs=metadata_payload.get("crs"), + bounds_json=metadata_payload.get("bounds"), + resolution_json=metadata_payload.get("resolution"), + bands_json={"dtype": metadata_payload.get("dtype")} if metadata_payload.get("dtype") is not None else None, + metadata_json=metadata_payload, + status="ready", + ) + db.add(derived_dataset) + db.commit() + db.refresh(derived_dataset) + return derived_id + + @staticmethod + def metadata(db, dataset_id: uuid.UUID) -> dict[str, Any]: + dataset = RasterOperationsService._load_dataset(db, dataset_id) + metadata = extract_raster_metadata(dataset.storage_path) + metadata["dataset_id"] = str(dataset.id) + metadata["size_bytes"] = dataset.size_bytes + metadata["checksum_sha256"] = dataset.checksum_sha256 + metadata["path"] = dataset.storage_path + return metadata + + @staticmethod + def inspect(db, dataset_id: uuid.UUID) -> dict[str, Any]: + dataset = RasterOperationsService._load_dataset(db, dataset_id) + profile = RasterOperationsService.metadata(db, dataset_id) + return { + "dataset_id": str(dataset.id), + "ready": True, + "metadata": profile, + "operation": "raster.inspect", + "output_dataset_id": None, + "source_dataset_id": None, + } + + @staticmethod + def preview(db, dataset_id: uuid.UUID) -> dict[str, Any]: + dataset = RasterOperationsService._load_dataset(db, dataset_id) + rasterio, _ = RasterOperationsService._raster_dependencies() + + preview_dir = StorageService.preview_root(str(dataset.project_id), str(dataset.id)) + preview_dir.mkdir(parents=True, exist_ok=True) + preview_path = preview_dir / "preview.png" + + try: + with rasterio.open(dataset.storage_path) as source: + width = int(source.width) + height = int(source.height) + preview_width, preview_height = RasterOperationsService._preview_dimensions(width, height) + if not preview_path.exists(): + data = source.read(1) + try: + preview_width, preview_height = RasterOperationsService._write_preview_image( + data=data, + preview_path=preview_path, + preview_width=preview_width, + preview_height=preview_height, + ) + except TypeError: + preview_width, preview_height = RasterOperationsService._write_preview_image(data, preview_path) + else: + try: + from PIL import Image + + with Image.open(preview_path) as cached: + preview_width = int(cached.width) + preview_height = int(cached.height) + except Exception: + # best effort fallback; keep computed dimensions. + pass + except AppError: + raise + except Exception as exc: # pragma: no cover + if isinstance(exc, AppError): + raise + raise AppError(code="RASTER_PREVIEW_ERROR", message="Unable to generate raster preview", status_code=500) from exc + + metadata = RasterOperationsService._dataset_metadata( + dataset.id, + { + "storage_path": dataset.storage_path, + "size_bytes": dataset.size_bytes, + "checksum_sha256": dataset.checksum_sha256, + }, + extra=extract_raster_metadata(dataset.storage_path), + ) + return { + "dataset_id": str(dataset.id), + "ready": True, + "preview": { + "path": str(preview_path), + "format": "PNG", + "width": preview_width, + "height": preview_height, + }, + "metadata": metadata, + "operation": "raster.preview", + "source_dataset_id": str(dataset.id), + } + + @staticmethod + def _compute_spectral_index( + db, + dataset_id: uuid.UUID, + mapping: dict[str, int], + operation_name: str, + formula: str, + output_name: str, + subtraction_order: str = "second_minus_first", + ) -> uuid.UUID: + dataset = RasterOperationsService._load_dataset(db, dataset_id) + rasterio, numpy = RasterOperationsService._index_dependencies() + + output_id = uuid.uuid4() + output_filename = f"{output_name or operation_name}.tif" + output_path = Path(StorageService.derived_raster_root(str(dataset.project_id), str(output_id)) / output_filename) + output_path.parent.mkdir(parents=True, exist_ok=True) + + with rasterio.open(dataset.storage_path) as source: + source_band_count = int(source.count) + validated_mapping = RasterOperationsService._validate_band_mapping( + dataset=dataset, + source_band_count=source_band_count, + mapping={str(key): int(value) for key, value in mapping.items()}, + ) + first_key = [key for key in ("red_band", "green_band", "swir_band") if key in validated_mapping][0] + first_band = validated_mapping[first_key] + second_band = validated_mapping["nir_band"] + + source_profile = source.profile.copy() + source_profile.update( + { + "count": 1, + "dtype": "float32", + "nodata": float("nan"), + }, + ) + + block_size = max(1, min(1024, int(source.width), int(source.height))) + with rasterio.open(output_path, "w", **source_profile) as destination: + for yoff in range(0, int(source.height), block_size): + row_count = min(block_size, int(source.height) - yoff) + for xoff in range(0, int(source.width), block_size): + column_count = min(block_size, int(source.width) - xoff) + window = rasterio.windows.Window(xoff, yoff, column_count, row_count) + first_data = numpy.asarray( + source.read(first_band, window=window, out_dtype="float32"), + ).astype("float32") + second_data = numpy.asarray( + source.read(second_band, window=window, out_dtype="float32"), + ).astype("float32") + + nodata = source.nodata + first_nodata = RasterOperationsService._normalize_nodata_for_band(nodata, first_band) + second_nodata = RasterOperationsService._normalize_nodata_for_band(nodata, second_band) + + valid = numpy.isfinite(first_data) & numpy.isfinite(second_data) + if first_nodata is not None: + valid &= first_data != first_nodata + if second_nodata is not None: + valid &= second_data != second_nodata + + denominator = first_data + second_data + computed = numpy.full_like(first_data, float("nan"), dtype="float32") + if not numpy.all(~valid): + np_valid = valid.astype(bool) + if np_valid.any(): + safe_denominator = denominator.copy() + safe_denominator[~np_valid] = 1.0 + with numpy.errstate(divide="ignore", invalid="ignore", over="ignore", under="ignore"): + if subtraction_order == "first_minus_second": + difference = first_data - second_data + else: + difference = second_data - first_data + computed_values = difference / safe_denominator + computed[~np_valid] = float("nan") + computed[np_valid] = numpy.where( + (first_data[np_valid] + second_data[np_valid]) == 0.0, + float("nan"), + computed_values[np_valid], + ) + destination.write(computed, indexes=1, window=window) + + output_metadata = extract_raster_metadata(str(output_path)) + output_metadata["operation"] = f"raster.{operation_name}" + output_metadata["source_dataset_id"] = str(dataset.id) + output_metadata["operation_parameters"] = { + **validated_mapping, + "formula": formula, + "nodata_strategy": "nan", + "source_band_count": source_band_count, + } + output_metadata["band_mapping"] = validated_mapping + output_metadata["formula"] = formula + output_metadata["output_dtype"] = "float32" + output_metadata["nodata_strategy"] = { + "mode": "nan", + "value_range_note": "Expected index range is approximately [-1, 1] before optional clipping.", + } + output_metadata["created_at"] = datetime.now(timezone.utc).isoformat() + output_metadata["path"] = str(output_path) + output_metadata["output_dataset_id"] = str(output_id) + + derived_id = RasterOperationsService._persist_derived_dataset( + db=db, + source_dataset=dataset, + source_dataset_id=dataset.id, + operation=f"raster.{operation_name}", + output_path=str(output_path), + output_name=output_filename, + metadata=output_metadata, + ) + return derived_id + + @staticmethod + def ndvi(db, dataset_id: uuid.UUID, nir_band: int, red_band: int, output_name: str | None = None) -> uuid.UUID: + return RasterOperationsService._compute_spectral_index( + db=db, + dataset_id=dataset_id, + mapping={"nir_band": nir_band, "red_band": red_band}, + operation_name="ndvi", + formula="(nir - red) / (nir + red)", + output_name=(output_name or "ndvi"), + ) + + @staticmethod + def ndwi(db, dataset_id: uuid.UUID, green_band: int, nir_band: int, output_name: str | None = None) -> uuid.UUID: + return RasterOperationsService._compute_spectral_index( + db=db, + dataset_id=dataset_id, + mapping={"green_band": green_band, "nir_band": nir_band}, + operation_name="ndwi", + formula="(nir - green) / (nir + green)", + output_name=(output_name or "ndwi"), + ) + + @staticmethod + def ndbi(db, dataset_id: uuid.UUID, swir_band: int, nir_band: int, output_name: str | None = None) -> uuid.UUID: + return RasterOperationsService._compute_spectral_index( + db=db, + dataset_id=dataset_id, + mapping={"swir_band": swir_band, "nir_band": nir_band}, + operation_name="ndbi", + formula="(swir - nir) / (swir + nir)", + output_name=(output_name or "ndbi"), + subtraction_order="first_minus_second", + ) + + @staticmethod + def stats(db, dataset_id: uuid.UUID) -> dict[str, Any]: + dataset = RasterOperationsService._load_dataset(db, dataset_id) + rasterio, numpy = RasterOperationsService._stats_dependencies() + with rasterio.open(dataset.storage_path) as source: + height = int(source.height) + width = int(source.width) + count = int(source.count) + dataset_profile = extract_raster_metadata(dataset.storage_path) + metadata = { + "dataset_id": str(dataset.id), + "source_dataset_id": str(dataset.id), + "size_bytes": dataset.size_bytes, + "checksum_sha256": dataset.checksum_sha256, + "profile": dataset_profile, + } + + bands = [] + chunk_rows = max(1, min(2048, height)) + for band_index in range(1, count + 1): + nodata = RasterOperationsService._normalize_nodata_for_band(source.nodata, band_index) + dtype = str(source.dtypes[band_index - 1]) if source.dtypes else None + band_min = None + band_max = None + valid_count = 0 + total_sum = 0.0 + total_sq = 0.0 + nodata_count = 0 + hist = None + hist_bins = None + + for row_offset in range(0, height, chunk_rows): + row_count = min(chunk_rows, height - row_offset) + data = source.read(band_index, window=rasterio.windows.Window(0, row_offset, width, row_count)) + values = numpy.asarray(data) + if values.size == 0: + continue + + finite = numpy.isfinite(values) + if nodata is not None: + valid = finite & (values != nodata) + nodata_count += int(values.size - valid.sum()) + else: + valid = finite + + band_values = values[valid].astype("float64") + if band_values.size == 0: + continue + + current_min = float(band_values.min()) + current_max = float(band_values.max()) + if band_min is None or current_min < band_min: + band_min = current_min + if band_max is None or current_max > band_max: + band_max = current_max + + valid_count += int(band_values.size) + total_sum += float(band_values.sum()) + total_sq += float((band_values**2).sum()) + + if hist is None: + hist, hist_bins = numpy.histogram(band_values, bins=RasterOperationsService.DEFAULT_STATS_HISTOGRAM_BINS) + else: + additional, _ = numpy.histogram(band_values, bins=hist_bins) + hist = hist + additional + + total_pixels = width * height + if valid_count == 0: + bands.append( + { + "band_index": band_index, + "dtype": dtype, + "min": None, + "max": None, + "mean": None, + "std": None, + "nodata_count": int(nodata_count), + "nodata_ratio": 1.0 if total_pixels else 0.0, + "valid_pixel_count": 0, + "histogram": None, + "histogram_bins": None, + }, + ) + continue + + mean = total_sum / valid_count + variance = max(0.0, (total_sq / valid_count) - (mean * mean)) + std = float(numpy.sqrt(variance)) + bands.append( + { + "band_index": band_index, + "dtype": dtype, + "min": float(band_min) if band_min is not None else None, + "max": float(band_max) if band_max is not None else None, + "mean": float(mean), + "std": float(std), + "nodata_count": int(nodata_count), + "nodata_ratio": float(nodata_count) / max(1, total_pixels), + "valid_pixel_count": int(valid_count), + "histogram": hist.astype(int).tolist() if hist is not None else None, + "histogram_bins": [float(item) for item in hist_bins] if hist_bins is not None else None, + }, + ) + + return { + "dataset_id": str(dataset.id), + "source_dataset_id": str(dataset.id), + "bands": bands, + "generated_at": datetime.now(timezone.utc).isoformat(), + "metadata": metadata, + } + + @staticmethod + def reproject( + db, + dataset_id: uuid.UUID, + target_crs: str | None, + output_name: str | None, + resampling: str = "nearest", + ) -> uuid.UUID: + dataset = RasterOperationsService._load_dataset(db, dataset_id) + target_crs = target_crs or RasterOperationsService.DEFAULT_REPROJECT_CRS + rasterio, _ = RasterOperationsService._raster_dependencies() + + try: + target = RasterOperationsService._coerce_rasterio_crs(rasterio, target_crs) + except Exception as exc: + raise AppError(code="INVALID_PARAMETERS", message="Invalid target CRS", status_code=400) from exc + + if not hasattr(target, "to_string"): + raise AppError(code="INVALID_CRS", message="Invalid target CRS", status_code=400) + + resampling_map = { + "nearest": getattr(rasterio.enums.Resampling, "nearest", None), + "bilinear": getattr(rasterio.enums.Resampling, "bilinear", None), + "cubic": getattr(rasterio.enums.Resampling, "cubic", None), + } + selected_resampling = resampling_map.get(resampling or "nearest") + if selected_resampling is None: + raise AppError(code="INVALID_PARAMETERS", message="Unsupported resampling method", status_code=400) + + output_name = (output_name or "raster_reprojected").strip() or "raster_reprojected" + output_id = uuid.uuid4() + output_filename = f"{output_name}.tif" + output_path = Path(StorageService.derived_raster_root(str(dataset.project_id), str(output_id)) / output_filename) + output_path.parent.mkdir(parents=True, exist_ok=True) + + with rasterio.open(dataset.storage_path) as source: + if not source.crs: + raise AppError(code="INVALID_DATASET_CRS", message="Source raster CRS is missing", status_code=400) + + source_transform = source.transform + source_crs = source.crs + output_kwargs = source.meta.copy() + source_bounds = getattr(source, "bounds", None) + try: + if source_bounds is not None: + source_bounds_tuple = ( + source_bounds.left, + source_bounds.bottom, + source_bounds.right, + source_bounds.top, + ) + else: + raise AttributeError + except Exception: + source_bounds_tuple = ( + 0.0, + 0.0, + float(source.width), + float(source.height), + ) + transform, width, height = rasterio.warp.calculate_default_transform( + source_crs, + target, + source.width, + source.height, + *source_bounds_tuple, + ) + output_kwargs.update( + { + "crs": target, + "transform": transform, + "width": int(width), + "height": int(height), + "count": source.count, + }, + ) + + with rasterio.open(output_path, "w", **output_kwargs) as destination: + for band_index in range(1, source.count + 1): + source_band_reader = rasterio.band + destination_band_reader = rasterio.band + if hasattr(source_band_reader, "__self__"): + source_band_reader = getattr(rasterio.__class__, "band", source_band_reader) + if hasattr(destination_band_reader, "__self__"): + destination_band_reader = getattr(rasterio.__class__, "band", destination_band_reader) + source_band = source_band_reader(source, band_index) + destination_band = destination_band_reader(destination, band_index) + rasterio.warp.reproject( + source=source_band, + destination=destination_band, + src_transform=source_transform, + src_crs=source_crs, + dst_transform=transform, + dst_crs=target, + resampling=selected_resampling, + ) + + output_metadata = extract_raster_metadata(str(output_path)) + output_metadata["operation"] = "raster.reproject" + output_metadata["source_dataset_id"] = str(dataset.id) + output_metadata["operation_parameters"] = { + "target_crs": target_crs, + "resampling": resampling, + } + output_metadata["target_crs"] = target_crs + output_metadata["output_dataset_id"] = str(output_id) + + derived_id = RasterOperationsService._persist_derived_dataset( + db=db, + source_dataset=dataset, + source_dataset_id=dataset.id, + operation="raster.reproject", + output_path=str(output_path), + output_name=output_filename, + metadata=output_metadata, + ) + return derived_id + + @staticmethod + def clip(db, dataset_id: uuid.UUID, area_id: uuid.UUID, output_name: str | None) -> uuid.UUID: + dataset = RasterOperationsService._load_dataset(db, dataset_id) + rasterio, _ = RasterOperationsService._raster_dependencies() + + area = db.get(Area, area_id) + if not area: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + if area.project_id != dataset.project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to dataset project", status_code=400) + + if not area.geometry: + raise AppError(code="INVALID_AREA_GEOMETRY", message="Area geometry is missing", status_code=400) + + area_geom = to_shape(area.geometry) + if not area_geom.is_valid: + area_geom = make_valid(area_geom) + if not area_geom.is_valid: + raise AppError(code="INVALID_GEOMETRY", message="Area geometry cannot be repaired", status_code=400) + if area_geom.is_empty: + raise AppError(code="INVALID_AREA_GEOMETRY", message="Area geometry is empty", status_code=400) + + with rasterio.open(dataset.storage_path) as source: + raw_source_crs = source.crs + source_crs = raw_source_crs.to_string() if hasattr(raw_source_crs, "to_string") else ( + str(raw_source_crs) if raw_source_crs else None + ) + if not source_crs: + raise AppError(code="INVALID_DATASET_CRS", message="Source raster CRS is missing", status_code=400) + + transformed_area = RasterOperationsService._transform_area_area_geometry(area_geom, area, source_crs) + if not transformed_area.is_valid: + transformed_area = make_valid(transformed_area) + if not transformed_area.is_valid: + raise AppError(code="INVALID_AREA_GEOMETRY", message="Area geometry cannot be repaired", status_code=400) + mask_input = [mapping(transformed_area)] + + try: + clipped_data, clipped_transform = rasterio.mask.mask(source, mask_input, crop=True, nodata=source.nodata, filled=True) + except Exception as exc: + raise AppError(code="RASTER_OPERATION_ERROR", message="Raster clipping failed", status_code=500) from exc + + clipped_has_data = True + try: + import numpy + + clipped_array = numpy.asarray(clipped_data) + clipped_has_data = bool(clipped_array.size and numpy.isfinite(clipped_array).any()) + except Exception: + clipped_has_data = clipped_data.size > 0 + + if not clipped_has_data: + raise AppError(code="RASTER_OPERATION_EMPTY_RESULT", message="Raster clip produced no output data", status_code=422) + + source_count = getattr(source, "count", None) + if not source_count: + if hasattr(clipped_data, "shape") and len(clipped_data.shape) >= 1: + source_count = int(clipped_data.shape[0]) + else: + source_count = 1 + source_count = int(source_count) + + profile = source.profile.copy() + profile.update( + { + "count": source_count, + "height": int(clipped_data.shape[1]), + "width": int(clipped_data.shape[2]), + "transform": clipped_transform, + }, + ) + + output_id = uuid.uuid4() + output_filename = f"{output_name or 'raster_clipped'}.tif" + output_path = Path(StorageService.derived_raster_root(str(dataset.project_id), str(output_id)) / output_filename) + output_path.parent.mkdir(parents=True, exist_ok=True) + + with rasterio.open(output_path, "w", **profile) as destination: + destination.write(clipped_data) + + if not output_path.exists(): + raise AppError(code="RASTER_OPERATION_ERROR", message="Failed to write clip output", status_code=500) + + derived_metadata = extract_raster_metadata(str(output_path)) + derived_metadata["operation"] = "raster.clip" + derived_metadata["source_dataset_id"] = str(dataset.id) + derived_metadata["operation_parameters"] = { + "area_id": str(area_id), + "source_crs": source_crs, + "area_crs": area.original_crs, + } + derived_metadata["output_dataset_id"] = str(output_id) + derived_id = RasterOperationsService._persist_derived_dataset( + db=db, + source_dataset=dataset, + source_dataset_id=dataset.id, + operation="raster.clip", + output_path=str(output_path), + output_name=output_filename, + metadata=derived_metadata, + ) + return derived_id + + @staticmethod + def tile( + db, + dataset_id: uuid.UUID, + tile_size: int = 512, + overlap: int = 64, + output_name: str | None = None, + ) -> dict[str, Any]: + RasterOperationsService._validate_tile_request(tile_size=tile_size, overlap=overlap) + dataset = RasterOperationsService._load_dataset(db, dataset_id) + rasterio, _ = RasterOperationsService._raster_dependencies() + + tile_set_id = str(uuid.uuid4()) + tile_root = StorageService.raster_tiles_root(str(dataset.project_id), str(dataset.id), tile_set_id) + tile_root.mkdir(parents=True, exist_ok=True) + + manifest_tiles: list[dict[str, Any]] = [] + tile_paths: list[str] = [] + + with rasterio.open(dataset.storage_path) as source: + source_count = getattr(source, "count", 0) + if not source_count: + source_count = 1 + if source_count == 0: + raise AppError(code="INVALID_DATASET", message="Dataset has no raster bands", status_code=400) + + source_width = int(source.width) + source_height = int(source.height) + step = max(1, tile_size - overlap) + tile_index = 0 + for yoff in range(0, source_height, step): + for xoff in range(0, source_width, step): + tile_width = min(tile_size, source_width - xoff) + tile_height = min(tile_size, source_height - yoff) + if tile_width <= 0 or tile_height <= 0: + continue + + window = rasterio.windows.Window(xoff, yoff, tile_width, tile_height) + tile_data = source.read(window=window) + if tile_data.size == 0: + continue + + bounds = rasterio.windows.bounds(window, source.transform) + transform = rasterio.windows.transform(window, source.transform) + tile_path = tile_root / f"tile_{tile_index:04d}.tif" + profile = source.profile.copy() + profile.update(width=int(tile_width), height=int(tile_height), transform=transform) + profile.pop("transform", None) + profile["transform"] = transform + + with rasterio.open(tile_path, "w", **profile) as tile_dest: + tile_dest.write(tile_data) + + tile_paths.append(str(tile_path)) + manifest_tiles.append( + { + "path": str(tile_path), + "pixel_window": [int(xoff), int(yoff), int(tile_width), int(tile_height)], + "bounds": [float(bounds.left), float(bounds.bottom), float(bounds.right), float(bounds.top)], + "transform": [float(item) for item in transform.to_gdal()], + "index": tile_index, + }, + ) + tile_index += 1 + + if not manifest_tiles: + raise AppError(code="RASTER_OPERATION_EMPTY_RESULT", message="Raster tile generation produced no tiles", status_code=422) + + try: + source_metadata = extract_raster_metadata(dataset.storage_path) + except AppError: + source_metadata = {"bounds": [0.0, 0.0, 0.0, 0.0]} + bounds = source_metadata.get("bounds", [0.0, 0.0, 0.0, 0.0]) + manifest_payload = { + "tile_set_id": tile_set_id, + "source_dataset_id": str(dataset.id), + "source_raster_id": str(dataset.id), + "bounds": [float(value) for value in bounds], + "tile_size": int(tile_size), + "overlap": int(overlap), + "parameters": { + "tile_size": int(tile_size), + "overlap": int(overlap), + "output_name": output_name, + }, + "created_at": datetime.now(timezone.utc).isoformat(), + "tile_paths": tile_paths, + "count": len(manifest_tiles), + "tiles": manifest_tiles, + "ai_inference": False, + "tile_server": None, + } + manifest_path = tile_root / "manifest.json" + manifest_path.write_text(json.dumps(manifest_payload), encoding="utf-8") + + return { + "dataset_id": str(dataset.id), + "ready": True, + "operation": "raster.tile", + "tile_set_id": tile_set_id, + "tile_size": tile_size, + "overlap": overlap, + "manifest_path": str(manifest_path), + "count": len(manifest_tiles), + "manifest": manifest_payload, + } diff --git a/backend/app/services/raster_service.py b/backend/app/services/raster_service.py new file mode 100644 index 00000000..50f4135b --- /dev/null +++ b/backend/app/services/raster_service.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from pathlib import Path + +from app.core.errors import AppError + + +def _import_rasterio(): + import importlib + + rasterio = importlib.import_module("rasterio") + errors = importlib.import_module("rasterio.errors") + return rasterio, errors + + +def extract_raster_metadata(path: str) -> dict: + try: + rasterio, errors = _import_rasterio() + except Exception as exc: # pragma: no cover - exercised via API-level fallback tests + raise AppError( + code="RASTER_PROCESSING_UNAVAILABLE", + message="Raster processing unavailable. Install rasterio and GDAL-compatible drivers to enable raster metadata extraction.", + status_code=503, + ) from exc + + dataset_path = Path(path) + try: + with rasterio.open(dataset_path) as dataset: + nodata = dataset.nodata + if isinstance(nodata, (list, tuple)): + nodata_value = [None if value is None else float(value) for value in nodata] + else: + nodata_value = None if nodata is None else float(nodata) + + transform = dataset.transform.to_gdal() if hasattr(dataset, "transform") else None + return { + "driver": dataset.driver, + "width": int(dataset.width), + "height": int(dataset.height), + "band_count": int(dataset.count), + "crs": str(dataset.crs) if dataset.crs else None, + "bounds": list(dataset.bounds), + "resolution": list(dataset.res), + "dtype": list(dataset.dtypes), + "nodata": nodata_value, + "transform": list(transform) if transform is not None else None, + } + except Exception as exc: + if isinstance(exc, errors.RasterioIOError): + raise AppError(code="INVALID_RASTER", message="Uploaded raster file is invalid", status_code=400) from exc + raise AppError(code="RASTER_METADATA_ERROR", message="Unable to read raster metadata", status_code=400) from exc diff --git a/backend/app/services/segmentation_adapter.py b/backend/app/services/segmentation_adapter.py new file mode 100644 index 00000000..8fd47676 --- /dev/null +++ b/backend/app/services/segmentation_adapter.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + + +@dataclass(frozen=True) +class SegmentationAdapterResult: + class_name: str + confidence: float | None + geometry: dict[str, Any] + bbox_json: dict[str, Any] | None = None + mask_path: str | None = None + source_tile_path: str | None = None + tile_index: int | None = None + properties_json: dict[str, Any] | None = None + provenance_json: dict[str, Any] | None = None + area_m2: float | None = None + + +class SegmentationAdapter(Protocol): + def segment(self, *args: Any, **kwargs: Any) -> list[SegmentationAdapterResult]: + """Future segmentation adapters must local-import model dependencies inside execution paths.""" + + +class FixtureSegmentationAdapter: + def segment(self, raw_segmentations: Any) -> list[SegmentationAdapterResult]: + if not isinstance(raw_segmentations, list): + return [] + results: list[SegmentationAdapterResult] = [] + for raw in raw_segmentations: + if not isinstance(raw, dict): + continue + results.append( + SegmentationAdapterResult( + class_name=str(raw.get("class_name") or ""), + confidence=float(raw["confidence"]) if raw.get("confidence") is not None else None, + geometry=raw.get("geometry"), + bbox_json=raw.get("bbox_json"), + mask_path=raw.get("mask_path"), + source_tile_path=raw.get("source_tile_path"), + tile_index=raw.get("tile_index"), + properties_json=raw.get("properties_json"), + provenance_json=raw.get("provenance_json"), + area_m2=raw.get("area_m2"), + ) + ) + return results diff --git a/backend/app/services/segmentation_service.py b/backend/app/services/segmentation_service.py new file mode 100644 index 00000000..93b45f11 --- /dev/null +++ b/backend/app/services/segmentation_service.py @@ -0,0 +1,512 @@ +from __future__ import annotations + +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from geoalchemy2.shape import from_shape, to_shape +from shapely.geometry import MultiPolygon, Polygon, mapping, shape +from shapely.validation import make_valid + +from app.core.config import Settings, get_settings +from app.core.errors import AppError +from app.models import AnalysisRun, Dataset, Job, Project, Segmentation, VectorFeature +from app.schemas.segmentation import ( + SegmentationListResponse, + SegmentationRead, + SegmentationRunListResponse, + SegmentationRunRead, + SegmentationRunResponse, +) +from app.services.model_registry_service import ModelRegistryService +from app.services.qa_service import QaService +from app.services.quality_service import QualityService +from app.services.segmentation_adapter import FixtureSegmentationAdapter + + +class SegmentationService: + @staticmethod + def _now() -> datetime: + return datetime.now(UTC) + + @staticmethod + def run_segmentation( + db, + project_id: uuid.UUID, + dataset_id: uuid.UUID, + model_id: str, + confidence_threshold: float, + class_filter: list[str] | None = None, + tile_manifest_path: str | None = None, + parameters_json: dict[str, Any] | None = None, + settings: Settings | None = None, + ) -> SegmentationRunResponse: + parameters = dict(parameters_json or {}) + resolved_settings = settings or get_settings() + project = db.get(Project, project_id) + if not project: + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + dataset = db.get(Dataset, dataset_id) + if not dataset or dataset.project_id != project_id: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + if dataset.dataset_type != "raster": + raise AppError( + code="INVALID_DATASET_TYPE", + message="Segmentation requires a raster dataset", + details={"dataset_type": dataset.dataset_type}, + status_code=400, + ) + + model = ModelRegistryService.get_model_capability(model_id, task_type="segmentation") + if model is None: + raise AppError(code="SEGMENTATION_MODEL_NOT_FOUND", message="Segmentation model not found", status_code=404) + if model.model_id == "fixture-segmenter" and parameters.get("fixture_mode") is not True: + raise AppError( + code="FIXTURE_MODE_REQUIRED", + message="Fixture segmenter requires explicit fixture_mode=true", + status_code=400, + ) + + run_parameters = { + "model_id": model.model_id, + "confidence_threshold": confidence_threshold, + "class_filter": class_filter or [], + "tile_manifest_path": tile_manifest_path, + "parameters_json": parameters, + } + job = SegmentationService._create_job(db, project_id, dataset_id, run_parameters) + analysis_run = SegmentationService._create_analysis_run(db, project_id, dataset_id, job.id, model, run_parameters) + + if not model.configured: + message = model.limitation_message + SegmentationService._mark_failed( + db, + analysis_run, + job, + code="SEGMENTATION_MODEL_UNAVAILABLE", + message=message, + ) + return SegmentationRunResponse( + analysis_run_id=analysis_run.id, + job_id=job.id, + project_id=project_id, + dataset_id=dataset_id, + model_id=model.model_id, + status="failed", + segmentation_count=0, + error_code="SEGMENTATION_MODEL_UNAVAILABLE", + message=message, + ) + + if model.model_id == "fixture-segmenter": + segmentations = SegmentationService._persist_fixture_segmentations( + db=db, + project_id=project_id, + dataset_id=dataset_id, + analysis_run=analysis_run, + job=job, + model_name=model.model_id, + model_version=model.version, + raw_segmentations=parameters.get("fixture_segmentations"), + confidence_threshold=confidence_threshold, + class_filter=class_filter or [], + settings=resolved_settings, + ) + SegmentationService._mark_success(db, analysis_run, job, segmentation_count=len(segmentations)) + return SegmentationRunResponse( + analysis_run_id=analysis_run.id, + job_id=job.id, + project_id=project_id, + dataset_id=dataset_id, + model_id=model.model_id, + status="success", + segmentation_count=len(segmentations), + message="Fixture segmentations persisted.", + ) + + raise AppError(code="SEGMENTATION_MODEL_UNAVAILABLE", message="Segmentation model is unavailable", status_code=503) + + @staticmethod + def get_run(db, analysis_run_id: uuid.UUID) -> SegmentationRunRead: + run = db.get(AnalysisRun, analysis_run_id) + if not run or run.analysis_type != "segmentation": + raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404) + return SegmentationRunRead.model_validate(run) + + @staticmethod + def list_runs( + db, + *, + project_id: uuid.UUID | None = None, + dataset_id: uuid.UUID | None = None, + ) -> SegmentationRunListResponse: + query = db.query(AnalysisRun).filter(AnalysisRun.analysis_type == "segmentation") + if project_id is not None: + query = query.filter(AnalysisRun.project_id == project_id) + if dataset_id is not None: + query = query.filter(AnalysisRun.dataset_id == dataset_id) + rows = query.order_by(AnalysisRun.created_at.desc()).all() + return SegmentationRunListResponse(items=[SegmentationRunRead.model_validate(row) for row in rows], total=len(rows)) + + @staticmethod + def list_segmentations( + db, + analysis_run_id: uuid.UUID | None = None, + *, + dataset_id: uuid.UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + ) -> SegmentationListResponse: + if analysis_run_id is not None: + run = db.get(AnalysisRun, analysis_run_id) + if not run or run.analysis_type != "segmentation": + raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404) + rows = SegmentationService._query_segmentation_rows( + db, + analysis_run_id=analysis_run_id, + dataset_id=dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ) + items = [SegmentationRead.model_validate(row) for row in rows] + return SegmentationListResponse(items=items, total=len(items)) + + @staticmethod + def get_segmentation(db, segmentation_id: uuid.UUID) -> SegmentationRead: + segmentation = db.get(Segmentation, segmentation_id) + if not segmentation: + raise AppError(code="SEGMENTATION_NOT_FOUND", message="Segmentation not found", status_code=404) + return SegmentationRead.model_validate(segmentation) + + @staticmethod + def segmentations_to_geojson( + db, + *, + analysis_run_id: uuid.UUID | None = None, + dataset_id: uuid.UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + ) -> dict[str, Any]: + segmentations = SegmentationService._query_segmentation_rows( + db, + analysis_run_id=analysis_run_id, + dataset_id=dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ) + return { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": str(segmentation.id), + "properties": SegmentationService._segmentation_properties(segmentation), + "geometry": mapping(to_shape(segmentation.geometry)), + } + for segmentation in segmentations + ], + } + + @staticmethod + def compare_segmentations_with_reference( + db, + analysis_run_id: uuid.UUID, + reference_dataset_id: uuid.UUID, + iou_threshold: float = 0.5, + class_name: str | None = None, + min_confidence: float | None = None, + ) -> dict[str, Any]: + run = db.get(AnalysisRun, analysis_run_id) + if not run or run.analysis_type != "segmentation": + raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404) + reference_dataset = db.get(Dataset, reference_dataset_id) + if not reference_dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Reference dataset not found", status_code=404) + if reference_dataset.project_id != run.project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Reference dataset does not belong to segmentation project", status_code=400) + if reference_dataset.dataset_type not in {"vector", "geojson"}: + raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400) + + segmentations = SegmentationService._query_segmentation_rows( + db, + analysis_run_id=analysis_run_id, + dataset_id=run.dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ) + if not segmentations: + raise AppError( + code="SEGMENTATIONS_NOT_FOUND", + message="Segmentation run has no persisted geometries for QA", + status_code=422, + ) + references = db.query(VectorFeature).filter(VectorFeature.dataset_id == reference_dataset_id).all() + if not references: + raise AppError( + code="REFERENCE_FEATURES_NOT_FOUND", + message="Reference dataset has no persisted vector features for QA", + status_code=422, + ) + + candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in segmentations] + reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references] + matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics( + candidate_geometries, + reference_geometries, + iou_threshold, + ) + mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values) + precision = matches / (matches + false_positives) if matches + false_positives > 0 else None + recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None + f1_score = None + if precision is not None and recall is not None: + f1_score = (2 * precision * recall) / (precision + recall) if precision + recall > 0 else 0.0 + status = "unsupported" if unsupported else "ok" + quality_check = QualityService.persist_quality_check( + db=db, + project_id=run.project_id, + analysis_run_id=analysis_run_id, + candidate_dataset_id=run.dataset_id, + reference_dataset_id=reference_dataset_id, + check_type="segmentations_vs_reference", + status=status, + score=f1_score, + parameters={ + "analysis_run_id": str(analysis_run_id), + "reference_dataset_id": str(reference_dataset_id), + "iou_threshold": iou_threshold, + "class_name": class_name, + "min_confidence": min_confidence, + }, + findings={ + "matches": matches, + "false_positives": false_positives, + "false_negatives": false_negatives, + "warnings": warnings, + "unsupported_geometry": unsupported, + }, + metrics={ + "precision": precision, + "recall": recall, + "f1": f1_score, + "mean_iou": mean_iou, + "false_positive_count": false_positives, + "false_negative_count": false_negatives, + }, + ) + return { + "status": status, + "quality_check_id": str(quality_check.id), + "analysis_run_id": str(analysis_run_id), + "reference_dataset_id": str(reference_dataset_id), + "candidate_feature_count": len(candidate_geometries), + "reference_feature_count": len(reference_geometries), + "matches": matches, + "false_positives": false_positives, + "false_negatives": false_negatives, + "precision": precision, + "recall": recall, + "f1_score": f1_score, + "mean_iou": mean_iou, + "iou_threshold": iou_threshold, + "warnings": warnings, + } + + @staticmethod + def mask_artifact_path(storage_root: str, project_id: uuid.UUID, analysis_run_id: uuid.UUID, tile_index: int | None, segmentation_id: uuid.UUID) -> str: + tile_folder = f"tile_{tile_index if tile_index is not None else 0}" + return (Path(storage_root) / "masks" / str(project_id) / str(analysis_run_id) / tile_folder / f"mask_{segmentation_id}.png").as_posix() + + @staticmethod + def _create_job(db, project_id: uuid.UUID, dataset_id: uuid.UUID, parameters: dict[str, Any]) -> Job: + job = Job( + id=uuid.uuid4(), + job_type="segmentation.run", + status="running", + project_id=project_id, + dataset_id=dataset_id, + input_dataset_id=dataset_id, + parameters_json=parameters, + started_at=SegmentationService._now(), + ) + db.add(job) + db.commit() + db.refresh(job) + return job + + @staticmethod + def _create_analysis_run(db, project_id, dataset_id, job_id, model, parameters: dict[str, Any]) -> AnalysisRun: + analysis_run = AnalysisRun( + id=uuid.uuid4(), + project_id=project_id, + dataset_id=dataset_id, + job_id=job_id, + analysis_type="segmentation", + status="running", + model_name=model.model_id, + model_version=model.version, + parameters_json=parameters, + started_at=SegmentationService._now(), + ) + db.add(analysis_run) + db.commit() + db.refresh(analysis_run) + return analysis_run + + @staticmethod + def _mark_failed(db, analysis_run: AnalysisRun, job: Job, code: str, message: str) -> None: + result = {"error_code": code, "message": message, "segmentation_count": 0} + analysis_run.status = "failed" + analysis_run.finished_at = SegmentationService._now() + analysis_run.error_message = message + analysis_run.result_json = result + job.status = "failed" + job.finished_at = analysis_run.finished_at + job.error_message = message + job.result_json = result + db.add(analysis_run) + db.add(job) + db.commit() + db.refresh(analysis_run) + db.refresh(job) + + @staticmethod + def _mark_success(db, analysis_run: AnalysisRun, job: Job, segmentation_count: int) -> None: + result = {"segmentation_count": segmentation_count} + analysis_run.status = "success" + analysis_run.finished_at = SegmentationService._now() + analysis_run.result_json = result + job.status = "success" + job.finished_at = analysis_run.finished_at + job.result_json = result + db.add(analysis_run) + db.add(job) + db.commit() + db.refresh(analysis_run) + db.refresh(job) + + @staticmethod + def _persist_fixture_segmentations( + db, + project_id: uuid.UUID, + dataset_id: uuid.UUID, + analysis_run: AnalysisRun, + job: Job, + model_name: str, + model_version: str | None, + raw_segmentations: Any, + confidence_threshold: float, + class_filter: list[str], + settings: Settings, + ) -> list[Segmentation]: + if not isinstance(raw_segmentations, list): + raise AppError(code="INVALID_FIXTURE_SEGMENTATIONS", message="fixture_segmentations must be a list", status_code=400) + adapter = FixtureSegmentationAdapter() + adapter_results = adapter.segment(raw_segmentations) + if len(adapter_results) != len(raw_segmentations): + raise AppError(code="INVALID_FIXTURE_SEGMENTATION", message="Each fixture segmentation must be an object", status_code=400) + persisted: list[Segmentation] = [] + allowed_classes = set(class_filter) + for raw in adapter_results: + class_name = raw.class_name + confidence = raw.confidence + if allowed_classes and class_name not in allowed_classes: + continue + if confidence is not None and confidence < confidence_threshold: + continue + if not isinstance(raw.geometry, dict): + raise AppError(code="INVALID_FIXTURE_SEGMENTATION", message="Fixture segmentation geometry is required", status_code=400) + geometry = SegmentationService._validated_multipolygon(raw.geometry) + segmentation_id = uuid.uuid4() + mask_path = raw.mask_path or SegmentationService.mask_artifact_path( + settings.storage_root, + project_id, + analysis_run.id, + raw.tile_index, + segmentation_id, + ) + segmentation = Segmentation( + id=segmentation_id, + project_id=project_id, + dataset_id=dataset_id, + analysis_run_id=analysis_run.id, + job_id=job.id, + model_name=model_name, + model_version=model_version, + class_name=class_name, + confidence=confidence, + geometry=from_shape(geometry, srid=4326), + bbox_json=raw.bbox_json, + area_m2=raw.area_m2, + mask_path=mask_path, + source_tile_path=raw.source_tile_path, + tile_index=raw.tile_index, + properties_json=raw.properties_json, + provenance_json={**dict(raw.provenance_json or {}), "fixture_mode": True}, + ) + db.add(segmentation) + persisted.append(segmentation) + db.commit() + for segmentation in persisted: + db.refresh(segmentation) + return persisted + + @staticmethod + def _validated_multipolygon(geometry_payload: dict[str, Any]) -> MultiPolygon: + try: + geometry = shape(geometry_payload) + except Exception as exc: + raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must be valid GeoJSON", status_code=400) from exc + if geometry.is_empty: + raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must not be empty", status_code=400) + if not geometry.is_valid: + geometry = make_valid(geometry) + if geometry.is_empty or not geometry.is_valid: + raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must be valid", status_code=400) + if isinstance(geometry, Polygon): + geometry = MultiPolygon([geometry]) + if not isinstance(geometry, MultiPolygon): + raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must be Polygon or MultiPolygon", status_code=400) + if geometry.area <= 0: + raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must have positive area", status_code=400) + return geometry + + @staticmethod + def _query_segmentation_rows( + db, + *, + analysis_run_id: uuid.UUID | None = None, + dataset_id: uuid.UUID | None = None, + class_name: str | None = None, + min_confidence: float | None = None, + ) -> list[Segmentation]: + query = db.query(Segmentation) + if analysis_run_id is not None: + query = query.filter(Segmentation.analysis_run_id == analysis_run_id) + if dataset_id is not None: + query = query.filter(Segmentation.dataset_id == dataset_id) + if class_name: + query = query.filter(Segmentation.class_name == class_name) + if min_confidence is not None: + query = query.filter(Segmentation.confidence >= min_confidence) + return query.order_by(Segmentation.created_at.desc()).all() + + @staticmethod + def _segmentation_properties(segmentation: Segmentation) -> dict[str, Any]: + return { + "segmentation_id": str(segmentation.id), + "class_name": segmentation.class_name, + "confidence": segmentation.confidence, + "area_m2": segmentation.area_m2, + "model_name": segmentation.model_name, + "model_version": segmentation.model_version, + "analysis_run_id": str(segmentation.analysis_run_id) if segmentation.analysis_run_id else None, + "dataset_id": str(segmentation.dataset_id) if segmentation.dataset_id else None, + "job_id": str(segmentation.job_id) if segmentation.job_id else None, + "source_tile_path": segmentation.source_tile_path, + "tile_index": segmentation.tile_index, + "mask_path": segmentation.mask_path, + "bbox_json": segmentation.bbox_json, + "provenance_json": segmentation.provenance_json, + } diff --git a/backend/app/services/storage_service.py b/backend/app/services/storage_service.py new file mode 100644 index 00000000..fc3694f3 --- /dev/null +++ b/backend/app/services/storage_service.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import hashlib +import shutil +from pathlib import Path +from typing import Any + +from app.core.config import get_settings + + +class StorageService: + @staticmethod + def _base_dir() -> Path: + return Path(get_settings().storage_root).resolve() + + @staticmethod + def normalize_dataset_type(dataset_type: str) -> str: + normalized = dataset_type.strip().lower() + if normalized == "geojson": + return "vector" + return normalized + + @staticmethod + def _safe_filename(value: str) -> str: + value = value.strip().replace("\\", "/").split("/")[-1] + fallback = "upload" + if not value: + return fallback + allowed = [] + for char in value: + if char.isalnum() or char in "-_ .": + allowed.append(char) + else: + allowed.append("_") + cleaned = "".join(allowed) + cleaned = cleaned.strip(" .") + return cleaned or fallback + + @staticmethod + def dataset_root(project_id: str, dataset_id: str, dataset_type: str) -> Path: + return StorageService._base_dir() / "uploads" / project_id / dataset_type / dataset_id + + @staticmethod + def derived_raster_root(project_id: str, dataset_id: str) -> Path: + return StorageService._base_dir() / "rasters" / "derived" / project_id / dataset_id + + @staticmethod + def preview_root(project_id: str, dataset_id: str) -> Path: + return StorageService._base_dir() / "previews" / project_id / dataset_id + + @staticmethod + def raster_tiles_root(project_id: str, source_dataset_id: str, tile_set_id: str) -> Path: + return StorageService._base_dir() / "tiles" / project_id / source_dataset_id / tile_set_id + + @staticmethod + def dataset_file_path( + project_id: str, + dataset_id: str, + dataset_type: str, + original_filename: str, + ) -> str: + safe_original = StorageService._safe_filename(original_filename) + stored_filename = f"{dataset_id}_{safe_original}" + return str(StorageService.dataset_root(project_id, dataset_id, dataset_type) / stored_filename) + + @staticmethod + def calculate_checksum_sha256(content: bytes) -> str: + digest = hashlib.sha256() + digest.update(content) + return digest.hexdigest() + + @staticmethod + def persist_dataset_file( + project_id: str, + dataset_id: str, + dataset_type: str, + original_filename: str, + content: bytes, + content_type: str | None, + ) -> dict[str, Any]: + normalized_type = StorageService.normalize_dataset_type(dataset_type) + file_path = Path(StorageService.dataset_file_path(project_id, dataset_id, normalized_type, original_filename)) + file_path.parent.mkdir(parents=True, exist_ok=True) + + with file_path.open("wb") as stream: + stream.write(content) + + metadata: dict[str, Any] = { + "original_filename": StorageService._safe_filename(original_filename), + "stored_filename": file_path.name, + "content_type": content_type or "application/octet-stream", + "size_bytes": len(content), + "checksum_sha256": StorageService.calculate_checksum_sha256(content), + "storage_path": str(file_path), + } + return metadata + + @staticmethod + def persist_file( + storage_path: str, + content: bytes, + original_filename: str, + content_type: str | None, + ) -> dict[str, Any]: + target = Path(storage_path) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("wb") as stream: + stream.write(content) + + metadata: dict[str, Any] = { + "original_filename": StorageService._safe_filename(original_filename), + "stored_filename": target.name, + "content_type": content_type or "application/octet-stream", + "size_bytes": len(content), + "checksum_sha256": StorageService.calculate_checksum_sha256(content), + "storage_path": str(target), + } + return metadata + + @staticmethod + def remove_dataset_file(path: str) -> None: + target = Path(path) + if target.exists(): + target.unlink(missing_ok=True) + + dataset_parent = target.parent + if dataset_parent.exists() and dataset_parent.is_dir(): + has_files = any(dataset_parent.iterdir()) + if not has_files: + shutil.rmtree(dataset_parent, ignore_errors=True) + + @staticmethod + def dataset_export_path(project_id: str, dataset_id: str, filename: str) -> str: + output_dir = StorageService._base_dir() / "exports" / project_id / "datasets" + output_dir.mkdir(parents=True, exist_ok=True) + return str(output_dir / f"{dataset_id}_{StorageService._safe_filename(filename)}") diff --git a/backend/app/services/vector_feature_service.py b/backend/app/services/vector_feature_service.py new file mode 100644 index 00000000..e9f2e94b --- /dev/null +++ b/backend/app/services/vector_feature_service.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from typing import Any +from uuid import UUID + +from geoalchemy2.shape import from_shape +from shapely.geometry import shape +from shapely.validation import make_valid + +from app.core.errors import AppError +from app.models import VectorFeature + + +class VectorFeatureService: + @staticmethod + def persist_geojson_features( + db, + dataset_id: UUID, + payload: dict[str, Any], + feature_class: str | None = None, + *, + commit: bool = True, + ) -> list[VectorFeature]: + features = payload.get("features") + if payload.get("type") != "FeatureCollection" or not isinstance(features, list): + raise AppError(code="INVALID_GEOJSON", message="GeoJSON payload must be a FeatureCollection", status_code=400) + + persisted: list[VectorFeature] = [] + for index, feature in enumerate(features): + if not isinstance(feature, dict): + raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400) + geometry_payload = feature.get("geometry") + if geometry_payload is None: + continue + try: + geometry = shape(geometry_payload) + except Exception as exc: + raise AppError(code="INVALID_GEOJSON", message=f"Invalid feature geometry at index {index}", status_code=400) from exc + if geometry.is_empty: + continue + if not geometry.is_valid: + geometry = make_valid(geometry) + if geometry.is_empty or not geometry.is_valid: + raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400) + + properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {} + source_feature_id = feature.get("id") + if source_feature_id is None: + source_feature_id = properties.get("id") or properties.get("source_feature_id") + + row = VectorFeature( + dataset_id=dataset_id, + feature_class=feature_class, + source_feature_id=str(source_feature_id) if source_feature_id is not None else None, + properties_json=properties, + geometry=from_shape(geometry, srid=4326), + ) + db.add(row) + persisted.append(row) + + if commit: + db.commit() + for row in persisted: + db.refresh(row) + return persisted diff --git a/backend/app/services/vector_operations_service.py b/backend/app/services/vector_operations_service.py new file mode 100644 index 00000000..7bb63341 --- /dev/null +++ b/backend/app/services/vector_operations_service.py @@ -0,0 +1,329 @@ +from __future__ import annotations + +import json +import uuid +from pathlib import Path +from typing import Any + +from geoalchemy2.shape import to_shape +from shapely.geometry import GeometryCollection, MultiPolygon, shape +from shapely.geometry.base import BaseGeometry +from shapely.geometry import mapping +from shapely.ops import unary_union +from shapely.validation import make_valid +from sqlalchemy.orm import Session + +from app.core.errors import AppError +from app.models import Area, Dataset +from app.schemas.operations import VectorOperationResult +from app.services.geojson_service import parse_geojson_payload +from app.services.storage_service import StorageService + + +class VectorOperationsService: + @staticmethod + def _require_vector_dataset(dataset: Dataset) -> None: + if dataset.dataset_type not in {"vector", "geojson"}: + raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400) + + @staticmethod + def _load_dataset_payload(dataset: Dataset) -> tuple[dict[str, Any], list[dict[str, Any]]]: + if not dataset.storage_path: + raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) + path = Path(dataset.storage_path) + if not path.exists(): + raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=400) from exc + + if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection": + raise AppError(code="INVALID_GEOJSON", message="Dataset payload is not a FeatureCollection", status_code=400) + + features = payload.get("features") + if not isinstance(features, list): + raise AppError(code="INVALID_GEOJSON", message="Dataset payload is missing features", status_code=400) + return payload, [feature for feature in features if isinstance(feature, dict)] + + @staticmethod + def _extract_geometries(features: list[dict[str, Any]]) -> list[tuple[dict[str, Any], BaseGeometry]]: + geometries: list[tuple[dict[str, Any], BaseGeometry]] = [] + for feature in features: + if not isinstance(feature, dict): + continue + geometry = feature.get("geometry") + if not geometry: + continue + try: + shapely_geom = shape(geometry) + except Exception as exc: + raise AppError(code="INVALID_GEOMETRY", message="Feature geometry invalid", status_code=400) from exc + if not shapely_geom.is_valid: + shapely_geom = make_valid(shapely_geom) + if not shapely_geom.is_valid: + raise AppError(code="INVALID_GEOMETRY", message="Feature geometry cannot be repaired", status_code=400) + + geometries.append((feature, shapely_geom)) + + if not geometries: + raise AppError(code="EMPTY_VECTOR_DATASET", message="Dataset has no parseable geometries", status_code=422) + return geometries + + @staticmethod + def inspect(db: Session, dataset_id: uuid.UUID) -> VectorOperationResult: + dataset = db.get(Dataset, dataset_id) + if not dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + VectorOperationsService._require_vector_dataset(dataset) + + payload, features = VectorOperationsService._load_dataset_payload(dataset) + geometries = VectorOperationsService._extract_geometries(features) + + geometry_type_summary: dict[str, int] = {} + for _, geometry in geometries: + geometry_type_summary[geometry.geom_type] = geometry_type_summary.get(geometry.geom_type, 0) + 1 + + unioned = unary_union([geometry for _, geometry in geometries]) + bounds = unioned.bounds + return VectorOperationResult( + source_dataset_id=str(dataset_id), + feature_count=len(geometries), + geometry_type_summary=geometry_type_summary, + bounds_json={"min_x": float(bounds[0]), "min_y": float(bounds[1]), "max_x": float(bounds[2]), "max_y": float(bounds[3])}, + crs=payload.get("crs") if isinstance(payload.get("crs"), str) else dataset.crs, + ) + + @staticmethod + def bbox(db: Session, dataset_id: uuid.UUID) -> dict[str, Any]: + summary = VectorOperationsService.inspect(db, dataset_id) + return { + "dataset_id": str(dataset_id), + "bounds_json": summary.bounds_json, + "feature_count": summary.feature_count, + "crs": summary.crs, + } + + @staticmethod + def stats(db: Session, dataset_id: uuid.UUID) -> dict[str, Any]: + summary = VectorOperationsService.inspect(db, dataset_id) + return { + "dataset_id": str(dataset_id), + "feature_count": summary.feature_count, + "geometry_type_summary": summary.geometry_type_summary, + "bounds_json": summary.bounds_json, + "crs": summary.crs, + } + + @staticmethod + def clip_by_area(db: Session, dataset_id: uuid.UUID, area_id: uuid.UUID, output_name: str | None) -> uuid.UUID: + source_dataset = db.get(Dataset, dataset_id) + if not source_dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + VectorOperationsService._require_vector_dataset(source_dataset) + + area = db.get(Area, area_id) + if not area: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + if area.project_id != source_dataset.project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to dataset project", status_code=400) + + payload, features = VectorOperationsService._load_dataset_payload(source_dataset) + geometries = VectorOperationsService._extract_geometries(features) + area_geom = to_shape(area.geometry) + if area_geom.is_empty: + raise AppError(code="INVALID_AREA_GEOMETRY", message="Area geometry is empty", status_code=400) + + if isinstance(area_geom, GeometryCollection): + area_geom = unary_union(area_geom.geoms) + if area_geom.geom_type == "MultiPolygon": + area_geom = MultiPolygon(area_geom.geoms) + + if not area_geom.is_valid: + area_geom = make_valid(area_geom) + if not area_geom.is_valid: + raise AppError(code="INVALID_AREA_GEOMETRY", message="Area geometry cannot be repaired", status_code=400) + + output_features: list[dict[str, Any]] = [] + for feature, source_geom in geometries: + clipped = source_geom.intersection(area_geom) + if clipped.is_empty: + continue + if not clipped.is_valid: + clipped = make_valid(clipped) + if not clipped.is_valid: + raise AppError(code="INVALID_GEOMETRY", message="Clipped geometry became invalid", status_code=400) + output_features.append({ + "type": "Feature", + "geometry": mapping(clipped), + "properties": feature.get("properties", {}) or {}, + }) + + if not output_features: + raise AppError(code="VECTOR_OPERATION_EMPTY_RESULT", message="Clip operation produced no output features", status_code=422) + + return VectorOperationsService._persist_derived_dataset( + db=db, + source_dataset=source_dataset, + source_id=dataset_id, + operation="clip", + feature_collection={"type": "FeatureCollection", "features": output_features}, + output_name=output_name, + default_name="vector_clipped", + ) + + @staticmethod + def buffer(db: Session, dataset_id: uuid.UUID, distance_m: float, dissolve: bool, output_name: str | None) -> uuid.UUID: + source_dataset = db.get(Dataset, dataset_id) + if not source_dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + VectorOperationsService._require_vector_dataset(source_dataset) + if distance_m <= 0: + raise AppError(code="INVALID_PARAMETERS", message="distance_m must be greater than 0", status_code=400) + + _, features = VectorOperationsService._load_dataset_payload(source_dataset) + geometries = VectorOperationsService._extract_geometries(features) + buffered_features = [(feature, geometry.buffer(distance_m)) for feature, geometry in geometries] + + output_features: list[dict[str, Any]] = [] + for feature, geometry in buffered_features: + if geometry.is_empty: + continue + if not geometry.is_valid: + geometry = make_valid(geometry) + if not geometry.is_valid: + raise AppError(code="INVALID_GEOMETRY", message="Buffer geometry became invalid", status_code=400) + output_features.append({ + "type": "Feature", + "geometry": mapping(geometry), + "properties": feature.get("properties", {}) or {}, + }) + + if dissolve: + dissolved = unary_union([shape(feature["geometry"]) for feature in output_features]) + output_features = [{ + "type": "Feature", + "geometry": mapping(dissolved), + "properties": {"operation": "vector_buffer", "distance_m": distance_m, "dissolve": True}, + }] + + if not output_features: + raise AppError(code="VECTOR_OPERATION_EMPTY_RESULT", message="Buffer operation produced no output features", status_code=422) + + return VectorOperationsService._persist_derived_dataset( + db=db, + source_dataset=source_dataset, + source_id=dataset_id, + operation="buffer", + feature_collection={"type": "FeatureCollection", "features": output_features}, + output_name=output_name, + default_name="vector_buffered", + ) + + @staticmethod + def intersect( + db: Session, + source_dataset_id: uuid.UUID, + target_dataset_id: uuid.UUID, + output_name: str | None, + ) -> uuid.UUID: + if source_dataset_id == target_dataset_id: + raise AppError(code="INVALID_PARAMETERS", message="other_dataset_id must be different from source dataset", status_code=400) + + source_dataset = db.get(Dataset, source_dataset_id) + if not source_dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404) + VectorOperationsService._require_vector_dataset(source_dataset) + + target_dataset = db.get(Dataset, target_dataset_id) + if not target_dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Target dataset not found", status_code=404) + VectorOperationsService._require_vector_dataset(target_dataset) + if target_dataset.project_id != source_dataset.project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Datasets must belong to same project", status_code=400) + + source_payload, source_features = VectorOperationsService._load_dataset_payload(source_dataset) + target_payload, _ = VectorOperationsService._load_dataset_payload(target_dataset) + source_geometries = VectorOperationsService._extract_geometries(source_features) + target_geometries = VectorOperationsService._extract_geometries(target_payload.get("features", [])) + target_union = unary_union([geometry for _, geometry in target_geometries]) + + output_features: list[dict[str, Any]] = [] + for source_feature, source_geometry in source_geometries: + intersection = source_geometry.intersection(target_union) + if intersection.is_empty: + continue + if not intersection.is_valid: + intersection = make_valid(intersection) + if not intersection.is_valid: + raise AppError(code="INVALID_GEOMETRY", message="Intersection geometry became invalid", status_code=400) + output_features.append({ + "type": "Feature", + "geometry": mapping(intersection), + "properties": source_feature.get("properties", {}) or {}, + }) + + if not output_features: + raise AppError(code="VECTOR_OPERATION_EMPTY_RESULT", message="Intersection operation produced no output features", status_code=422) + + return VectorOperationsService._persist_derived_dataset( + db=db, + source_dataset=source_dataset, + source_id=source_dataset_id, + operation="intersect", + feature_collection={"type": "FeatureCollection", "features": output_features}, + output_name=output_name, + default_name="vector_intersect", + ) + + @staticmethod + def _persist_derived_dataset( + db: Session, + source_dataset: Dataset, + source_id: uuid.UUID, + operation: str, + feature_collection: dict[str, Any], + output_name: str | None, + default_name: str, + ) -> uuid.UUID: + derived_id = uuid.uuid4() + output_name_value = f"{(output_name or default_name)}.geojson" + if not output_name_value.strip(): + output_name_value = f"{default_name}.geojson" + + stored = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + storage_info = StorageService.persist_dataset_file( + project_id=str(source_dataset.project_id), + dataset_id=str(derived_id), + dataset_type="vector", + original_filename=output_name_value, + content=stored, + content_type="application/geo+json", + ) + + metadata = parse_geojson_payload(json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":"))) + derived_dataset = Dataset( + id=derived_id, + project_id=source_dataset.project_id, + area_id=source_dataset.area_id, + name=output_name_value, + dataset_type="vector", + source=f"operation:{operation}", + storage_path=storage_info["storage_path"], + original_filename=storage_info["original_filename"], + stored_filename=storage_info["stored_filename"], + content_type=storage_info["content_type"], + size_bytes=storage_info["size_bytes"], + checksum_sha256=storage_info["checksum_sha256"], + derived_from_dataset_id=source_id, + crs=metadata.get("crs"), + bounds_json=metadata.get("bounds_json"), + resolution_json=metadata.get("resolution_json"), + bands_json=metadata.get("bands_json"), + metadata_json=metadata, + status="ready", + ) + db.add(derived_dataset) + db.commit() + db.refresh(derived_dataset) + return derived_id diff --git a/backend/app/services/yolo_adapter.py b/backend/app/services/yolo_adapter.py new file mode 100644 index 00000000..4f9ff0b9 --- /dev/null +++ b/backend/app/services/yolo_adapter.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + +from app.core.config import Settings +from app.core.errors import AppError + + +class YoloDetectionAdapter: + def __init__(self, settings: Settings) -> None: + self.settings = settings + + @staticmethod + def dependencies_available() -> bool: + return importlib.util.find_spec("ultralytics") is not None and importlib.util.find_spec("torch") is not None + + def load_model(self, model_path: Path): + if not model_path.exists() or not model_path.is_file(): + raise AppError( + code="DETECTION_MODEL_UNAVAILABLE", + message="Configured YOLO model file does not exist", + details={"model_path": str(model_path)}, + status_code=503, + ) + if not self.dependencies_available(): + raise AppError( + code="DETECTION_DEPENDENCY_UNAVAILABLE", + message="YOLO dependencies are not installed. Install backend optional extras with geointel-backend[ai].", + status_code=503, + ) + + try: + from ultralytics import YOLO + except ImportError as exc: + raise AppError( + code="DETECTION_DEPENDENCY_UNAVAILABLE", + message="YOLO dependencies are not importable. Install backend optional extras with geointel-backend[ai].", + status_code=503, + ) from exc + + try: + return YOLO(str(model_path)) + except Exception as exc: + raise AppError( + code="DETECTION_MODEL_LOAD_FAILED", + message="Configured YOLO model could not be loaded", + details={"model_path": str(model_path)}, + status_code=503, + ) from exc + + def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict[str, Any]]: + if not tile_path.exists() or not tile_path.is_file(): + raise AppError( + code="DETECTION_TILE_NOT_FOUND", + message="Tile referenced by manifest does not exist", + details={"tile_path": str(tile_path)}, + status_code=422, + ) + results = model.predict( + source=str(tile_path), + conf=float(confidence_threshold), + imgsz=int(self.settings.yolo_image_size), + device=self.settings.yolo_device, + verbose=False, + ) + + detections: list[dict[str, Any]] = [] + for result in results: + names = getattr(result, "names", {}) or {} + boxes = getattr(result, "boxes", None) + if boxes is None: + continue + xyxy_values = _to_list(getattr(boxes, "xyxy", [])) + confidence_values = _to_list(getattr(boxes, "conf", [])) + class_values = _to_list(getattr(boxes, "cls", [])) + for index, bbox in enumerate(xyxy_values): + class_id = int(class_values[index]) if index < len(class_values) else -1 + detections.append( + { + "class_name": str(names.get(class_id, class_id)), + "confidence": float(confidence_values[index]) if index < len(confidence_values) else 0.0, + "bbox": [float(value) for value in bbox], + "properties": {"class_id": class_id}, + } + ) + return detections + + +def _to_list(value: Any) -> list[Any]: + if hasattr(value, "detach"): + value = value.detach() + if hasattr(value, "cpu"): + value = value.cpu() + if hasattr(value, "numpy"): + value = value.numpy() + if hasattr(value, "tolist"): + return value.tolist() + return list(value) diff --git a/backend/app/services/yolo_preflight_service.py b/backend/app/services/yolo_preflight_service.py new file mode 100644 index 00000000..3b721fe4 --- /dev/null +++ b/backend/app/services/yolo_preflight_service.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Type + +from app.core.config import Settings, get_settings +from app.core.errors import AppError +from app.services.detection_service import DetectionService +from app.services.yolo_adapter import YoloDetectionAdapter + + +class YoloPreflightService: + @staticmethod + def run( + *, + settings: Settings | None = None, + tile_manifest_path: str | None = None, + yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter, + assume_dependencies: bool = False, + ) -> dict[str, Any]: + resolved_settings = settings or get_settings() + result: dict[str, Any] = { + "model_id": resolved_settings.yolo_model_id, + "model_path": resolved_settings.yolo_model_path, + "tile_manifest_path": tile_manifest_path, + "status": "not_configured", + "message": "", + "checks": { + "enabled": resolved_settings.yolo_enabled, + "dependencies_available": None, + "model_path_set": None, + "model_file_exists": None, + "manifest_path_set": None, + "manifest_valid": None, + "tile_paths_exist": None, + "tile_limit_ok": None, + }, + "tile_count": 0, + "max_tiles": resolved_settings.yolo_max_tiles, + "will_download_models": False, + "will_run_inference": False, + } + + if not resolved_settings.yolo_enabled: + result["message"] = "YOLO is disabled. Set YOLO_ENABLED=true for configured local inference." + return result + + dependencies_available = True if assume_dependencies else yolo_adapter_class.dependencies_available() + result["checks"]["dependencies_available"] = dependencies_available + if not dependencies_available: + result["status"] = "dependency_unavailable" + result["message"] = "YOLO dependencies are not installed. Install backend optional extras with geointel-backend[ai]." + return result + + result["checks"]["model_path_set"] = bool(resolved_settings.yolo_model_path) + if not resolved_settings.yolo_model_path: + result["message"] = "YOLO_MODEL_PATH is not set. GeoIntel will not download model weights automatically." + return result + + model_path = Path(resolved_settings.yolo_model_path).expanduser() + model_exists = model_path.exists() and model_path.is_file() + result["checks"]["model_file_exists"] = model_exists + if not model_exists: + result["message"] = "YOLO_MODEL_PATH does not point to an existing local model file." + return result + + result["checks"]["manifest_path_set"] = bool(tile_manifest_path) + if not tile_manifest_path: + result["status"] = "manifest_unavailable" + result["message"] = "Configured YOLO inference requires an existing raster tile manifest path." + return result + + try: + manifest = DetectionService._load_tile_manifest(tile_manifest_path, resolved_settings.yolo_max_tiles) + tile_paths = [DetectionService._resolve_tile_path(tile, Path(tile_manifest_path).expanduser()) for tile in manifest["tiles"]] + except AppError as exc: + result["status"] = "manifest_invalid" + result["message"] = exc.message + result["error_code"] = exc.code + result["checks"]["manifest_valid"] = False + if exc.code != "DETECTION_TILE_LIMIT_EXCEEDED": + result["checks"]["tile_limit_ok"] = None + else: + result["checks"]["tile_limit_ok"] = False + return result + + result["checks"]["manifest_valid"] = True + result["checks"]["tile_paths_exist"] = all(path.exists() and path.is_file() for path in tile_paths) + result["checks"]["tile_limit_ok"] = len(tile_paths) <= resolved_settings.yolo_max_tiles + result["tile_count"] = len(tile_paths) + result["status"] = "ready" + result["message"] = "Configured YOLO preflight passed. No model was loaded and no inference was run." + return result diff --git a/backend/app/storage/.gitkeep b/backend/app/storage/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/utils/.gitkeep b/backend/app/utils/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/utils/geometry.py b/backend/app/utils/geometry.py new file mode 100644 index 00000000..645ceae7 --- /dev/null +++ b/backend/app/utils/geometry.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from typing import Any + +from pyproj import Transformer +from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, box, shape +from shapely.ops import transform +from shapely.validation import make_valid + + +def normalize_to_multipolygon(raw_geometry: dict[str, Any]) -> MultiPolygon: + geom = shape(raw_geometry) + if geom.is_empty: + raise ValueError("Geometry is empty") + + if not geom.is_valid: + geom = make_valid(geom) + + if not geom.is_valid: + raise ValueError("Geometry is invalid and could not be repaired") + + if geom.geom_type == "Polygon": + return MultiPolygon([geom]) + if geom.geom_type == "MultiPolygon": + return MultiPolygon(geom.geoms) + if isinstance(geom, GeometryCollection): + polygons = [g for g in geom.geoms if isinstance(g, Polygon)] + multipolygons = [g for g in geom.geoms if g.geom_type == "MultiPolygon"] + if not polygons and not multipolygons: + raise ValueError("Only polygon geometries are supported for AOI") + normalized = [] + normalized.extend(polygons) + for mp in multipolygons: + normalized.extend(mp.geoms) + return MultiPolygon(normalized) + + raise ValueError("Only Polygon or MultiPolygon geometries are accepted") + + +def area_bounds_multipolygon(geom: MultiPolygon): + return { + "min_x": float(geom.bounds[0]), + "min_y": float(geom.bounds[1]), + "max_x": float(geom.bounds[2]), + "max_y": float(geom.bounds[3]), + } + + +def area_m2(geom: MultiPolygon) -> float: + projected = transform( + Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True).transform, + geom, + ) + return float(projected.area) + + +def geometry_bbox_polygon(geom: MultiPolygon): + return box(*geom.bounds) diff --git a/backend/app/utils/response.py b/backend/app/utils/response.py new file mode 100644 index 00000000..1f0dc198 --- /dev/null +++ b/backend/app/utils/response.py @@ -0,0 +1,5 @@ +from typing import Any + + +def envelope(payload: Any) -> dict[str, Any]: + return {"data": payload} diff --git a/backend/app/workers/.gitkeep b/backend/app/workers/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/docker_start.sh b/backend/docker_start.sh new file mode 100644 index 00000000..80830477 --- /dev/null +++ b/backend/docker_start.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env sh +set -eu + +echo "Waiting for database connection..." +python - <<'PY' +import time + +from sqlalchemy import create_engine, text + +from app.core.config import get_settings + +settings = get_settings() +last_error = None + +for attempt in range(1, 31): + try: + engine = create_engine(settings.database_url, pool_pre_ping=True, future=True) + with engine.connect() as connection: + connection.execute(text("SELECT 1")) + print(f"Database connection ready after attempt {attempt}.") + break + except Exception as exc: + last_error = exc + print(f"Database not ready yet ({attempt}/30): {exc}") + time.sleep(2) +else: + raise SystemExit(f"Database did not become ready: {last_error}") +PY + +python -m alembic upgrade head +exec uvicorn app.main:app --host 0.0.0.0 --port 8000 diff --git a/backend/pyproject.toml b/backend/pyproject.toml new file mode 100644 index 00000000..809f3498 --- /dev/null +++ b/backend/pyproject.toml @@ -0,0 +1,48 @@ +[project] +name = "geointel-backend" +version = "0.1.0" +description = "GeoIntel Kempen backend" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "fastapi>=0.112.0", + "uvicorn[standard]>=0.30.6", + "SQLAlchemy>=2.0.34", + "psycopg[binary]>=3.2.1", + "pydantic>=2.9.0", + "pydantic-settings>=2.4.0", + "geoalchemy2>=0.15.0", + "shapely>=2.0.4", + "pyproj>=3.6.1", + "python-multipart>=0.0.9", + "alembic>=1.13.2", +] + +[project.optional-dependencies] +raster = [ + "rasterio>=1.4.3", + "numpy>=2.1.0", + "pillow>=10.4.0", +] +gis = [ + "rasterio>=1.4.3", + "numpy>=2.1.0", + "pillow>=10.4.0", + "geopandas>=1.0.1", + "pyogrio>=0.10.0", +] +ai = [ + "ultralytics>=8.3,<9", + "torch>=2.4", +] +dev = ["pytest>=8.3.2", "httpx>=0.27.0", "ruff>=0.6.9"] + +[project.scripts] +geointel-backend = "app.main:main" + +[build-system] +requires = ["setuptools>=74.1.2", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["app"] diff --git a/backend/scripts/gis_import_smoke.py b/backend/scripts/gis_import_smoke.py new file mode 100644 index 00000000..9afcee08 --- /dev/null +++ b/backend/scripts/gis_import_smoke.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import importlib +import json +from typing import Any + + +REQUIRED_MODULES = ("rasterio", "geopandas", "pyogrio") + + +def _module_version(module_name: str) -> str | None: + module = importlib.import_module(module_name) + version = getattr(module, "__version__", None) + return str(version) if version is not None else None + + +def main() -> int: + versions: dict[str, Any] = {} + for module_name in REQUIRED_MODULES: + versions[module_name] = _module_version(module_name) + + print( + json.dumps( + { + "status": "ok", + "gis_imports": versions, + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tests/.gitkeep b/backend/tests/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/test_alembic_logging_config.py b/backend/tests/test_alembic_logging_config.py new file mode 100644 index 00000000..02c99061 --- /dev/null +++ b/backend/tests/test_alembic_logging_config.py @@ -0,0 +1,10 @@ +from pathlib import Path + + +def test_alembic_logging_formatter_uses_runtime_interpolation_tokens() -> None: + config = Path(__file__).resolve().parents[1] / "alembic.ini" + content = config.read_text(encoding="utf-8") + + assert "format = %(levelname)-5.5s [%(name)s] %(message)s" in content + assert "%%(levelname)" not in content + assert "%%(message)" not in content diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py new file mode 100644 index 00000000..251528fc --- /dev/null +++ b/backend/tests/test_docker_runtime_config.py @@ -0,0 +1,173 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_backend_dockerfile_copies_package_sources_before_pip_install() -> None: + dockerfile = ROOT / "backend" / "Dockerfile" + lines = dockerfile.read_text(encoding="utf-8").splitlines() + + pip_install_index = lines.index('RUN pip install --no-cache-dir ".[gis]"') + preceding = "\n".join(lines[:pip_install_index]) + + assert "COPY pyproject.toml README.md /app/" in preceding + assert "COPY app /app/app" in preceding + + +def test_backend_dockerfile_installs_approved_gis_runtime_stack() -> None: + dockerfile = (ROOT / "backend" / "Dockerfile").read_text(encoding="utf-8") + + assert 'RUN pip install --no-cache-dir ".[gis]"' in dockerfile + assert "RUN python scripts/gis_import_smoke.py" in dockerfile + assert "gdal-bin" in dockerfile + assert "libgdal-dev" in dockerfile + assert "libgeos-dev" in dockerfile + assert "libproj-dev" in dockerfile + assert "proj-bin" in dockerfile + + +def test_backend_pyproject_exposes_gis_optional_dependency_group() -> None: + pyproject = (ROOT / "backend" / "pyproject.toml").read_text(encoding="utf-8") + + assert "gis = [" in pyproject + assert '"rasterio>=1.4.3"' in pyproject + assert '"geopandas>=1.0.1"' in pyproject + assert '"pyogrio>=0.10.0"' in pyproject + assert '"ultralytics>=8.3,<9"' not in pyproject.split("gis = [", 1)[1].split("]", 1)[0] + + +def test_compose_does_not_require_missing_root_env_file() -> None: + compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") + + assert "env_file:" not in compose + assert "DATABASE_URL: postgresql+psycopg://geointel:geointel@db:5432/geointel" in compose + + +def test_compose_exposes_frontend_on_host_port_1202_with_cors_origin() -> None: + compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") + env_example = (ROOT / ".env.example").read_text(encoding="utf-8") + + assert '"1202:80"' in compose + assert "CORS_ORIGINS: http://localhost:1202,http://127.0.0.1:1202" in compose + assert "http://localhost:1202" in env_example + assert "http://127.0.0.1:1202" in env_example + + +def test_env_example_uses_runtime_env_names_read_by_backend_and_frontend() -> None: + env_example = (ROOT / ".env.example").read_text(encoding="utf-8") + + assert "YOLO_ENABLED=false" in env_example + assert "YOLO_MODEL_PATH=" in env_example + assert "YOLO_MAX_TILES=100" in env_example + assert "ENABLE_YOLO" not in env_example + assert "ENABLE_SAM" not in env_example + assert "VITE_API_BASE_URL=" in env_example + assert "VITE_API_PROXY_TARGET=http://localhost:8000" in env_example + + +def test_frontend_uses_same_origin_api_proxy_by_default() -> None: + api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8") + nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8") + dockerfile = (ROOT / "frontend" / "Dockerfile").read_text(encoding="utf-8") + + assert '?? ""' in api_client + assert "http://localhost:8000" not in api_client + assert "FROM nginx:" in dockerfile + assert "COPY --from=build /app/dist /usr/share/nginx/html" in dockerfile + assert "location /api/" in nginx_config + assert "proxy_pass http://backend:8000/api/" in nginx_config + assert "location = /health" in nginx_config + assert "try_files $uri $uri/ /index.html" in nginx_config + + +def test_compose_does_not_publish_postgis_on_default_host_port() -> None: + compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") + + assert '"5432:5432"' not in compose + + +def test_compose_waits_for_healthy_database_and_applies_migrations() -> None: + compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") + + assert "pg_isready -U geointel -d geointel" in compose + assert "condition: service_healthy" in compose + assert "sh /app/docker_start.sh" in compose + + +def test_compose_has_backend_and_frontend_healthchecks() -> None: + compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") + + assert "http://127.0.0.1:8000/health" in compose + assert "urllib.request.urlopen" in compose + assert "http://127.0.0.1/health" in compose + assert "wget -q -O -" in compose + assert "start_period: 30s" in compose + assert "start_period: 10s" in compose + + +def test_frontend_waits_for_healthy_backend_in_compose() -> None: + compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") + frontend_section = compose.split(" frontend:", 1)[1] + + assert "backend:" in frontend_section + assert "condition: service_healthy" in frontend_section + + +def test_backend_docker_start_script_waits_for_sql_connection_before_migrations() -> None: + script = (ROOT / "backend" / "docker_start.sh").read_text(encoding="utf-8") + + assert "Waiting for database connection" in script + assert "create_engine(settings.database_url" in script + assert "SELECT 1" in script + assert "python -m alembic upgrade head" in script + assert "uvicorn app.main:app --host 0.0.0.0 --port 8000" in script + + +def test_docker_build_contexts_exclude_vendor_build_and_cache_outputs() -> None: + required_patterns = { + "node_modules", + "dist", + "__pycache__", + "*.pyc", + ".pytest_cache", + } + + for relative_path in ("backend/.dockerignore", "frontend/.dockerignore"): + content = (ROOT / relative_path).read_text(encoding="utf-8") + for pattern in required_patterns: + assert pattern in content + + +def test_browser_runtime_verification_script_detects_proxy_contract() -> None: + script = (ROOT / "scripts" / "verify_browser_runtime.sh").read_text(encoding="utf-8") + + assert "/api/v1/projects" in script + assert " None: + script = (ROOT / "scripts" / "verify_gis_runtime.sh").read_text(encoding="utf-8") + + assert "/api/v1/system/capabilities" in script + assert '"postgis":true' in script + assert '"rasterio":true' in script + assert '"geopandas":true' in script + assert " None: + docker_script = (ROOT / "backend" / "scripts" / "gis_import_smoke.py").read_text(encoding="utf-8") + root_wrapper = (ROOT / "scripts" / "gis_import_smoke.py").read_text(encoding="utf-8") + + assert 'REQUIRED_MODULES = ("rasterio", "geopandas", "pyogrio")' in docker_script + assert "importlib.import_module" in docker_script + assert '"gis_imports"' in docker_script + assert 'ROOT / "backend" / "scripts"' in root_wrapper + assert "from gis_import_smoke import main" in root_wrapper + + +def test_backend_docker_context_contains_gis_import_smoke_script() -> None: + assert (ROOT / "backend" / "scripts" / "gis_import_smoke.py").exists() diff --git a/backend/tests/test_geojson_dataset_service.py b/backend/tests/test_geojson_dataset_service.py new file mode 100644 index 00000000..42e876dc --- /dev/null +++ b/backend/tests/test_geojson_dataset_service.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from uuid import uuid4 + +from app.core.errors import AppError +from app.services import geojson_service +from app.services.dataset_service import DatasetService + + +def test_parse_geojson_payload_extracts_metadata() -> None: + payload = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [4.5, 51.3], + }, + } + ], + } + + metadata = geojson_service.parse_geojson_payload(payload) + + assert metadata["feature_count"] == 1 + assert metadata["feature_geometry_count"] == 1 + assert metadata["bounds_json"] == { + "min_x": 4.5, + "min_y": 51.3, + "max_x": 4.5, + "max_y": 51.3, + } + assert metadata["geometry_types"] == ["Point"] + + +def test_parse_geojson_payload_rejects_non_feature_collection() -> None: + payload = {"type": "Feature", "features": []} + + try: + geojson_service.parse_geojson_payload(payload) + except ValueError as exc: + assert "FeatureCollection" in str(exc) + else: + raise AssertionError("Invalid GeoJSON should raise ValueError") + + +def test_get_dataset_geojson_reads_stored_payload(tmp_path, monkeypatch) -> None: + file_path = tmp_path / "dataset.geojson" + file_path.write_text( + json.dumps({"type": "FeatureCollection", "features": []}), + encoding="utf-8", + ) + + dataset = SimpleNamespace(dataset_type="vector", storage_path=str(file_path)) + monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset) + + payload = DatasetService.get_dataset_geojson(Path("."), uuid4()) + + assert payload["type"] == "FeatureCollection" + + +def test_get_dataset_geojson_rejects_invalid_stored_json(tmp_path, monkeypatch) -> None: + file_path = tmp_path / "invalid.geojson" + file_path.write_text("not-json", encoding="utf-8") + + dataset = SimpleNamespace(dataset_type="vector", storage_path=str(file_path)) + monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset) + + try: + DatasetService.get_dataset_geojson(Path("."), uuid4()) + except AppError as exc: + assert exc.code == "INVALID_GEOJSON" + else: + raise AssertionError("Invalid stored payload should raise AppError") + + +def test_parse_geojson_payload_returns_vector_metadata() -> None: + payload = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [4.3, 51.2], + [4.4, 51.2], + [4.4, 51.3], + [4.3, 51.3], + [4.3, 51.2], + ] + ], + }, + } + ], + "crs": {"type": "name", "properties": {"name": "EPSG:31370"}}, + } + + metadata = geojson_service.parse_geojson_payload(payload) + + assert metadata["feature_count"] == 1 + assert metadata["feature_geometry_count"] == 1 + assert metadata["geometry_types"] == ["Polygon"] + assert metadata["bounds_json"] == { + "min_x": 4.3, + "min_y": 51.2, + "max_x": 4.4, + "max_y": 51.3, + } + assert metadata["crs"] == "EPSG:31370" + assert metadata["approximate_area_m2"] is not None + assert metadata["approximate_area_m2"] >= 0.0 + + +def test_parse_geojson_payload_rejects_invalid_geometry() -> None: + payload = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": "invalid", + }, + } + ], + } + + try: + geojson_service.parse_geojson_payload(payload) + except ValueError as exc: + assert "Invalid feature geometry" in str(exc) + else: + raise AssertionError("Invalid geometry should raise ValueError") + + +def test_get_dataset_geojson_accepts_legacy_geojson_type(tmp_path, monkeypatch) -> None: + file_path = tmp_path / "legacy.geojson" + file_path.write_text( + json.dumps({"type": "FeatureCollection", "features": []}), + encoding="utf-8", + ) + + dataset = SimpleNamespace(dataset_type="geojson", storage_path=str(file_path)) + monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset) + + payload = DatasetService.get_dataset_geojson(Path("."), uuid4()) + assert payload["type"] == "FeatureCollection" + + +def test_vector_summary_supports_legacy_geojson_type(monkeypatch) -> None: + dataset = SimpleNamespace( + dataset_type="geojson", + metadata_json={ + "feature_count": 7, + "geometry_types": ["Point"], + "bounds_json": {"min_x": 0.0, "min_y": 0.0, "max_x": 1.0, "max_y": 1.0}, + }, + storage_path="", + ) + monkeypatch.setattr(DatasetService, "get_dataset", lambda _db, _id: dataset) + + summary = DatasetService.vector_summary(Path("."), uuid4()) + assert summary["feature_count"] == 7 + assert summary["geometry_types"] == ["Point"] diff --git a/backend/tests/test_health.py b/backend/tests/test_health.py new file mode 100644 index 00000000..d690b31c --- /dev/null +++ b/backend/tests/test_health.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from fastapi.testclient import TestClient + +from app.api.routes import health +from app.main import app + + +def test_health_endpoint_returns_status_payload() -> None: + client = TestClient(app) + response = client.get("/health") + + assert response.status_code == 200 + + payload = response.json() + assert payload["status"] in {"ok", "degraded"} + assert payload["service"] == "geointel-backend" + assert payload["version"] == "0.1.0" + + +def test_system_capabilities_reports_gis_dependency_flags(monkeypatch) -> None: + monkeypatch.setattr( + health, + "_dependency_enabled", + lambda module_name: module_name in {"rasterio", "geopandas"}, + ) + + client = TestClient(app) + response = client.get("/api/v1/system/capabilities") + + assert response.status_code == 200 + payload = response.json() + assert payload["data"]["rasterio"] is True + assert payload["data"]["geopandas"] is True diff --git a/backend/tests/test_live_migration_smoke_script.py b/backend/tests/test_live_migration_smoke_script.py new file mode 100644 index 00000000..1302fab0 --- /dev/null +++ b/backend/tests/test_live_migration_smoke_script.py @@ -0,0 +1,24 @@ +from pathlib import Path + + +def test_live_migration_smoke_checks_postgis_after_migrations() -> None: + script = Path(__file__).resolve().parents[2] / "scripts" / "live_migration_smoke.sh" + content = script.read_text(encoding="utf-8") + + upgrade_index = content.index("-m alembic upgrade head") + postgis_index = content.index("PostGIS_Version()") + + assert upgrade_index < postgis_index + + +def test_live_migration_smoke_checks_required_runtime_schema_objects() -> None: + script = Path(__file__).resolve().parents[2] / "scripts" / "live_migration_smoke.sh" + content = script.read_text(encoding="utf-8") + + assert "to_regclass(:object_name)" in content + assert '"public.projects"' in content + assert '"public.datasets"' in content + assert '"public.vector_features"' in content + assert '"public.detections"' in content + assert '"public.segmentations"' in content + assert '"public.ix_segmentations_geometry"' in content diff --git a/backend/tests/test_qa_service.py b/backend/tests/test_qa_service.py new file mode 100644 index 00000000..e2002dc2 --- /dev/null +++ b/backend/tests/test_qa_service.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from uuid import uuid4 + +from app.models import Area, Dataset +from app.services.qa_service import QaService + + +class FakeSession: + def __init__(self, datasets=None, areas=None): + self.datasets = {item.id: item for item in (datasets or [])} + self.areas = {item.id: item for item in (areas or [])} + + def get(self, model, item_id): + if model.__name__ == "Dataset": + return self.datasets.get(item_id) + if model.__name__ == "Area": + return self.areas.get(item_id) + return None + + +def _write_dataset(path: Path, coordinates: list[list[list[float]]]) -> None: + payload = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": coordinates, + }, + }, + ], + } + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None: + project_id = uuid4() + candidate_id = uuid4() + reference_id = uuid4() + candidate_path = tmp_path / "candidate.geojson" + reference_path = tmp_path / "reference.geojson" + polygon = [[[4.0, 51.0], [4.1, 51.0], [4.1, 51.1], [4.0, 51.1], [4.0, 51.0]]] + _write_dataset(candidate_path, polygon) + _write_dataset(reference_path, polygon) + + candidate = Dataset( + id=candidate_id, + project_id=project_id, + name="candidate.geojson", + dataset_type="vector", + source="test", + storage_path=str(candidate_path), + crs="EPSG:4326", + metadata_json={"crs_assumed": False}, + ) + reference = Dataset( + id=reference_id, + project_id=project_id, + name="reference.geojson", + dataset_type="vector", + source="test", + storage_path=str(reference_path), + crs="EPSG:4326", + metadata_json={"crs_assumed": False}, + ) + + result = QaService.compare_candidate_with_reference( + db=FakeSession([candidate, reference]), + project_id=project_id, + candidate_dataset_id=candidate_id, + reference_dataset_id=reference_id, + iou_threshold=0.5, + ) + + assert result.status == "ok" + assert result.matches == 1 + assert result.false_positives == 0 + assert result.false_negatives == 0 + assert result.precision == 1.0 + assert result.recall == 1.0 + assert result.f1_score == 1.0 + + +def test_dataset_reference_metadata_migration_declares_required_columns() -> None: + migration_path = Path(__file__).parents[1] / "alembic" / "versions" / "202606120001_add_dataset_reference_metadata.py" + migration_text = migration_path.read_text(encoding="utf-8") + for column_name in ( + "dataset_role", + "source_name", + "reference_layer_name", + "source_metadata", + "provenance_metadata", + "imported_at", + ): + assert column_name in migration_text diff --git a/backend/tests/test_raster_operations_service.py b/backend/tests/test_raster_operations_service.py new file mode 100644 index 00000000..52ecffe7 --- /dev/null +++ b/backend/tests/test_raster_operations_service.py @@ -0,0 +1,1370 @@ +from __future__ import annotations + +from types import ModuleType, SimpleNamespace +from uuid import uuid4 +from pathlib import Path +import importlib + +from geoalchemy2.shape import from_shape +from app.core.errors import AppError +from app.models import Area, Dataset +from app.services.raster_operations_service import RasterOperationsService +from app.api.routes.datasets import _run_job_sync +from shapely.geometry import box +import pytest + + +class FakeSession: + def __init__(self, datasets=None, areas=None): + self.datasets = {item.id: item for item in (datasets or [])} + self.areas = {item.id: item for item in (areas or [])} + self.added = [] + + def get(self, model, item_id): + if model.__name__ == "Dataset": + return self.datasets.get(item_id) + if model.__name__ == "Area": + return self.areas.get(item_id) + return None + + def add(self, item): + self.added.append(item) + + def commit(self): + return None + + def refresh(self, _item): + return None + + +def test_raster_preview_dependency_aware_when_rasterio_unavailable(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "raster.tif" + source.write_bytes(b"\x00\x01") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="raster.tif", + stored_filename="raster.tif", + content_type="image/tiff", + size_bytes=2, + ) + db = FakeSession([dataset]) + monkeypatch.setattr( + "app.services.raster_operations_service._import_rasterio", + lambda: (_ for _ in ()).throw(ImportError("rasterio not installed")), + ) + try: + RasterOperationsService.preview(db, dataset_id) + except AppError as exc: + assert exc.code == "RASTER_PROCESSING_UNAVAILABLE" + else: + raise AssertionError("Missing rasterio should raise RASTER_PROCESSING_UNAVAILABLE") + + +def test_raster_stats_dependency_aware_when_numpy_unavailable(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "raster.tif" + source.write_bytes(b"dummy") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="raster.tif", + stored_filename="raster.tif", + content_type="image/tiff", + size_bytes=5, + ) + db = FakeSession([dataset]) + + class FakeRasterio: + def open(self, *_args, **_kwargs): + raise AssertionError("stats should fail before opening raster when numpy import fails") + + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (FakeRasterio(), None)) + monkeypatch.setattr("app.services.raster_operations_service._import_numpy", lambda: (_ for _ in ()).throw(ImportError("numpy not installed"))) + + try: + RasterOperationsService.stats(db, dataset_id) + except AppError as exc: + assert exc.code == "RASTER_PROCESSING_UNAVAILABLE" + else: + raise AssertionError("Missing numpy should raise RASTER_PROCESSING_UNAVAILABLE for stats") + + +def test_raster_preview_returns_metadata_payload(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "raster.tif" + source.write_bytes(b"dummy-raster") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="raster.tif", + stored_filename="raster.tif", + content_type="image/tiff", + size_bytes=13, + checksum_sha256="checksum", + ) + db = FakeSession([dataset]) + + class FakeSource: + width = 200 + height = 120 + count = 4 + + def __init__(self): + self.shape = (4, 120, 200) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + def read(self, *args, **kwargs): + return [[[1, 2], [3, 4]]] + + class FakeWindowsModule: + pass + + class FakeRasterio: + class enums: + class Resampling: + nearest = "nearest" + + windows = FakeWindowsModule() + + def open(self, _path): + return FakeSource() + + metadata = { + "width": 200, + "height": 120, + "band_count": 4, + "bounds": [0.0, 0.0, 1.0, 1.0], + "crs": "EPSG:3857", + "dtype": ["uint8"], + "resolution": [1.0, 1.0], + "transform": [1, 0, 0, 0, -1, 0, 0, 0, 1], + "nodata": None, + "driver": "GTiff", + } + + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (FakeRasterio(), None)) + monkeypatch.setattr( + "app.services.raster_operations_service.extract_raster_metadata", + lambda _path: dict(metadata), + ) + monkeypatch.setattr("app.services.storage_service.get_settings", lambda: SimpleNamespace(storage_root=str(tmp_path))) + monkeypatch.setattr( + "app.services.raster_operations_service.RasterOperationsService._write_preview_image", + lambda _data, _path: (100, 80), + ) + + payload = RasterOperationsService.preview(db, dataset_id) + assert payload["dataset_id"] == str(dataset_id) + assert payload["ready"] is True + assert payload["preview"]["format"] == "PNG" + assert payload["metadata"]["size_bytes"] == 13 + assert payload["metadata"]["checksum_sha256"] == "checksum" + + +def test_raster_reproject_rejects_invalid_crs(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "raster.tif" + source.write_bytes(b"dummy") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="raster.tif", + stored_filename="raster.tif", + content_type="image/tiff", + size_bytes=5, + ) + db = FakeSession([dataset]) + + class FakeCRS: + @staticmethod + def from_user_input(_value): + raise ValueError("invalid") + + class FakeRasterio: + class crs: + CRS = FakeCRS + + def open(self, *_args, **_kwargs): + raise AssertionError("Invalid CRS should fail before opening raster") + + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (FakeRasterio(), None)) + + try: + RasterOperationsService.reproject(db, dataset_id, target_crs="not-a-crs", output_name=None) + except AppError as exc: + assert exc.code == "INVALID_PARAMETERS" + else: + raise AssertionError("Invalid target CRS should fail with INVALID_PARAMETERS") + + +def test_raster_reproject_returns_persisted_derived_dataset(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + output_id = uuid4() + source = tmp_path / "raster.tif" + source.write_bytes(b"dummy") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="raster.tif", + stored_filename="raster.tif", + content_type="image/tiff", + size_bytes=5, + ) + db = FakeSession([dataset]) + + class FakeWarp: + @staticmethod + def calculate_default_transform(*_args, **_kwargs): + return ("transform", 8, 9) + + @staticmethod + def reproject(**_kwargs): + return None + + class FakeResampling: + nearest = "nearest" + bilinear = "nearest" + cubic = "nearest" + + class FakeCRS: + def __init__(self, value: str): + self.value = value + + def __str__(self): + return self.value + + def to_string(self): + return self.value + + @staticmethod + def from_user_input(value: str): + return FakeCRS(value) + + class FakeTransform: + @staticmethod + def to_gdal(): + return [1, 0, 0, 0, 1, 0, 0, 0, 1] + + class FakeSource: + width = 10 + height = 12 + count = 2 + crs = FakeCRS("EPSG:3857") + transform = FakeTransform() + nodata = 0 + meta = { + "driver": "GTiff", + "dtype": "uint8", + "count": 2, + "width": 10, + "height": 12, + "crs": "EPSG:3857", + "transform": FakeTransform(), + } + + @staticmethod + def band(_source, band_index): + return (band_index,) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeOutput: + def __init__(self, path: str): + self._path = Path(path) + + def write(self, _data): + self._path.parent.mkdir(parents=True, exist_ok=True) + self._path.write_bytes(b"reprojected") + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeRasterio: + crs = FakeCRS + enums = type("enums", (), {"Resampling": FakeResampling}) + band = FakeSource.band + warp = FakeWarp + windows = type("windows", (), {}) + + def open(self, path: str, mode: str = "r", **_kwargs): + if "w" in mode: + return FakeOutput(path) + return FakeSource() + + metadata = { + "width": 8, + "height": 9, + "band_count": 2, + "bounds": [0.0, 0.0, 8.0, 9.0], + "crs": "EPSG:31370", + "dtype": ["uint8", "uint8"], + "resolution": [1.0, 1.0], + "transform": [1, 0, 0, 0, -1, 0, 0, 0, 1], + "nodata": None, + "driver": "GTiff", + } + + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (FakeRasterio(), None)) + monkeypatch.setattr("app.services.raster_operations_service.extract_raster_metadata", lambda _path: dict(metadata)) + monkeypatch.setattr("app.services.storage_service.get_settings", lambda: SimpleNamespace(storage_root=str(tmp_path))) + monkeypatch.setattr("uuid.uuid4", lambda: output_id) + + result_id = RasterOperationsService.reproject( + db, + dataset_id=dataset_id, + target_crs="EPSG:31370", + output_name="reprojected_raster", + ) + + assert result_id == output_id + assert len(db.added) == 1 + derived = db.added[0] + assert derived.id == output_id + assert derived.metadata_json is not None + assert derived.metadata_json["operation"] == "raster.reproject" + assert derived.metadata_json["source_dataset_id"] == str(dataset_id) + assert derived.metadata_json["operation_parameters"]["target_crs"] == "EPSG:31370" + assert derived.metadata_json["target_crs"] == "EPSG:31370" + assert derived.metadata_json["output_dataset_id"] == str(output_id) + + +def test_raster_inspect_returns_payload(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "raster.tif" + source.write_bytes(b"dummy-raster") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="raster.tif", + stored_filename="raster.tif", + content_type="image/tiff", + size_bytes=13, + checksum_sha256="checksum", + ) + db = FakeSession([dataset]) + metadata = { + "width": 200, + "height": 120, + "band_count": 4, + "bounds": [0.0, 0.0, 1.0, 1.0], + "crs": "EPSG:3857", + "dtype": ["uint8"], + "resolution": [1.0, 1.0], + "transform": [1, 0, 0, 0, -1, 0, 0, 0, 1], + "nodata": None, + "driver": "GTiff", + } + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (SimpleNamespace(), None)) + monkeypatch.setattr("app.services.raster_operations_service.extract_raster_metadata", lambda _path: dict(metadata)) + payload = RasterOperationsService.inspect(db, dataset_id) + assert payload["dataset_id"] == str(dataset_id) + assert payload["ready"] is True + assert payload["metadata"]["driver"] == "GTiff" + assert payload["metadata"]["dataset_id"] == str(dataset_id) + assert payload["metadata"]["size_bytes"] == 13 + + +def test_raster_inspect_rejects_non_raster_dataset(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "not-raster.json" + source.write_text("{}", encoding="utf-8") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="not-raster.json", + dataset_type="vector", + source="user_upload", + storage_path=str(source), + original_filename="not-raster.json", + stored_filename="not-raster.json", + content_type="application/geo+json", + ) + db = FakeSession([dataset]) + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (SimpleNamespace(), None)) + try: + RasterOperationsService.inspect(db, dataset_id) + except AppError as exc: + assert exc.code == "INVALID_DATASET_TYPE" + else: + raise AssertionError("Inspecting vector dataset as raster should fail") + + +def test_raster_tile_dependency_aware_when_rasterio_unavailable(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "raster.tif" + source.write_bytes(b"\x00\x01") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="raster.tif", + stored_filename="raster.tif", + content_type="image/tiff", + size_bytes=2, + ) + db = FakeSession([dataset]) + monkeypatch.setattr( + "app.services.raster_operations_service._import_rasterio", + lambda: (_ for _ in ()).throw(ImportError("rasterio not installed")), + ) + try: + RasterOperationsService.tile(db, dataset_id, tile_size=512, overlap=64) + except AppError as exc: + assert exc.code == "RASTER_PROCESSING_UNAVAILABLE" + else: + raise AssertionError("Missing rasterio should raise RASTER_PROCESSING_UNAVAILABLE for tile") + + +def test_raster_tile_validation_rejects_bad_parameters() -> None: + try: + RasterOperationsService._validate_tile_request(tile_size=0, overlap=0) + except AppError as exc: + assert exc.code == "INVALID_PARAMETERS" + else: + raise AssertionError("Tile size 0 should be rejected") + + try: + RasterOperationsService._validate_tile_request(tile_size=256, overlap=300) + except AppError as exc: + assert exc.code == "INVALID_PARAMETERS" + else: + raise AssertionError("Overlap larger than tile size should be rejected") + + +def test_raster_clip_rejects_non_raster_dataset(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + area_id = uuid4() + source = tmp_path / "not-raster.geojson" + source.write_text("{}", encoding="utf-8") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="not-raster.geojson", + dataset_type="vector", + source="user_upload", + storage_path=str(source), + original_filename="not-raster.geojson", + stored_filename="not-raster.geojson", + content_type="application/geo+json", + ) + area = Area(id=area_id, project_id=project_id, geometry="POINT(0 0)", original_crs="EPSG:4326") + db = FakeSession([dataset], [area]) + try: + RasterOperationsService.clip(db, dataset_id, area_id, None) + except AppError as exc: + assert exc.code == "INVALID_DATASET_TYPE" + else: + raise AssertionError("Clipping vector dataset as raster should fail") + + +def test_raster_clip_rejects_missing_area(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + area_id = uuid4() + source = tmp_path / "raster.tif" + source.write_bytes(b"\x00\x01") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="raster.tif", + stored_filename="raster.tif", + content_type="image/tiff", + size_bytes=2, + ) + db = FakeSession([dataset], []) + monkeypatch.setattr( + "app.services.raster_operations_service._import_rasterio", + lambda: (SimpleNamespace(), SimpleNamespace()), + ) + try: + RasterOperationsService.clip(db, dataset_id, area_id, None) + except AppError as exc: + assert exc.code == "AREA_NOT_FOUND" + else: + raise AssertionError("Clipping without area should fail with AREA_NOT_FOUND") + + +def test_raster_tile_returns_manifest_payload(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "raster.tif" + source.write_bytes(b"source") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="raster.tif", + stored_filename="raster.tif", + content_type="image/tiff", + size_bytes=6, + ) + db = FakeSession([dataset]) + + class FakeArray: + shape = (1, 10, 10) + + @property + def size(self): + return self.shape[0] * self.shape[1] * self.shape[2] + + class FakeWindow: + def __init__(self, xoff: float, yoff: float, width: float, height: float): + self.xoff = xoff + self.yoff = yoff + self.width = width + self.height = height + + class FakeWindowTransform: + def __init__(self, xoff: float, yoff: float, width: float, height: float): + self.xoff = xoff + self.yoff = yoff + self.width = width + self.height = height + + def to_gdal(self): + return [1.0, 0.0, self.xoff, 0.0, -1.0, self.yoff, 0.0, 0.0, 1.0] + + class FakeWindowBounds: + def __init__(self, xoff: float, yoff: float, width: float, height: float): + self.left = float(xoff) + self.right = float(xoff + width) + self.bottom = float(yoff) + self.top = float(yoff + height) + + class FakeWindows: + @staticmethod + def Window(xoff: float, yoff: float, width: float, height: float): + return FakeWindow(xoff, yoff, width, height) + + @staticmethod + def transform(window: FakeWindow, _source_transform): + return FakeWindowTransform(window.xoff, window.yoff, window.width, window.height) + + @staticmethod + def bounds(window: FakeWindow, _source_transform): + return FakeWindowBounds(window.xoff, window.yoff, window.width, window.height) + + class FakeSource: + width = 10 + height = 10 + + def __init__(self): + self.profile = {"width": self.width, "height": self.height, "count": 1, "dtype": "uint8", "transform": None} + self.transform = None + self.nodata = 0 + + def read(self, *args, **kwargs): + return FakeArray() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeOutput: + def __init__(self, path: Path): + self._path = path + + def write(self, _data): + self._path.parent.mkdir(parents=True, exist_ok=True) + self._path.write_text("tile") + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeRasterio: + windows = FakeWindows + + def __call__(self, *_args, **_kwargs): + return FakeSource() + + def open(self, path: str, mode: str = "r", **_kwargs): + if mode and "w" in mode: + return FakeOutput(Path(path)) + return FakeSource() + + fake_rasterio = FakeRasterio() + + monkeypatch.setattr( + "app.services.raster_operations_service._import_rasterio", + lambda: (fake_rasterio, SimpleNamespace()), + ) + + payload = RasterOperationsService.tile(db, dataset_id, tile_size=4, overlap=1, output_name="fixture") + + assert payload["dataset_id"] == str(dataset_id) + assert payload["ready"] is True + assert payload["tile_set_id"] is not None + assert payload["count"] >= 1 + assert payload["count"] == len(payload["manifest"]["tiles"]) + assert payload["count"] == len(payload["manifest"]["tile_paths"]) + assert payload["manifest"]["tiles"][0]["index"] == 0 + assert payload["manifest_path"].endswith(".json") + assert payload["manifest"]["tile_size"] == 4 + assert payload["manifest"]["overlap"] == 1 + assert payload["manifest"]["source_dataset_id"] == str(dataset_id) + assert payload["manifest"]["source_raster_id"] == str(dataset_id) + assert payload["manifest"]["count"] == payload["count"] + assert payload["manifest"]["ai_inference"] is False + assert payload["manifest"]["tile_server"] is None + + +def test_raster_clip_persists_derived_dataset(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "source.tif" + source.write_bytes(b"source") + output_id = uuid4() + area_id = uuid4() + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="source.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="source.tif", + stored_filename="source.tif", + content_type="image/tiff", + size_bytes=6, + ) + area = Area( + id=area_id, + project_id=project_id, + geometry=from_shape(box(0.0, 0.0, 1.0, 1.0), srid=4326), + original_crs="EPSG:4326", + ) + db = FakeSession([dataset], [area]) + + class FakeClippedData: + shape = (1, 3, 4) + + @property + def size(self): + return 12 + + class FakeOutput: + def __init__(self, output_file: Path): + self.output_file = output_file + + def write(self, _data): + self.output_file.parent.mkdir(parents=True, exist_ok=True) + self.output_file.write_bytes(b"derived") + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeSource: + width = 10 + height = 10 + crs = SimpleNamespace(to_string=lambda: "EPSG:3857") + nodata = 0.0 + profile = {"width": 10, "height": 10, "count": 1, "dtype": "uint8", "transform": "identity"} + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeMask: + @staticmethod + def mask(_source, _geom, crop=True, nodata=None, filled=True): + return FakeClippedData(), SimpleNamespace(to_gdal=lambda: [1, 0, 0, 0, 1, 0, 0, 0, 1]) + + class FakeRasterio: + mask = FakeMask() + + def open(self, path: str, mode: str = "r", **_kwargs): + if "w" in mode: + return FakeOutput(Path(path)) + return FakeSource() + + metadata = { + "width": 4, + "height": 3, + "band_count": 1, + "bounds": [0.0, 0.0, 4.0, 3.0], + "crs": "EPSG:3857", + "dtype": ["uint8"], + "resolution": [1.0, 1.0], + "transform": [1, 0, 0, 0, -1, 0, 0, 0, 1], + "nodata": None, + "driver": "GTiff", + } + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (FakeRasterio(), None)) + monkeypatch.setattr("app.services.raster_operations_service.extract_raster_metadata", lambda _path: dict(metadata)) + monkeypatch.setattr("app.services.storage_service.get_settings", lambda: SimpleNamespace(storage_root=str(tmp_path))) + + # Force a deterministic derived output id so we can assert provenance fields. + monkeypatch.setattr( + "uuid.uuid4", + lambda: output_id, + ) + + result_id = RasterOperationsService.clip(db, dataset_id, area_id, "clip-result.tif") + + assert result_id == output_id + assert len(db.added) == 1 + derived = db.added[0] + assert isinstance(derived, Dataset) + assert derived.id == output_id + assert derived.source == "operation:raster.clip" + assert derived.dataset_type == "raster" + assert derived.derived_from_dataset_id == dataset_id + assert derived.metadata_json is not None + assert derived.metadata_json.get("operation") == "raster.clip" + assert derived.metadata_json.get("source_dataset_id") == str(dataset_id) + assert derived.metadata_json.get("operation_parameters", {}).get("area_id") == str(area_id) + assert derived.storage_path is not None + assert Path(derived.storage_path).exists() + + +def test_raster_clip_rejects_dataset_without_crs(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + area_id = uuid4() + source = tmp_path / "raster.tif" + source.write_bytes(b"\x00\x01") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="raster.tif", + stored_filename="raster.tif", + content_type="image/tiff", + size_bytes=2, + ) + area = Area(id=area_id, project_id=project_id, geometry=from_shape(box(0.0, 0.0, 1.0, 1.0), srid=4326), original_crs="EPSG:4326") + db = FakeSession([dataset], [area]) + + class FakeSource: + width = 10 + height = 10 + crs = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeRasterio: + def open(self, _path): + return FakeSource() + + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (FakeRasterio(), SimpleNamespace())) + try: + RasterOperationsService.clip(db, dataset_id, area_id, None) + except AppError as exc: + assert exc.code == "INVALID_DATASET_CRS" + else: + raise AssertionError("Clipping raster without CRS should fail") + + +def test_run_job_sync_persists_job_output_dataset_for_raster_ops(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + + recorded = {} + + class FakeJobRecord: + def __init__(self, job_id): + self.id = job_id + self.job_type = "raster.reproject" + self.status = "success" + self.project_id = project_id + self.dataset_id = None + self.input_dataset_id = dataset_id + self.output_dataset_id = None + self.parameters_json = {} + self.result_json = {} + self.error_message = None + self.created_at = None + self.started_at = None + self.finished_at = None + + def model_dump(self) -> dict: + return { + "id": self.id, + "job_type": self.job_type, + "status": self.status, + "project_id": self.project_id, + "dataset_id": self.dataset_id, + "input_dataset_id": self.input_dataset_id, + "output_dataset_id": self.output_dataset_id, + "parameters_json": self.parameters_json, + "result_json": self.result_json, + "error_message": self.error_message, + "created_at": self.created_at, + "started_at": self.started_at, + "finished_at": self.finished_at, + } + + fake_job_id = uuid4() + fake_output_dataset_id = uuid4() + + def fake_create_job(_db, payload): + recorded["payload"] = payload + return FakeJobRecord(fake_job_id) + + def fake_mark_running(_db, _job_id): + recorded["running_called_with"] = _job_id + return FakeJobRecord(fake_job_id) + + def fake_mark_success(_db, _job_id, result=None, output_dataset_id=None): + record = FakeJobRecord(fake_job_id) + record.result_json = result + record.output_dataset_id = output_dataset_id + return record + + def fake_mark_failed(*_args, **_kwargs): + raise AssertionError("Raster job failure path should not execute") + + monkeypatch.setattr("app.api.routes.datasets.JobService.create_job", fake_create_job) + monkeypatch.setattr("app.api.routes.datasets.JobService.mark_running", fake_mark_running) + monkeypatch.setattr("app.api.routes.datasets.JobService.mark_success", fake_mark_success) + monkeypatch.setattr("app.api.routes.datasets.JobService.mark_failed", fake_mark_failed) + + result = _run_job_sync( + db=SimpleNamespace(add=lambda _item: None, commit=lambda: None, refresh=lambda _item: None), + project_id=project_id, + input_dataset_id=dataset_id, + job_type="raster.reproject", + parameters={}, + operation=lambda: fake_output_dataset_id, + ) + + assert result["output_dataset_id"] == str(fake_output_dataset_id) + assert result["result_json"]["output_dataset_id"] == str(fake_output_dataset_id) + assert recorded["running_called_with"] == fake_job_id + + +def test_run_job_sync_records_raster_job_error(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + recorded = {} + + class FakeJobRecord: + def __init__(self, job_id): + self.id = job_id + self.job_type = "raster.clip" + self.status = "failed" + self.project_id = project_id + self.dataset_id = None + self.input_dataset_id = dataset_id + self.output_dataset_id = None + self.parameters_json = {} + self.result_json = {} + self.error_message = "Operation failed" + self.created_at = None + self.started_at = None + self.finished_at = None + + def model_dump(self) -> dict: + return { + "id": self.id, + "job_type": self.job_type, + "status": self.status, + "project_id": self.project_id, + "dataset_id": self.dataset_id, + "input_dataset_id": self.input_dataset_id, + "output_dataset_id": self.output_dataset_id, + "parameters_json": self.parameters_json, + "result_json": self.result_json, + "error_message": self.error_message, + "created_at": self.created_at, + "started_at": self.started_at, + "finished_at": self.finished_at, + } + + fake_job_id = uuid4() + + def fake_create_job(_db, _payload): + return FakeJobRecord(fake_job_id) + + def fake_mark_running(_db, _job_id): + recorded["running_called_with"] = _job_id + return FakeJobRecord(fake_job_id) + + def fake_mark_failed(_db, _job_id, error_message, details): + record = FakeJobRecord(_job_id) + record.error_message = error_message + record.result_json = details + recorded["mark_failed_payload"] = {"error_message": error_message, "details": details} + return record + + monkeypatch.setattr("app.api.routes.datasets.JobService.create_job", fake_create_job) + monkeypatch.setattr("app.api.routes.datasets.JobService.mark_running", fake_mark_running) + monkeypatch.setattr("app.api.routes.datasets.JobService.mark_failed", fake_mark_failed) + + try: + _run_job_sync( + db=SimpleNamespace(add=lambda _item: None, commit=lambda: None, refresh=lambda _item: None), + project_id=project_id, + input_dataset_id=dataset_id, + job_type="raster.reproject", + parameters={}, + operation=lambda: (_ for _ in ()).throw(AppError(code="INVALID_DATASET_CRS", message="Missing CRS", status_code=400)), + ) + raise AssertionError("Expected AppError to be raised") + except AppError as exc: + assert exc.code == "INVALID_DATASET_CRS" + + assert recorded["running_called_with"] == fake_job_id + assert recorded["mark_failed_payload"]["error_message"] == "Missing CRS" + assert recorded["mark_failed_payload"]["details"]["code"] == "INVALID_DATASET_CRS" + + +def test_raster_clip_rejects_empty_raster_clip(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + area_id = uuid4() + source = tmp_path / "source.tif" + source.write_bytes(b"source") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="source.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="source.tif", + stored_filename="source.tif", + content_type="image/tiff", + size_bytes=6, + ) + area = Area( + id=area_id, + project_id=project_id, + geometry=from_shape(box(0.0, 0.0, 1.0, 1.0), srid=4326), + original_crs="EPSG:4326", + ) + db = FakeSession([dataset], [area]) + + class FakeOutputData: + size = 0 + + class FakeMask: + @staticmethod + def mask(_source, _geom, crop=True, nodata=None, filled=True): + return FakeOutputData(), SimpleNamespace(to_gdal=lambda: [1, 0, 0, 0, 1, 0, 0, 0, 1]) + + class FakeSource: + width = 10 + height = 10 + crs = SimpleNamespace(to_string=lambda: "EPSG:3857") + nodata = 0 + profile = {"width": 10, "height": 10, "count": 1, "dtype": "uint8", "transform": "identity"} + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeRasterio: + mask = FakeMask() + + def open(self, _path): + return FakeSource() + + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (FakeRasterio(), SimpleNamespace())) + monkeypatch.setattr( + "app.services.raster_operations_service._import_numpy", + lambda: SimpleNamespace(asarray=lambda _values: _values, isfinite=lambda _values: False), + ) + try: + RasterOperationsService.clip(db, dataset_id, area_id, None) + except AppError as exc: + assert exc.code == "RASTER_OPERATION_EMPTY_RESULT" + else: + raise AssertionError("Clip that produces no raster data should fail") + + +def test_raster_ndvi_rejects_invalid_band_index(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "source.tif" + source.write_bytes(b"source") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="source.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="source.tif", + stored_filename="source.tif", + content_type="image/tiff", + size_bytes=6, + ) + db = FakeSession([dataset]) + + class FakeSource: + width = 4 + height = 4 + count = 3 + profile = {"dtype": "uint16", "count": 3, "width": 4, "height": 4} + nodata = 0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeRasterio: + def open(self, path, *args, **kwargs): + return FakeSource() + + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (FakeRasterio(), None)) + monkeypatch.setattr( + "app.services.raster_operations_service._import_numpy", + lambda: importlib.import_module("numpy"), + ) + try: + RasterOperationsService.ndvi(db, dataset_id, nir_band=4, red_band=1) + except AppError as exc: + assert exc.code == "INVALID_PARAMETERS" + assert "nir_band exceeds available band count" in exc.message + else: + raise AssertionError("Band index exceeding source band count should fail") + + +def test_raster_ndvi_dependency_aware(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "source.tif" + source.write_bytes(b"source") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="source.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="source.tif", + stored_filename="source.tif", + content_type="image/tiff", + size_bytes=6, + ) + db = FakeSession([dataset]) + + monkeypatch.setattr( + "app.services.raster_operations_service._import_rasterio", + lambda: (_ for _ in ()).throw(ImportError("rasterio not installed")), + ) + try: + RasterOperationsService.ndvi(db, dataset_id, nir_band=1, red_band=1) + except AppError as exc: + assert exc.code == "RASTER_PROCESSING_UNAVAILABLE" + else: + raise AssertionError("Missing rasterio should raise RASTER_PROCESSING_UNAVAILABLE for spectral index") + + +def test_raster_ndbi_dependency_aware_when_numpy_missing(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "source.tif" + source.write_bytes(b"source") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="source.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="source.tif", + stored_filename="source.tif", + content_type="image/tiff", + size_bytes=6, + ) + db = FakeSession([dataset]) + + class FakeSource: + width = 4 + height = 4 + count = 6 + profile = {"dtype": "uint16", "count": 6, "width": 4, "height": 4} + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + def read(self, *_args, **_kwargs): + raise AssertionError("ndbi should fail before raster read when numpy is unavailable") + + class FakeRasterio: + def open(self, path, *args, **kwargs): + return FakeSource() + + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (FakeRasterio(), None)) + monkeypatch.setattr( + "app.services.raster_operations_service._import_numpy", + lambda: (_ for _ in ()).throw(ImportError("numpy missing")), + ) + + try: + RasterOperationsService.ndbi(db, dataset_id, swir_band=1, nir_band=2) + except AppError as exc: + assert exc.code == "RASTER_PROCESSING_UNAVAILABLE" + else: + raise AssertionError("Missing numpy should raise RASTER_PROCESSING_UNAVAILABLE for spectral index") + + +def test_raster_index_records_provenance_and_dtype(tmp_path, monkeypatch) -> None: + try: + numpy = importlib.import_module("numpy") + except Exception as exc: + pytest.skip(f"numpy unavailable: {exc}") + + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "source.tif" + source.write_bytes(b"source") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="source.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="source.tif", + stored_filename="source.tif", + content_type="image/tiff", + size_bytes=6, + ) + output_dataset_id = uuid4() + db = FakeSession([dataset]) + + class FakeOutput: + def __init__(self, path: Path): + self.path = path + + def write(self, _data, indexes=1, window=None): + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_bytes(b"indexed") + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeSource: + width = 4 + height = 4 + count = 4 + nodata = 0 + profile = { + "driver": "GTiff", + "dtype": "uint16", + "count": 4, + "width": 4, + "height": 4, + "transform": "identity", + } + + def read(self, band_index, window=None, out_dtype=None): + return numpy.array( + [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]], + dtype=out_dtype, + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeWindows: + @staticmethod + def Window(xoff, yoff, width, height): + return (xoff, yoff, width, height) + + class FakeRasterio: + windows = FakeWindows + + def open(self, path: str, mode: str = "r", **_kwargs): + if "w" in mode: + return FakeOutput(Path(path)) + return FakeSource() + + metadata = { + "width": 4, + "height": 4, + "band_count": 4, + "bounds": [0.0, 0.0, 4.0, 4.0], + "crs": "EPSG:31370", + "dtype": ["uint16"], + "resolution": [1.0, 1.0], + "transform": [1, 0, 0, 0, -1, 0, 0, 0, 1], + "nodata": None, + "driver": "GTiff", + } + + monkeypatch.setattr("app.services.raster_operations_service._import_rasterio", lambda: (FakeRasterio(), None)) + monkeypatch.setattr("app.services.raster_operations_service._import_numpy", lambda: numpy) + monkeypatch.setattr("app.services.raster_operations_service.extract_raster_metadata", lambda _path: dict(metadata)) + monkeypatch.setattr("app.services.storage_service.get_settings", lambda: SimpleNamespace(storage_root=str(tmp_path))) + monkeypatch.setattr("uuid.uuid4", lambda: output_dataset_id) + + result_dataset_id = RasterOperationsService.ndvi(db, dataset_id, nir_band=4, red_band=3, output_name="ndvi-test") + assert result_dataset_id == output_dataset_id + assert len(db.added) == 1 + derived = db.added[0] + assert derived.id == output_dataset_id + assert derived.metadata_json is not None + assert derived.metadata_json["operation"] == "raster.ndvi" + assert derived.metadata_json["source_dataset_id"] == str(dataset_id) + assert derived.metadata_json["band_mapping"]["nir_band"] == 4 + assert derived.metadata_json["band_mapping"]["red_band"] == 3 + assert derived.metadata_json["formula"] == "(nir - red) / (nir + red)" + assert derived.metadata_json["output_dtype"] == "float32" + assert derived.metadata_json["nodata_strategy"]["mode"] == "nan" + assert "path" in derived.metadata_json + assert derived.metadata_json["path"] == derived.storage_path + assert derived.storage_path is not None + assert derived.metadata_json["created_at"] is not None + assert derived.metadata_json["output_dataset_id"] == str(output_dataset_id) + + +def test_run_job_sync_serializes_index_job_output_dataset_id(monkeypatch) -> None: + project_id = uuid4() + dataset_id = uuid4() + output_dataset_id = uuid4() + + class FakeJobRecord: + def __init__(self, job_id): + self.id = job_id + self.job_type = "raster.ndvi" + self.status = "success" + self.project_id = project_id + self.dataset_id = None + self.input_dataset_id = dataset_id + self.output_dataset_id = None + self.parameters_json = {} + self.result_json = {} + self.error_message = None + self.created_at = None + self.started_at = None + self.finished_at = None + + def model_dump(self) -> dict: + return { + "id": self.id, + "job_type": self.job_type, + "status": self.status, + "project_id": self.project_id, + "dataset_id": self.dataset_id, + "input_dataset_id": self.input_dataset_id, + "output_dataset_id": self.output_dataset_id, + "parameters_json": self.parameters_json, + "result_json": self.result_json, + "error_message": self.error_message, + "created_at": self.created_at, + "started_at": self.started_at, + "finished_at": self.finished_at, + } + + fake_job_id = uuid4() + + def fake_create_job(_db, payload): + return FakeJobRecord(fake_job_id) + + def fake_mark_running(_db, _job_id): + return FakeJobRecord(fake_job_id) + + def fake_mark_success(_db, _job_id, result=None, output_dataset_id=None): + record = FakeJobRecord(fake_job_id) + record.result_json = result + record.output_dataset_id = output_dataset_id + return record + + monkeypatch.setattr("app.api.routes.datasets.JobService.create_job", fake_create_job) + monkeypatch.setattr("app.api.routes.datasets.JobService.mark_running", fake_mark_running) + monkeypatch.setattr("app.api.routes.datasets.JobService.mark_success", fake_mark_success) + + result = _run_job_sync( + db=SimpleNamespace(add=lambda _item: None, commit=lambda: None, refresh=lambda _item: None), + project_id=project_id, + input_dataset_id=dataset_id, + job_type="raster.ndvi", + parameters={"nir_band": 4, "red_band": 3}, + operation=lambda: output_dataset_id, + ) + + assert result["job_type"] == "raster.ndvi" + assert result["output_dataset_id"] == str(output_dataset_id) + assert result["result_json"]["output_dataset_id"] == str(output_dataset_id) + diff --git a/backend/tests/test_raster_service.py b/backend/tests/test_raster_service.py new file mode 100644 index 00000000..c7e51398 --- /dev/null +++ b/backend/tests/test_raster_service.py @@ -0,0 +1,67 @@ +from app.core.errors import AppError +from app.services.raster_service import extract_raster_metadata + + +def test_extract_raster_metadata_returns_dependency_aware_error(monkeypatch, tmp_path) -> None: + monkeypatch.setattr("app.services.raster_service._import_rasterio", lambda: (_ for _ in ()).throw(ImportError("rasterio not installed"))) + + file_path = tmp_path / "missing.tif" + file_path.write_bytes(b"\x00\x01\x02") + + try: + extract_raster_metadata(str(file_path)) + except AppError as exc: + assert exc.code == "RASTER_PROCESSING_UNAVAILABLE" + else: + raise AssertionError("Missing rasterio should raise AppError code RASTER_PROCESSING_UNAVAILABLE") + + +def test_extract_raster_metadata_maps_basic_profile_fields(monkeypatch, tmp_path) -> None: + file_path = tmp_path / "sample.tif" + file_path.write_bytes(b"fake") + + class FakeDataset: + width = 1024 + height = 768 + count = 4 + driver = "GTiff" + crs = "EPSG:31370" + bounds = (100.0, 200.0, 500.0, 800.0) + res = (0.25, 0.25) + dtypes = ["uint16", "uint16", "uint16", "uint16"] + nodata = -9999 + + class transform: + @staticmethod + def to_gdal(): + return (0.25, 0.0, 100.0, 0.0, -0.25, 800.0, 0.0, 0.0, 1.0) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + class FakeRasterio: + class errors: + class RasterioIOError(Exception): + ... + + def open(self, *_): + return FakeDataset() + + class FakeErrors: + RasterioIOError = FakeRasterio.errors.RasterioIOError + + monkeypatch.setattr("app.services.raster_service._import_rasterio", lambda: (FakeRasterio(), FakeErrors())) + + metadata = extract_raster_metadata(str(file_path)) + assert metadata["driver"] == "GTiff" + assert metadata["width"] == 1024 + assert metadata["height"] == 768 + assert metadata["band_count"] == 4 + assert metadata["crs"] == "EPSG:31370" + assert metadata["bounds"] == [100.0, 200.0, 500.0, 800.0] + assert metadata["resolution"] == [0.25, 0.25] + assert metadata["dtype"] == ["uint16", "uint16", "uint16", "uint16"] + assert metadata["nodata"] == -9999.0 diff --git a/backend/tests/test_readiness_gate.py b/backend/tests/test_readiness_gate.py new file mode 100644 index 00000000..508ee2d9 --- /dev/null +++ b/backend/tests/test_readiness_gate.py @@ -0,0 +1,42 @@ +from pathlib import Path + + +def test_readiness_gate_treats_deprecation_warnings_as_errors() -> None: + script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh" + content = script.read_text(encoding="utf-8") + + assert "-W error::DeprecationWarning" in content + + +def test_readiness_gate_runs_contract_smoke() -> None: + script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh" + content = script.read_text(encoding="utf-8") + + assert "scripts/smoke_contracts.py" in content + + +def test_readiness_gate_checks_demo_export_workflow_script_syntax() -> None: + script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh" + content = script.read_text(encoding="utf-8") + + assert "bash -n scripts/verify_demo_export_workflow.sh" in content + + +def test_demo_export_workflow_script_verifies_export_endpoints() -> None: + script = Path(__file__).resolve().parents[2] / "scripts" / "verify_demo_export_workflow.sh" + content = script.read_text(encoding="utf-8") + + assert "/api/v1/demo/workflow" in content + assert "/api/v1/exports/metadata" in content + assert "/api/v1/exports/report" in content + assert "/api/v1/exports/geojson" in content + assert "/download" in content + + +def test_pass_end_check_excludes_vendor_and_build_outputs() -> None: + script = Path(__file__).resolve().parents[2] / "scripts" / "codex_pass_end_check.sh" + content = script.read_text(encoding="utf-8") + + assert "--exclude-dir=node_modules" in content + assert "--exclude-dir=dist" in content + assert "--exclude-dir=__pycache__" in content diff --git a/backend/tests/test_sprint12_golden_qa_benchmark.py b/backend/tests/test_sprint12_golden_qa_benchmark.py new file mode 100644 index 00000000..ec25c2cf --- /dev/null +++ b/backend/tests/test_sprint12_golden_qa_benchmark.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_golden_qa_expected_metrics_are_documented() -> None: + expected_path = ROOT / "fixtures" / "golden" / "expected_qa_metrics.json" + expected = json.loads(expected_path.read_text(encoding="utf-8")) + + assert expected["benchmark_id"] == "golden-buildings-partial-match-v1" + assert expected["iou_threshold"] == 0.5 + assert expected["candidate_feature_count"] == 2 + assert expected["reference_feature_count"] == 2 + assert expected["matches"] == 1 + assert expected["false_positive_count"] == 1 + assert expected["false_negative_count"] == 1 + assert expected["precision"] == 0.5 + assert expected["recall"] == 0.5 + assert expected["f1"] == 0.5 + assert expected["mean_iou"] > 0.8 + assert expected["tolerance"] <= 1e-9 + + +def test_golden_qa_benchmark_command_passes_and_reports_persistence() -> None: + script = ROOT / "scripts" / "run_golden_qa_benchmark.py" + result = subprocess.run( + [sys.executable, str(script), "--json"], + cwd=ROOT, + check=True, + text=True, + capture_output=True, + ) + payload = json.loads(result.stdout) + + assert payload["status"] == "passed" + assert payload["benchmark_id"] == "golden-buildings-partial-match-v1" + assert payload["metrics"]["precision"] == 0.5 + assert payload["metrics"]["recall"] == 0.5 + assert payload["metrics"]["f1"] == 0.5 + assert payload["metrics"]["false_positive_count"] == 1 + assert payload["metrics"]["false_negative_count"] == 1 + assert payload["persistence"]["quality_check_count"] == 1 + assert payload["persistence"]["metric_count"] == 6 + assert sorted(payload["persistence"]["metric_keys"]) == [ + "f1", + "false_negative_count", + "false_positive_count", + "mean_iou", + "precision", + "recall", + ] diff --git a/backend/tests/test_sprint13_yolo_preflight.py b/backend/tests/test_sprint13_yolo_preflight.py new file mode 100644 index 00000000..bf4f994c --- /dev/null +++ b/backend/tests/test_sprint13_yolo_preflight.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +from app.core.config import Settings +from app.services.yolo_preflight_service import YoloPreflightService + + +ROOT = Path(__file__).resolve().parents[2] + + +class AvailableAdapter: + @staticmethod + def dependencies_available() -> bool: + return True + + +class MissingDependencyAdapter: + @staticmethod + def dependencies_available() -> bool: + return False + + +def _manifest(tmp_path: Path, tile_count: int = 1) -> Path: + tiles = [] + for index in range(tile_count): + tile_path = tmp_path / f"tile_{index:04d}.tif" + tile_path.write_bytes(b"tile") + tiles.append( + { + "path": str(tile_path), + "pixel_window": [0, 0, 100, 100], + "bounds": [4.0, 51.0, 5.0, 52.0], + "transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01], + "index": index, + } + ) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps({"tiles": tiles, "count": tile_count}), encoding="utf-8") + return manifest_path + + +def test_yolo_preflight_reports_disabled_without_loading_model(tmp_path: Path) -> None: + result = YoloPreflightService.run( + settings=Settings(yolo_enabled=False, yolo_model_path=str(tmp_path / "missing.pt")), + tile_manifest_path=str(tmp_path / "missing-manifest.json"), + yolo_adapter_class=AvailableAdapter, + ) + + assert result["status"] == "not_configured" + assert result["checks"]["enabled"] is False + assert result["checks"]["dependencies_available"] is None + assert result["checks"]["model_file_exists"] is None + assert result["checks"]["manifest_valid"] is None + + +def test_yolo_preflight_distinguishes_missing_dependencies_from_missing_model(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"weights") + + result = YoloPreflightService.run( + settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path)), + tile_manifest_path=str(_manifest(tmp_path)), + yolo_adapter_class=MissingDependencyAdapter, + ) + + assert result["status"] == "dependency_unavailable" + assert result["checks"]["dependencies_available"] is False + assert result["checks"]["model_file_exists"] is None + assert result["checks"]["manifest_valid"] is None + + +def test_yolo_preflight_validates_model_and_manifest_without_importing_yolo(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"weights") + manifest_path = _manifest(tmp_path, tile_count=2) + + result = YoloPreflightService.run( + settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4), + tile_manifest_path=str(manifest_path), + yolo_adapter_class=AvailableAdapter, + ) + + assert result["status"] == "ready" + assert result["checks"]["dependencies_available"] is True + assert result["checks"]["model_file_exists"] is True + assert result["checks"]["manifest_valid"] is True + assert result["tile_count"] == 2 + assert result["will_download_models"] is False + assert result["will_run_inference"] is False + + +def test_yolo_preflight_script_outputs_json(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"weights") + manifest_path = _manifest(tmp_path) + + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "yolo_preflight.py"), + "--model-path", + str(model_path), + "--tile-manifest-path", + str(manifest_path), + "--assume-dependencies", + "--json", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(result.stdout) + + assert payload["status"] == "ready" + assert payload["model_path"] == str(model_path) + assert payload["tile_manifest_path"] == str(manifest_path) diff --git a/backend/tests/test_sprint15_demo_workflow.py b/backend/tests/test_sprint15_demo_workflow.py new file mode 100644 index 00000000..1e0dc06c --- /dev/null +++ b/backend/tests/test_sprint15_demo_workflow.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from uuid import uuid4 + +from fastapi.testclient import TestClient + +from app.main import app +from app.schemas.demo import DemoWorkflowResponse +from app.services.demo_workflow_service import DemoWorkflowService + + +def test_demo_workflow_endpoint_returns_canonical_envelope(monkeypatch) -> None: + project_id = uuid4() + area_id = uuid4() + reference_dataset_id = uuid4() + candidate_dataset_id = uuid4() + quality_check_id = uuid4() + + monkeypatch.setattr( + DemoWorkflowService, + "seed", + lambda _db: DemoWorkflowResponse( + project_id=project_id, + area_id=area_id, + reference_dataset_id=reference_dataset_id, + candidate_dataset_id=candidate_dataset_id, + quality_check_id=quality_check_id, + metric_count=6, + status="ready", + message="Demo workflow seeded from explicit local fixtures.", + created=True, + ), + ) + + response = TestClient(app).post("/api/v1/demo/workflow") + + assert response.status_code == 201 + payload = response.json() + assert set(payload) == {"data"} + assert payload["data"]["project_id"] == str(project_id) + assert payload["data"]["reference_dataset_id"] == str(reference_dataset_id) + assert payload["data"]["candidate_dataset_id"] == str(candidate_dataset_id) + assert payload["data"]["quality_check_id"] == str(quality_check_id) + assert payload["data"]["metric_count"] == 6 + assert payload["data"]["status"] == "ready" + assert payload["data"]["created"] is True + + +def test_demo_workflow_service_uses_explicit_golden_fixtures() -> None: + reference_path = DemoWorkflowService._fixture_path("reference_buildings.geojson") + candidate_path = DemoWorkflowService._fixture_path("predicted_buildings.geojson") + + assert reference_path.exists() + assert candidate_path.exists() + assert DemoWorkflowService.PROJECT_NAME == "GeoIntel Demo - Building QA" + assert DemoWorkflowService.REFERENCE_FILENAME == "demo_reference_buildings.geojson" + assert DemoWorkflowService.CANDIDATE_FILENAME == "demo_predicted_buildings.geojson" diff --git a/backend/tests/test_sprint16_quality_checks_dashboard.py b/backend/tests/test_sprint16_quality_checks_dashboard.py new file mode 100644 index 00000000..c602f3a5 --- /dev/null +++ b/backend/tests/test_sprint16_quality_checks_dashboard.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import uuid4 + +from fastapi.testclient import TestClient + +from app.main import app +from app.models import Metric, QualityCheck +from app.schemas.qa import QualityCheckRead +from app.services.quality_check_service import QualityCheckService + + +class FakeQuery: + def __init__(self, rows): + self.rows = rows + + def filter(self, *_args): + return self + + def order_by(self, *_args): + return self + + def count(self): + return len(self.rows) + + def offset(self, _offset): + return self + + def limit(self, _limit): + return self + + def all(self): + return self.rows + + +class FakeSession: + def __init__(self, quality_checks, metrics): + self.quality_checks = quality_checks + self.metrics = metrics + + def query(self, model): + if model is QualityCheck: + return FakeQuery(self.quality_checks) + if model is Metric: + return FakeQuery(self.metrics) + return FakeQuery([]) + + +def test_quality_check_service_lists_checks_with_metrics() -> None: + project_id = uuid4() + quality_check_id = uuid4() + reference_dataset_id = uuid4() + candidate_dataset_id = uuid4() + created_at = datetime.now(timezone.utc) + quality_check = QualityCheck( + id=quality_check_id, + project_id=project_id, + candidate_dataset_id=candidate_dataset_id, + reference_dataset_id=reference_dataset_id, + check_type="demo_candidate_vs_reference", + status="ok", + score=0.5, + parameters_json={"iou_threshold": 0.5}, + findings_json={"matches": 1}, + created_at=created_at, + completed_at=created_at, + ) + metric = Metric( + id=uuid4(), + quality_check_id=quality_check_id, + metric_key="precision", + metric_value=0.5, + metadata_json={}, + created_at=created_at, + ) + + items, total = QualityCheckService.list_quality_checks( + FakeSession([quality_check], [metric]), + project_id=project_id, + ) + + assert total == 1 + assert len(items) == 1 + assert items[0].id == quality_check_id + assert items[0].metrics[0].metric_key == "precision" + assert items[0].metrics[0].metric_value == 0.5 + + +def test_quality_checks_endpoint_returns_canonical_envelope(monkeypatch) -> None: + project_id = uuid4() + quality_check_id = uuid4() + reference_dataset_id = uuid4() + + monkeypatch.setattr( + QualityCheckService, + "list_quality_checks", + lambda *_args, **_kwargs: ( + [ + QualityCheckRead( + id=quality_check_id, + project_id=project_id, + reference_dataset_id=reference_dataset_id, + check_type="demo_candidate_vs_reference", + status="ok", + score=0.5, + metrics=[], + ) + ], + 1, + ), + ) + + response = TestClient(app).get(f"/api/v1/projects/{project_id}/quality-checks") + + assert response.status_code == 200 + payload = response.json() + assert set(payload) == {"data"} + assert payload["data"]["total"] == 1 + assert payload["data"]["items"][0]["id"] == str(quality_check_id) + assert payload["data"]["items"][0]["check_type"] == "demo_candidate_vs_reference" diff --git a/backend/tests/test_sprint17_export_foundation.py b/backend/tests/test_sprint17_export_foundation.py new file mode 100644 index 00000000..0e74f842 --- /dev/null +++ b/backend/tests/test_sprint17_export_foundation.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from uuid import uuid4 + +from fastapi.testclient import TestClient + +from app.core.errors import AppError +from app.main import app +from app.models import Dataset, Export, Project, QualityCheck +from app.schemas.export import ExportCreateResponse +from app.services.export_service import ExportService +from app.services.storage_service import StorageService + + +class FakeQuery: + def __init__(self, rows): + self.rows = rows + + def filter(self, *_args): + return self + + def order_by(self, *_args): + return self + + def offset(self, _offset): + return self + + def limit(self, _limit): + return self + + def count(self): + return len(self.rows) + + def all(self): + return self.rows + + +class FakeSession: + def __init__(self, rows): + self.rows = rows + self.added = [] + + def get(self, model, row_id): + row = self.rows.get((model, row_id)) + if row is not None: + return row + for item in self.added: + if isinstance(item, model) and item.id == row_id: + return item + return None + + def query(self, model): + rows = [row for (row_model, _row_id), row in self.rows.items() if row_model is model] + rows.extend([row for row in self.added if isinstance(row, model)]) + return FakeQuery(rows) + + def add(self, row): + self.added.append(row) + + def commit(self): + return None + + def refresh(self, row): + return row + + +def test_dataset_geojson_export_persists_export_and_writes_artifact(tmp_path, monkeypatch) -> None: + project_id = uuid4() + dataset_id = uuid4() + dataset_path = tmp_path / "input.geojson" + dataset_path.write_text(json.dumps({"type": "FeatureCollection", "features": []}), encoding="utf-8") + export_path = tmp_path / "exports" / "buildings.geojson" + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="buildings.geojson", + dataset_type="vector", + source="fixture", + storage_path=str(dataset_path), + status="ready", + ) + db = FakeSession({(Dataset, dataset_id): dataset}) + monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path)) + + response = ExportService.export_dataset_geojson(db, dataset_id, name="buildings") + + exports = [item for item in db.added if isinstance(item, Export)] + assert len(exports) == 1 + assert response.export_id == exports[0].id + assert response.export_type == "dataset_geojson" + assert response.metadata_json["feature_count"] == 0 + assert json.loads(export_path.read_text(encoding="utf-8"))["type"] == "FeatureCollection" + + +def test_dataset_geojson_export_rejects_raster_dataset(tmp_path, monkeypatch) -> None: + dataset_id = uuid4() + dataset = Dataset( + id=dataset_id, + project_id=uuid4(), + name="ortho.tif", + dataset_type="raster", + source="fixture", + storage_path=str(tmp_path / "ortho.tif"), + status="ready", + ) + db = FakeSession({(Dataset, dataset_id): dataset}) + monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(tmp_path / "unused.geojson")) + + try: + ExportService.export_dataset_geojson(db, dataset_id) + except AppError as exc: + assert exc.code == "INVALID_DATASET_TYPE" + else: + raise AssertionError("Raster datasets must not be exported as dataset GeoJSON") + + +def test_project_metadata_export_persists_json_summary(tmp_path, monkeypatch) -> None: + project_id = uuid4() + dataset_id = uuid4() + quality_check_id = uuid4() + project = Project(id=project_id, name="Demo", region="Kempen", status="active") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="reference.geojson", + dataset_type="vector", + source="fixture", + dataset_role="reference", + source_name="fixture", + status="ready", + metadata_json={"feature_count": 2}, + ) + quality_check = QualityCheck( + id=quality_check_id, + project_id=project_id, + reference_dataset_id=dataset_id, + check_type="demo_candidate_vs_reference", + status="ok", + score=0.5, + created_at=datetime.now(timezone.utc), + ) + previous_export_id = uuid4() + previous_export = Export( + id=previous_export_id, + project_id=project_id, + export_type="dataset_geojson", + storage_path="storage/exports/previous.geojson", + metadata_json={"source": "dataset"}, + created_at=datetime.now(timezone.utc), + ) + export_path = tmp_path / "metadata.json" + db = FakeSession( + { + (Project, project_id): project, + (Dataset, dataset_id): dataset, + (QualityCheck, quality_check_id): quality_check, + (Export, previous_export_id): previous_export, + } + ) + monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path)) + + response = ExportService.export_project_metadata(db, project_id) + + payload = json.loads(export_path.read_text(encoding="utf-8")) + assert response.export_type == "project_metadata_json" + assert payload["project"]["id"] == str(project_id) + assert payload["datasets"][0]["id"] == str(dataset_id) + assert payload["quality_checks"][0]["id"] == str(quality_check_id) + assert payload["exports"][0]["id"] == str(previous_export_id) + assert response.metadata_json["export_count"] == 1 + + +def test_project_report_export_persists_html_artifact(tmp_path, monkeypatch) -> None: + project_id = uuid4() + dataset_id = uuid4() + project = Project(id=project_id, name="Demo ", description="QA report", region="Kempen", status="active") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="reference.geojson", + dataset_type="vector", + source="fixture", + dataset_role="reference", + status="ready", + metadata_json={"feature_count": 2}, + ) + previous_export_id = uuid4() + previous_export = Export( + id=previous_export_id, + project_id=project_id, + export_type="project_metadata_json", + storage_path="storage/exports/metadata.json", + metadata_json={"source": "project_metadata"}, + created_at=datetime.now(timezone.utc), + ) + export_path = tmp_path / "report.html" + db = FakeSession({(Project, project_id): project, (Dataset, dataset_id): dataset, (Export, previous_export_id): previous_export}) + monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path)) + + response = ExportService.export_project_report(db, project_id) + + html = export_path.read_text(encoding="utf-8") + assert response.export_type == "project_report_html" + assert response.metadata_json["format"] == "html" + assert "" in html + assert "Demo <Kempen>" in html + assert "reference.geojson" in html + assert "Export History (1)" in html + assert "project_metadata_json" in html + + +def test_export_content_reads_persisted_artifact(tmp_path) -> None: + export_id = uuid4() + export_path = tmp_path / "artifact.json" + export_path.write_text(json.dumps({"hello": "world"}), encoding="utf-8") + export = Export( + id=export_id, + project_id=uuid4(), + export_type="project_metadata_json", + storage_path=str(export_path), + metadata_json={}, + ) + db = FakeSession({(Export, export_id): export}) + + response = ExportService.get_export_content(db, export_id) + + assert response.export_id == export_id + assert response.content == {"hello": "world"} + + +def test_export_download_path_rejects_missing_artifact(tmp_path) -> None: + export_id = uuid4() + export = Export( + id=export_id, + project_id=uuid4(), + export_type="dataset_geojson", + storage_path=str(tmp_path / "missing.geojson"), + metadata_json={}, + ) + db = FakeSession({(Export, export_id): export}) + + try: + ExportService.get_export_download_path(db, export_id) + except AppError as exc: + assert exc.code == "EXPORT_CONTENT_NOT_FOUND" + else: + raise AssertionError("Missing export artifacts must fail clearly") + + +def test_export_geojson_endpoint_returns_canonical_envelope(monkeypatch) -> None: + export_id = uuid4() + dataset_id = uuid4() + + monkeypatch.setattr( + ExportService, + "export_dataset_geojson", + lambda *_args, **_kwargs: ExportCreateResponse( + export_id=export_id, + path="storage/exports/demo.geojson", + status="ready", + export_type="dataset_geojson", + metadata_json={"source": "dataset"}, + ), + ) + + response = TestClient(app).post("/api/v1/exports/geojson", json={"dataset_id": str(dataset_id)}) + + assert response.status_code == 200 + payload = response.json() + assert set(payload) == {"data"} + assert payload["data"]["export_id"] == str(export_id) + assert payload["data"]["export_type"] == "dataset_geojson" + + +def test_export_download_endpoint_returns_file_response(tmp_path, monkeypatch) -> None: + export_id = uuid4() + export_path = tmp_path / "download.geojson" + export_path.write_text(json.dumps({"type": "FeatureCollection", "features": []}), encoding="utf-8") + monkeypatch.setattr(ExportService, "get_export_download_path", lambda *_args, **_kwargs: export_path) + + response = TestClient(app).get(f"/api/v1/exports/{export_id}/download") + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("application/json") + assert "download.geojson" in response.headers["content-disposition"] + assert response.json()["type"] == "FeatureCollection" + + +def test_export_download_endpoint_returns_html_media_type(tmp_path, monkeypatch) -> None: + export_id = uuid4() + export_path = tmp_path / "report.html" + export_path.write_text("report", encoding="utf-8") + monkeypatch.setattr(ExportService, "get_export_download_path", lambda *_args, **_kwargs: export_path) + + response = TestClient(app).get(f"/api/v1/exports/{export_id}/download") + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/html") + assert "report.html" in response.headers["content-disposition"] + assert "report" in response.text diff --git a/backend/tests/test_sprint7a_persistence_foundation.py b/backend/tests/test_sprint7a_persistence_foundation.py new file mode 100644 index 00000000..4b0f7b48 --- /dev/null +++ b/backend/tests/test_sprint7a_persistence_foundation.py @@ -0,0 +1,292 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path +from uuid import uuid4 + +import pytest + +from app.api.routes.qa import compare_candidate_with_reference +from app.models import Dataset, Metric, Project, QualityCheck, VectorFeature +from app.schemas.qa import QaProviderComparisonRequest +from app.providers.registry import list_provider_capabilities +from app.services.dataset_service import DatasetService +from app.services.quality_service import QualityService +from app.services.vector_feature_service import VectorFeatureService + + +class FakeSession: + def __init__(self, objects=None) -> None: + self.added = [] + self.objects = objects or {} + self.commits = 0 + self.refreshes = [] + + def get(self, model, item_id): + return self.objects.get((model, item_id)) + + def add(self, item) -> None: + self.added.append(item) + + def commit(self) -> None: + self.commits += 1 + + def refresh(self, item) -> None: + self.refreshes.append(item) + + +def test_vector_feature_service_persists_geojson_features_with_properties() -> None: + db = FakeSession() + dataset_id = uuid4() + payload = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "building-1", + "properties": {"class": "building", "height": 7}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [4.0, 51.0], + [4.1, 51.0], + [4.1, 51.1], + [4.0, 51.1], + [4.0, 51.0], + ] + ], + }, + } + ], + } + + persisted = VectorFeatureService.persist_geojson_features( + db=db, + dataset_id=dataset_id, + payload=payload, + feature_class="building", + ) + + assert len(persisted) == 1 + assert isinstance(persisted[0], VectorFeature) + assert persisted[0].dataset_id == dataset_id + assert persisted[0].feature_class == "building" + assert persisted[0].source_feature_id == "building-1" + assert persisted[0].properties_json == {"class": "building", "height": 7} + assert db.added == persisted + assert db.commits == 1 + + +def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None: + project_id = uuid4() + db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Geel")}) + payload = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {"class": "building"}, + "geometry": { + "type": "Point", + "coordinates": [4.0, 51.0], + }, + } + ], + } + + class Upload: + filename = "reference.geojson" + content_type = "application/geo+json" + + async def read(self) -> bytes: + import json + + return json.dumps(payload).encode("utf-8") + + storage_path = tmp_path / "reference.geojson" + storage_path.write_text("{}", encoding="utf-8") + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_dataset_file", + lambda **_kwargs: { + "storage_path": str(storage_path), + "original_filename": "reference.geojson", + "stored_filename": "reference.geojson", + "content_type": "application/geo+json", + "size_bytes": 2, + "checksum_sha256": "0" * 64, + }, + ) + + result = asyncio.run( + DatasetService.upload_dataset( + db=db, + project_id=project_id, + file=Upload(), + dataset_type="vector", + source="user_upload", + dataset_role="reference", + reference_layer_name="buildings", + ) + ) + + persisted_features = [item for item in db.added if isinstance(item, VectorFeature)] + assert result.dataset_role == "reference" + assert result.source_name == "manual" + assert len(persisted_features) == 1 + assert persisted_features[0].dataset_id == result.id + + +def test_quality_service_persists_quality_check_and_metrics() -> None: + db = FakeSession() + project_id = uuid4() + candidate_dataset_id = uuid4() + reference_dataset_id = uuid4() + job_id = uuid4() + + quality_check = QualityService.persist_quality_check( + db=db, + project_id=project_id, + reference_dataset_id=reference_dataset_id, + check_type="candidate_vs_reference", + status="ok", + score=1.0, + parameters={"iou_threshold": 0.5}, + findings={"matches": 1, "false_positives": 0, "false_negatives": 0}, + candidate_dataset_id=candidate_dataset_id, + job_id=job_id, + metrics={ + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "false_positive_count": 0, + }, + ) + + assert isinstance(quality_check, QualityCheck) + assert quality_check.project_id == project_id + assert quality_check.job_id == job_id + assert quality_check.candidate_dataset_id == candidate_dataset_id + assert quality_check.reference_dataset_id == reference_dataset_id + assert quality_check.parameters_json == {"iou_threshold": 0.5} + assert quality_check.findings_json["matches"] == 1 + persisted_metrics = [item for item in db.added if isinstance(item, Metric)] + assert [metric.metric_key for metric in persisted_metrics] == [ + "precision", + "recall", + "f1", + "false_positive_count", + ] + assert persisted_metrics[0].quality_check_id == quality_check.id + assert db.commits == 1 + + +def test_dataset_role_validation_accepts_only_source_derived_reference() -> None: + assert DatasetService._normalize_dataset_role("source") == "source" + assert DatasetService._normalize_dataset_role("derived") == "derived" + assert DatasetService._normalize_dataset_role("reference") == "reference" + + with pytest.raises(Exception) as exc_info: + DatasetService._normalize_dataset_role("osm") + + assert getattr(exc_info.value, "code", None) == "INVALID_DATASET_ROLE" + + +def test_provider_capabilities_expose_sprint7a_contract() -> None: + capabilities = {capability.provider_name: capability.to_dict() for capability in list_provider_capabilities()} + + assert capabilities["osm"]["supported_layers"] == ["buildings", "roads", "water", "landuse"] + assert capabilities["osm"]["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"] + assert capabilities["osm"]["supported_query_modes"] == ["area"] + assert capabilities["osm"]["status"] == "not_configured" + assert capabilities["grb"]["supported_layers"] == ["buildings", "roads", "parcels"] + assert capabilities["grb"]["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"] + assert capabilities["grb"]["supported_query_modes"] == ["area"] + assert capabilities["grb"]["status"] == "not_configured" + + +def test_sprint7a_migration_declares_foundation_tables_and_indexes() -> None: + migration_path = Path(__file__).parents[1] / "alembic" / "versions" / "202606120700_sprint7a_persistence_foundation.py" + migration_text = migration_path.read_text(encoding="utf-8") + + for required_text in ( + "vector_features", + "quality_checks", + "metrics", + "ix_vector_features_geometry", + 'postgresql_using="gist"', + "ix_quality_checks_project_id", + "ix_metrics_quality_check_id", + ): + assert required_text in migration_text + + +def test_qa_route_persists_quality_check_domain_record(monkeypatch) -> None: + project_id = uuid4() + candidate_dataset_id = uuid4() + reference_dataset_id = uuid4() + job_id = uuid4() + candidate_dataset = Dataset( + id=candidate_dataset_id, + project_id=project_id, + name="candidate.geojson", + dataset_type="vector", + source="test", + ) + db = FakeSession(objects={(Dataset, candidate_dataset_id): candidate_dataset}) + + def run_sync_job(**kwargs): + result = kwargs["operation"]() + return { + "id": str(job_id), + "project_id": str(project_id), + "status": "success", + "result_json": result, + } + + monkeypatch.setattr("app.api.routes.qa.JobService.run_sync_job", run_sync_job) + monkeypatch.setattr( + "app.api.routes.qa.QaService.compare_candidate_with_reference", + lambda **_kwargs: type( + "Result", + (), + { + "model_dump": lambda self, **_kwargs: { + "status": "ok", + "matches": 1, + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "mean_iou": 1.0, + "iou_threshold": 0.5, + "warnings": [], + } + }, + )(), + ) + + response = compare_candidate_with_reference( + payload=QaProviderComparisonRequest( + candidate_dataset_id=candidate_dataset_id, + reference_dataset_id=reference_dataset_id, + iou_threshold=0.5, + ), + db=db, + ) + + persisted_quality_checks = [item for item in db.added if isinstance(item, QualityCheck)] + persisted_metrics = [item for item in db.added if isinstance(item, Metric)] + assert response["data"]["result_json"]["quality_check_id"] == str(persisted_quality_checks[0].id) + assert persisted_quality_checks[0].job_id == job_id + assert persisted_quality_checks[0].candidate_dataset_id == candidate_dataset_id + assert persisted_quality_checks[0].reference_dataset_id == reference_dataset_id + assert [metric.metric_key for metric in persisted_metrics] == [ + "precision", + "recall", + "f1", + "mean_iou", + "false_positive_count", + "false_negative_count", + ] diff --git a/backend/tests/test_sprint7b_provider_registry.py b/backend/tests/test_sprint7b_provider_registry.py new file mode 100644 index 00000000..3e5f87ec --- /dev/null +++ b/backend/tests/test_sprint7b_provider_registry.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from fastapi.testclient import TestClient + +from app.main import app +from app.providers.registry import ( + get_provider_dataset_mapping, + get_provider, + import_provider_dataset, + list_provider_capabilities, +) + + +def test_provider_registry_lists_sprint7b_providers() -> None: + providers = {provider.provider_name: provider for provider in list_provider_capabilities()} + + assert set(providers) == {"grb", "osm", "manual", "fixture"} + assert providers["grb"].authority_level == "authoritative" + assert providers["grb"].configured is False + assert providers["grb"].status == "not_configured" + assert providers["osm"].authority_level == "contextual" + assert providers["osm"].configured is False + assert providers["osm"].status == "not_configured" + assert providers["manual"].authority_level == "manual" + assert providers["manual"].configured is True + assert providers["manual"].status == "configured" + assert providers["fixture"].authority_level == "fixture" + assert providers["fixture"].configured is True + assert providers["fixture"].status == "configured" + + +def test_provider_capabilities_include_required_metadata() -> None: + grb = get_provider("grb").capability.to_dict() + + assert grb["provider_name"] == "grb" + assert grb["display_name"] == "GRB" + assert grb["supported_layers"] == ["buildings", "roads", "parcels"] + assert grb["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"] + assert grb["supported_query_modes"] == ["area"] + assert grb["limitation_message"] + assert grb["attribution"] + assert grb["license_note"] + + +def test_provider_to_dataset_mapping_is_enforced() -> None: + assert get_provider_dataset_mapping("grb").model_dump() == { + "provider_name": "grb", + "dataset_role": "reference", + "source_name": "grb", + "reference_required": True, + "write_path": "DatasetService", + } + assert get_provider_dataset_mapping("manual").dataset_role == "reference" + assert get_provider_dataset_mapping("fixture").source_name == "fixture" + assert get_provider_dataset_mapping("osm").dataset_role == "source" + assert get_provider_dataset_mapping("osm", requested_dataset_role="reference").dataset_role == "reference" + + +def test_grb_osm_import_contract_returns_not_configured_without_fetching() -> None: + grb = import_provider_dataset("grb", project_id="project", area_id="area", layers=["buildings"]) + osm = import_provider_dataset("osm", project_id="project", area_id="area", layers=["buildings"]) + + assert grb.status == "not_configured" + assert grb.dataset_id is None + assert "No live" in grb.message + assert osm.status == "not_configured" + assert osm.dataset_id is None + + +def test_manual_fixture_import_contract_points_to_existing_flows() -> None: + manual = import_provider_dataset("manual", project_id="project", area_id=None, layers=["buildings"]) + fixture = import_provider_dataset("fixture", project_id="project", area_id=None, layers=["buildings"]) + + assert manual.status == "upload_flow_required" + assert "upload" in manual.message.lower() + assert fixture.status == "fixture_flow_required" + assert "fixture" in fixture.message.lower() + + +def test_provider_api_envelopes_and_invalid_provider() -> None: + client = TestClient(app) + + list_response = client.get("/api/v1/external/providers") + assert list_response.status_code == 200 + assert {provider["provider_name"] for provider in list_response.json()["data"]["providers"]} == { + "grb", + "osm", + "manual", + "fixture", + } + + detail_response = client.get("/api/v1/external/providers/grb") + assert detail_response.status_code == 200 + assert detail_response.json()["data"]["provider_name"] == "grb" + + layers_response = client.get("/api/v1/external/providers/osm/layers") + assert layers_response.status_code == 200 + assert layers_response.json()["data"]["layers"] == ["buildings", "roads", "water", "landuse"] + + status_response = client.get("/api/v1/external/providers/manual/status") + assert status_response.status_code == 200 + assert status_response.json()["data"]["configured"] is True + + invalid_response = client.get("/api/v1/external/providers/unknown") + assert invalid_response.status_code == 404 + assert invalid_response.json()["error"]["code"] == "PROVIDER_NOT_FOUND" + + +def test_provider_import_api_returns_clear_not_configured_response() -> None: + client = TestClient(app) + + response = client.post( + "/api/v1/external/providers/grb/import", + json={ + "project_id": "project", + "area_id": "area", + "layers": ["buildings"], + }, + ) + + assert response.status_code == 200 + assert response.json()["data"]["provider_name"] == "grb" + assert response.json()["data"]["status"] == "not_configured" + assert response.json()["data"]["dataset_id"] is None + + +def test_live_migration_smoke_script_exists() -> None: + from pathlib import Path + + script = Path(__file__).parents[2] / "scripts" / "live_migration_smoke.sh" + text = script.read_text(encoding="utf-8") + + assert "alembic upgrade head" in text + assert "SELECT PostGIS_Version()" in text + assert "alembic heads" in text diff --git a/backend/tests/test_sprint8_detection_foundation.py b/backend/tests/test_sprint8_detection_foundation.py new file mode 100644 index 00000000..15fe3180 --- /dev/null +++ b/backend/tests/test_sprint8_detection_foundation.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +from pathlib import Path +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient + +from app.main import app +from app.models import AnalysisRun, Dataset, Detection, Job, Project +from app.services.detection_service import DetectionService +from app.services.model_registry_service import ModelRegistryService + + +class FakeSession: + def __init__(self, objects=None) -> None: + self.objects = objects or {} + self.added = [] + self.commits = 0 + self.refreshes = [] + + def get(self, model, item_id): + return self.objects.get((model, item_id)) + + def add(self, item) -> None: + self.added.append(item) + if getattr(item, "id", None) is not None: + self.objects[(item.__class__, item.id)] = item + + def commit(self) -> None: + self.commits += 1 + + def refresh(self, item) -> None: + self.refreshes.append(item) + + +def _project_and_dataset(dataset_type: str = "raster"): + project_id = uuid4() + dataset_id = uuid4() + project = Project(id=project_id, name="Geel") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="source.tif", + dataset_type=dataset_type, + source="user_upload", + storage_path="storage/uploads/source.tif", + ) + db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) + return db, project_id, dataset_id + + +def test_model_registry_returns_detection_placeholders() -> None: + models = {model.model_id: model for model in ModelRegistryService.list_model_capabilities()} + + assert set(models) == {"yolo-placeholder", "yolo-configured", "manual-fixture-detector"} + assert models["yolo-placeholder"].task_type == "object_detection" + assert models["yolo-placeholder"].configured is False + assert models["yolo-placeholder"].status == "not_configured" + assert models["yolo-configured"].configured is False + assert models["yolo-configured"].status == "not_configured" + assert models["manual-fixture-detector"].configured is True + assert "fixture" in models["manual-fixture-detector"].limitation_message.lower() + + +def test_unavailable_detector_creates_failed_run_and_job_without_detections() -> None: + db, project_id, dataset_id = _project_and_dataset() + + result = DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-placeholder", + confidence_threshold=0.5, + class_filter=["building"], + parameters_json={}, + ) + + runs = [item for item in db.added if isinstance(item, AnalysisRun)] + jobs = [item for item in db.added if isinstance(item, Job)] + detections = [item for item in db.added if isinstance(item, Detection)] + + assert result.status == "failed" + assert result.error_code == "DETECTION_MODEL_UNAVAILABLE" + assert result.detection_count == 0 + assert runs[0].analysis_type == "detection" + assert runs[0].status == "failed" + assert jobs[0].job_type == "detection.run" + assert jobs[0].status == "failed" + assert detections == [] + + +def test_non_raster_dataset_request_is_rejected() -> None: + db, project_id, dataset_id = _project_and_dataset(dataset_type="vector") + + with pytest.raises(Exception) as exc_info: + DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-placeholder", + confidence_threshold=0.5, + ) + + assert getattr(exc_info.value, "code", None) == "INVALID_DATASET_TYPE" + + +def test_fixture_detector_persists_detections_only_with_explicit_fixture_mode() -> None: + db, project_id, dataset_id = _project_and_dataset() + + with pytest.raises(Exception) as exc_info: + DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="manual-fixture-detector", + confidence_threshold=0.5, + parameters_json={"fixture_detections": []}, + ) + + assert getattr(exc_info.value, "code", None) == "FIXTURE_MODE_REQUIRED" + + result = DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="manual-fixture-detector", + confidence_threshold=0.5, + class_filter=["building"], + parameters_json={ + "fixture_mode": True, + "fixture_detections": [ + { + "class_name": "building", + "confidence": 0.92, + "bbox_json": {"x_min": 1, "y_min": 2, "x_max": 10, "y_max": 12}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [4.0, 51.0], + [4.1, 51.0], + [4.1, 51.1], + [4.0, 51.1], + [4.0, 51.0], + ] + ], + }, + "properties_json": {"source": "unit-test-fixture"}, + } + ], + }, + ) + + detections = [item for item in db.added if isinstance(item, Detection)] + assert result.status == "success" + assert result.detection_count == 1 + assert detections[0].project_id == project_id + assert detections[0].dataset_id == dataset_id + assert detections[0].analysis_run_id == result.analysis_run_id + assert detections[0].model_name == "manual-fixture-detector" + assert detections[0].class_name == "building" + assert detections[0].confidence == 0.92 + assert detections[0].bbox_json == {"x_min": 1, "y_min": 2, "x_max": 10, "y_max": 12} + + +def test_detection_models_api_uses_envelope() -> None: + response = TestClient(app).get("/api/v1/detection/models") + + assert response.status_code == 200 + assert "data" in response.json() + assert {model["model_id"] for model in response.json()["data"]["models"]} == { + "yolo-placeholder", + "yolo-configured", + "manual-fixture-detector", + } + + +def test_sprint8_migration_declares_detection_foundation() -> None: + migration_path = Path(__file__).parents[1] / "alembic" / "versions" / "202606120800_sprint8_detection_foundation.py" + migration_text = migration_path.read_text(encoding="utf-8") + + for required_text in ( + "detections", + "analysis_runs", + "dataset_id", + "job_id", + "model_name", + "model_version", + "result_json", + "ix_detections_project_id", + "ix_detections_dataset_id", + "ix_detections_analysis_run_id", + "ix_detections_class_name", + "ix_detections_geometry", + 'postgresql_using="gist"', + ): + assert required_text in migration_text diff --git a/backend/tests/test_sprint8b_yolo_foundation.py b/backend/tests/test_sprint8b_yolo_foundation.py new file mode 100644 index 00000000..7eb03409 --- /dev/null +++ b/backend/tests/test_sprint8b_yolo_foundation.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import json +from pathlib import Path +from uuid import uuid4 + +import pytest + +from app.core.config import Settings +from app.models import AnalysisRun, Dataset, Detection, Job, Project +from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon +from app.services.detection_service import DetectionService +from app.services.model_registry_service import ModelRegistryService + + +class FakeSession: + def __init__(self, objects=None) -> None: + self.objects = objects or {} + self.added = [] + self.commits = 0 + self.refreshes = [] + + def get(self, model, item_id): + return self.objects.get((model, item_id)) + + def add(self, item) -> None: + self.added.append(item) + if getattr(item, "id", None) is not None: + self.objects[(item.__class__, item.id)] = item + + def commit(self) -> None: + self.commits += 1 + + def refresh(self, item) -> None: + self.refreshes.append(item) + + +class AvailableAdapter: + @staticmethod + def dependencies_available() -> bool: + return True + + +class MissingDependencyAdapter: + @staticmethod + def dependencies_available() -> bool: + return False + + +class MockYoloAdapter: + def __init__(self, settings: Settings) -> None: + self.settings = settings + self.loaded_model_path: Path | None = None + + @staticmethod + def dependencies_available() -> bool: + return True + + def load_model(self, model_path: Path): + self.loaded_model_path = model_path + return object() + + def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]: + assert tile_path.name == "tile_0000.tif" + assert confidence_threshold == 0.5 + return [ + { + "class_name": "building", + "confidence": 0.91, + "bbox": [10.0, 20.0, 30.0, 40.0], + "properties": {"adapter": "mock"}, + } + ] + + +def _project_and_dataset(dataset_type: str = "raster"): + project_id = uuid4() + dataset_id = uuid4() + project = Project(id=project_id, name="Geel") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="source.tif", + dataset_type=dataset_type, + source="user_upload", + storage_path="storage/uploads/source.tif", + ) + db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) + return db, project_id, dataset_id + + +def _settings(tmp_path: Path, **overrides) -> Settings: + model_path = tmp_path / "model.pt" + values = { + "yolo_enabled": True, + "yolo_model_path": str(model_path), + "yolo_max_tiles": 4, + } + values.update(overrides) + return Settings(**values) + + +def _manifest(tmp_path: Path, tile_count: int = 1) -> Path: + tiles = [] + for index in range(tile_count): + tile_path = tmp_path / f"tile_{index:04d}.tif" + tile_path.write_bytes(b"fixture") + tiles.append( + { + "path": str(tile_path), + "pixel_window": [0, 0, 100, 100], + "bounds": [4.0, 51.0, 5.0, 52.0], + "transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01], + "index": index, + } + ) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "tile_set_id": "tiles-fixture", + "source_dataset_id": str(uuid4()), + "source_raster_id": str(uuid4()), + "tile_size": 100, + "overlap": 0, + "count": tile_count, + "tiles": tiles, + } + ), + encoding="utf-8", + ) + return manifest_path + + +def test_yolo_configured_model_reports_not_configured_when_disabled(tmp_path: Path) -> None: + settings = _settings(tmp_path, yolo_enabled=False) + + models = {model.model_id: model for model in ModelRegistryService.list_model_capabilities(settings=settings)} + + assert "yolo-configured" in models + assert models["yolo-configured"].configured is False + assert models["yolo-configured"].status == "not_configured" + + +def test_yolo_configured_model_reports_dependency_unavailable(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"local weights") + settings = _settings(tmp_path, yolo_model_path=str(model_path)) + + model = ModelRegistryService.get_model_capability( + "yolo-configured", + settings=settings, + yolo_adapter_class=MissingDependencyAdapter, + ) + + assert model is not None + assert model.configured is False + assert model.status == "dependency_unavailable" + + +def test_yolo_configured_model_reports_configured_with_local_model_and_dependencies(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"local weights") + settings = _settings(tmp_path, yolo_model_path=str(model_path)) + + model = ModelRegistryService.get_model_capability("yolo-configured", settings=settings, yolo_adapter_class=AvailableAdapter) + + assert model is not None + assert model.configured is True + assert model.status == "configured" + assert model.version == settings.yolo_model_version + + +def test_yolo_run_requires_tile_manifest_path(tmp_path: Path) -> None: + db, project_id, dataset_id = _project_and_dataset() + settings = _settings(tmp_path) + + with pytest.raises(Exception) as exc_info: + DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-configured", + confidence_threshold=0.5, + settings=settings, + yolo_adapter_class=AvailableAdapter, + ) + + assert getattr(exc_info.value, "code", None) == "DETECTION_TILE_MANIFEST_REQUIRED" + + +def test_yolo_run_rejects_manifest_over_tile_limit(tmp_path: Path) -> None: + db, project_id, dataset_id = _project_and_dataset() + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"local weights") + settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1) + manifest_path = _manifest(tmp_path, tile_count=2) + + result = DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-configured", + confidence_threshold=0.5, + tile_manifest_path=str(manifest_path), + settings=settings, + yolo_adapter_class=MockYoloAdapter, + ) + + assert result.status == "failed" + assert result.error_code == "DETECTION_TILE_LIMIT_EXCEEDED" + + +def test_yolo_run_rejects_missing_tile_manifest_file(tmp_path: Path) -> None: + db, project_id, dataset_id = _project_and_dataset() + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"local weights") + settings = _settings(tmp_path, yolo_model_path=str(model_path)) + + result = DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-configured", + confidence_threshold=0.5, + tile_manifest_path=str(tmp_path / "missing-manifest.json"), + settings=settings, + yolo_adapter_class=MockYoloAdapter, + ) + + assert result.status == "failed" + assert result.error_code == "DETECTION_TILE_MANIFEST_NOT_FOUND" + + +def test_yolo_run_rejects_invalid_tile_manifest_json(tmp_path: Path) -> None: + db, project_id, dataset_id = _project_and_dataset() + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"local weights") + settings = _settings(tmp_path, yolo_model_path=str(model_path)) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text("{not-json", encoding="utf-8") + + result = DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-configured", + confidence_threshold=0.5, + tile_manifest_path=str(manifest_path), + settings=settings, + yolo_adapter_class=MockYoloAdapter, + ) + + assert result.status == "failed" + assert result.error_code == "DETECTION_TILE_MANIFEST_INVALID" + + +def test_pixel_bbox_to_epsg4326_polygon_from_gdal_transform() -> None: + polygon = pixel_bbox_to_epsg4326_polygon( + bbox=[10, 20, 30, 40], + tile={ + "transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01], + "bounds": [4.0, 51.0, 5.0, 52.0], + }, + crs="EPSG:4326", + ) + + assert polygon.bounds == pytest.approx((4.1, 51.6, 4.3, 51.8)) + + +def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> None: + db, project_id, dataset_id = _project_and_dataset() + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"local weights") + settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test") + manifest_path = _manifest(tmp_path, tile_count=1) + + result = DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-configured", + confidence_threshold=0.5, + class_filter=["building"], + tile_manifest_path=str(manifest_path), + settings=settings, + yolo_adapter_class=MockYoloAdapter, + ) + + detections = [item for item in db.added if isinstance(item, Detection)] + runs = [item for item in db.added if isinstance(item, AnalysisRun)] + jobs = [item for item in db.added if isinstance(item, Job)] + + assert result.status == "success" + assert result.detection_count == 1 + assert detections[0].model_name == "yolo-configured" + assert detections[0].model_version == "local-test" + assert detections[0].class_name == "building" + assert detections[0].confidence == 0.91 + assert detections[0].source_tile_path.endswith("tile_0000.tif") + assert detections[0].bbox_json == {"x_min": 10.0, "y_min": 20.0, "x_max": 30.0, "y_max": 40.0} + assert detections[0].properties_json == {"adapter": "mock", "tile_index": 0} + assert runs[0].status == "success" + assert jobs[0].status == "success" diff --git a/backend/tests/test_sprint8c_detection_visualization_qa.py b/backend/tests/test_sprint8c_detection_visualization_qa.py new file mode 100644 index 00000000..94982b28 --- /dev/null +++ b/backend/tests/test_sprint8c_detection_visualization_qa.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import json +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from geoalchemy2.shape import from_shape +from shapely.geometry import box + +from app.main import app +from app.db.session import get_db +from app.models import AnalysisRun, Dataset, Detection, Job, Metric, Project, QualityCheck, VectorFeature +from app.services.detection_service import DetectionService + + +class FakeQuery: + def __init__(self, rows): + self.rows = list(rows) + + def filter(self, *criteria): + for criterion in criteria: + left = getattr(criterion, "left", None) + right = getattr(criterion, "right", None) + operator = getattr(criterion, "operator", None) + name = getattr(left, "name", None) + value = getattr(right, "value", right) + if name and operator: + if operator.__name__ == "eq": + self.rows = [row for row in self.rows if getattr(row, name) == value] + elif operator.__name__ == "ge": + self.rows = [row for row in self.rows if getattr(row, name) >= value] + return self + + def order_by(self, *_args): + return self + + def all(self): + return list(self.rows) + + def first(self): + return self.rows[0] if self.rows else None + + +class FakeSession: + def __init__(self, objects=None, query_rows=None) -> None: + self.objects = objects or {} + self.query_rows = query_rows or {} + self.added = [] + self.commits = 0 + self.refreshes = [] + + def get(self, model, item_id): + return self.objects.get((model, item_id)) + + def query(self, model): + return FakeQuery(self.query_rows.get(model, [])) + + def add(self, item) -> None: + self.added.append(item) + if getattr(item, "id", None) is not None: + self.objects[(item.__class__, item.id)] = item + + def commit(self) -> None: + self.commits += 1 + + def refresh(self, item) -> None: + self.refreshes.append(item) + + +def _detection(project_id, dataset_id, analysis_run_id, class_name="building", confidence=0.91, geom=None): + return Detection( + id=uuid4(), + project_id=project_id, + dataset_id=dataset_id, + analysis_run_id=analysis_run_id, + job_id=uuid4(), + model_name="yolo-configured", + model_version="local-test", + class_name=class_name, + confidence=confidence, + geometry=from_shape(geom or box(4.0, 51.0, 4.1, 51.1), srid=4326), + bbox_json={"x_min": 1, "y_min": 2, "x_max": 10, "y_max": 12}, + source_tile_path="storage/tiles/tile_0000.tif", + ) + + +def test_detection_geojson_feature_collection_shape() -> None: + project_id = uuid4() + dataset_id = uuid4() + analysis_run_id = uuid4() + detection = _detection(project_id, dataset_id, analysis_run_id) + db = FakeSession( + objects={(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, analysis_type="detection", status="success", parameters_json={})}, + query_rows={Detection: [detection]}, + ) + + feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id) + + assert feature_collection["type"] == "FeatureCollection" + assert len(feature_collection["features"]) == 1 + feature = feature_collection["features"][0] + assert feature["geometry"]["type"] == "Polygon" + assert feature["properties"]["detection_id"] == str(detection.id) + assert feature["properties"]["class_name"] == "building" + assert feature["properties"]["confidence"] == 0.91 + assert feature["properties"]["model_name"] == "yolo-configured" + assert feature["properties"]["analysis_run_id"] == str(analysis_run_id) + assert feature["properties"]["dataset_id"] == str(dataset_id) + assert feature["properties"]["job_id"] == str(detection.job_id) + assert feature["properties"]["source_tile_path"] == "storage/tiles/tile_0000.tif" + assert feature["properties"]["bbox_json"] == {"x_min": 1, "y_min": 2, "x_max": 10, "y_max": 12} + + +def test_detection_list_filters_by_dataset_class_and_confidence() -> None: + project_id = uuid4() + dataset_id = uuid4() + other_dataset_id = uuid4() + analysis_run_id = uuid4() + rows = [ + _detection(project_id, dataset_id, analysis_run_id, "building", 0.91), + _detection(project_id, dataset_id, analysis_run_id, "road", 0.95), + _detection(project_id, dataset_id, analysis_run_id, "building", 0.25), + _detection(project_id, other_dataset_id, analysis_run_id, "building", 0.99), + ] + db = FakeSession( + objects={(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, analysis_type="detection", status="success", parameters_json={})}, + query_rows={Detection: rows}, + ) + + result = DetectionService.list_detections( + db, + analysis_run_id=analysis_run_id, + dataset_id=dataset_id, + class_name="building", + min_confidence=0.5, + ) + + assert result.total == 1 + assert result.items[0].class_name == "building" + assert result.items[0].confidence == 0.91 + + +def test_detection_detail_returns_one_detection() -> None: + project_id = uuid4() + dataset_id = uuid4() + analysis_run_id = uuid4() + detection = _detection(project_id, dataset_id, analysis_run_id) + db = FakeSession(objects={(Detection, detection.id): detection}) + + result = DetectionService.get_detection(db, detection.id) + + assert result.id == detection.id + assert result.class_name == "building" + + +def test_detection_models_api_envelope_still_canonical() -> None: + response = TestClient(app).get("/api/v1/detection/models") + + assert response.status_code == 200 + assert "data" in response.json() + assert "models" in response.json()["data"] + + +def test_detection_geojson_api_uses_canonical_envelope(monkeypatch) -> None: + analysis_run_id = uuid4() + + monkeypatch.setattr( + "app.api.routes.detection.DetectionService.detections_to_geojson", + lambda *_args, **_kwargs: {"type": "FeatureCollection", "features": []}, + ) + app.dependency_overrides[get_db] = lambda: FakeSession() + try: + response = TestClient(app).get(f"/api/v1/detection/runs/{analysis_run_id}/geojson") + finally: + app.dependency_overrides.pop(get_db, None) + + assert response.status_code == 200 + assert response.json() == {"data": {"type": "FeatureCollection", "features": []}} + + +def test_detection_qa_persists_quality_check_and_metrics() -> None: + project_id = uuid4() + dataset_id = uuid4() + reference_dataset_id = uuid4() + analysis_run_id = uuid4() + detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1)) + reference_dataset = Dataset( + id=reference_dataset_id, + project_id=project_id, + name="reference.geojson", + dataset_type="vector", + source="manual", + dataset_role="reference", + ) + reference_feature = VectorFeature( + id=uuid4(), + dataset_id=reference_dataset_id, + feature_class="building", + geometry=from_shape(box(0, 0, 1, 1), srid=4326), + ) + db = FakeSession( + objects={ + (AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="detection", status="success", parameters_json={}), + (Dataset, reference_dataset_id): reference_dataset, + }, + query_rows={Detection: [detection], VectorFeature: [reference_feature]}, + ) + + result = DetectionService.compare_detections_with_reference( + db=db, + analysis_run_id=analysis_run_id, + reference_dataset_id=reference_dataset_id, + iou_threshold=0.5, + ) + + quality_checks = [item for item in db.added if isinstance(item, QualityCheck)] + metrics = [item for item in db.added if isinstance(item, Metric)] + assert result["matches"] == 1 + assert result["precision"] == 1.0 + assert result["recall"] == 1.0 + assert result["f1_score"] == 1.0 + assert result["quality_check_id"] == str(quality_checks[0].id) + assert quality_checks[0].analysis_run_id == analysis_run_id + assert quality_checks[0].reference_dataset_id == reference_dataset_id + assert [metric.metric_key for metric in metrics] == [ + "precision", + "recall", + "f1", + "mean_iou", + "false_positive_count", + "false_negative_count", + ] + + +def test_detection_qa_no_match_case_persists_zero_scores() -> None: + project_id = uuid4() + dataset_id = uuid4() + reference_dataset_id = uuid4() + analysis_run_id = uuid4() + detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1)) + reference_dataset = Dataset( + id=reference_dataset_id, + project_id=project_id, + name="reference.geojson", + dataset_type="vector", + source="manual", + dataset_role="reference", + ) + reference_feature = VectorFeature( + id=uuid4(), + dataset_id=reference_dataset_id, + feature_class="building", + geometry=from_shape(box(10, 10, 11, 11), srid=4326), + ) + db = FakeSession( + objects={ + (AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="detection", status="success", parameters_json={}), + (Dataset, reference_dataset_id): reference_dataset, + }, + query_rows={Detection: [detection], VectorFeature: [reference_feature]}, + ) + + result = DetectionService.compare_detections_with_reference( + db=db, + analysis_run_id=analysis_run_id, + reference_dataset_id=reference_dataset_id, + iou_threshold=0.5, + ) + + assert result["matches"] == 0 + assert result["false_positives"] == 1 + assert result["false_negatives"] == 1 + assert result["precision"] == 0.0 + assert result["recall"] == 0.0 + assert result["f1_score"] == 0.0 diff --git a/backend/tests/test_sprint9_segmentation_foundation.py b/backend/tests/test_sprint9_segmentation_foundation.py new file mode 100644 index 00000000..3959f20f --- /dev/null +++ b/backend/tests/test_sprint9_segmentation_foundation.py @@ -0,0 +1,418 @@ +from __future__ import annotations + +from pathlib import Path +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from geoalchemy2.shape import from_shape +from shapely.geometry import MultiPolygon, box, mapping + +from app.db.session import get_db +from app.main import app +from app.models import AnalysisRun, Dataset, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature +from app.services.model_registry_service import ModelRegistryService +from app.services.segmentation_service import SegmentationService + + +class FakeQuery: + def __init__(self, rows): + self.rows = list(rows) + + def filter(self, *criteria): + for criterion in criteria: + left = getattr(criterion, "left", None) + right = getattr(criterion, "right", None) + operator = getattr(criterion, "operator", None) + name = getattr(left, "name", None) + value = getattr(right, "value", right) + if name and operator: + if operator.__name__ == "eq": + self.rows = [row for row in self.rows if getattr(row, name) == value] + elif operator.__name__ == "ge": + self.rows = [row for row in self.rows if getattr(row, name) >= value] + return self + + def order_by(self, *_args): + return self + + def all(self): + return list(self.rows) + + +class FakeSession: + def __init__(self, objects=None, query_rows=None) -> None: + self.objects = objects or {} + self.query_rows = query_rows or {} + self.added = [] + self.commits = 0 + self.refreshes = [] + + def get(self, model, item_id): + return self.objects.get((model, item_id)) + + def query(self, model): + return FakeQuery(self.query_rows.get(model, [])) + + def add(self, item) -> None: + self.added.append(item) + if getattr(item, "id", None) is not None: + self.objects[(item.__class__, item.id)] = item + + def commit(self) -> None: + self.commits += 1 + + def refresh(self, item) -> None: + self.refreshes.append(item) + + +def _project_and_dataset(dataset_type: str = "raster"): + project_id = uuid4() + dataset_id = uuid4() + project = Project(id=project_id, name="Geel") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="source.tif", + dataset_type=dataset_type, + source="user_upload", + storage_path="storage/uploads/source.tif", + ) + db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) + return db, project_id, dataset_id + + +def _segmentation(project_id, dataset_id, analysis_run_id, class_name="vegetation", confidence=0.81, geom=None): + segmentation_id = uuid4() + return Segmentation( + id=segmentation_id, + project_id=project_id, + dataset_id=dataset_id, + analysis_run_id=analysis_run_id, + job_id=uuid4(), + model_name="fixture-segmenter", + model_version="fixture-v1", + class_name=class_name, + confidence=confidence, + geometry=from_shape(geom or MultiPolygon([box(4.0, 51.0, 4.1, 51.1)]), srid=4326), + bbox_json={"min_x": 4.0, "min_y": 51.0, "max_x": 4.1, "max_y": 51.1}, + area_m2=123.4, + mask_path=f"storage/masks/{project_id}/{analysis_run_id}/tile_0/mask_{segmentation_id}.png", + source_tile_path="storage/tiles/tile_0000.tif", + tile_index=0, + properties_json={"source": "unit-test-fixture"}, + provenance_json={"fixture_mode": True}, + ) + + +def test_model_registry_returns_segmentation_states() -> None: + models = {model.model_id: model for model in ModelRegistryService.list_model_capabilities(task_type="segmentation")} + + assert set(models) == {"segmentation-placeholder", "fixture-segmenter", "yolo-seg-configured", "sam-configured"} + assert models["segmentation-placeholder"].task_type == "segmentation" + assert models["segmentation-placeholder"].status == "not_configured" + assert models["fixture-segmenter"].configured is True + assert "fixture" in models["fixture-segmenter"].limitation_message.lower() + assert models["yolo-seg-configured"].status == "not_configured" + assert models["sam-configured"].status == "not_configured" + + +def test_unavailable_segmentation_model_creates_failed_run_without_segmentations() -> None: + db, project_id, dataset_id = _project_and_dataset() + + result = SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="segmentation-placeholder", + confidence_threshold=0.5, + ) + + runs = [item for item in db.added if isinstance(item, AnalysisRun)] + jobs = [item for item in db.added if isinstance(item, Job)] + segmentations = [item for item in db.added if isinstance(item, Segmentation)] + + assert result.status == "failed" + assert result.error_code == "SEGMENTATION_MODEL_UNAVAILABLE" + assert result.segmentation_count == 0 + assert runs[0].analysis_type == "segmentation" + assert runs[0].status == "failed" + assert jobs[0].job_type == "segmentation.run" + assert jobs[0].status == "failed" + assert segmentations == [] + + +def test_fixture_segmenter_requires_explicit_mode_and_persists_segmentations() -> None: + db, project_id, dataset_id = _project_and_dataset() + + with pytest.raises(Exception) as exc_info: + SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="fixture-segmenter", + confidence_threshold=0.5, + parameters_json={"fixture_segmentations": []}, + ) + assert getattr(exc_info.value, "code", None) == "FIXTURE_MODE_REQUIRED" + + geometry = mapping(box(4.0, 51.0, 4.1, 51.1)) + result = SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="fixture-segmenter", + confidence_threshold=0.5, + class_filter=["vegetation"], + parameters_json={ + "fixture_mode": True, + "fixture_segmentations": [ + { + "class_name": "vegetation", + "confidence": 0.88, + "geometry": geometry, + "bbox_json": {"min_x": 4.0, "min_y": 51.0, "max_x": 4.1, "max_y": 51.1}, + "source_tile_path": "storage/tiles/tile_0000.tif", + "tile_index": 0, + "properties_json": {"source": "unit-test-fixture"}, + "provenance_json": {"crs": "EPSG:4326"}, + } + ], + }, + ) + + segmentations = [item for item in db.added if isinstance(item, Segmentation)] + assert result.status == "success" + assert result.segmentation_count == 1 + assert segmentations[0].project_id == project_id + assert segmentations[0].dataset_id == dataset_id + assert segmentations[0].analysis_run_id == result.analysis_run_id + assert segmentations[0].model_name == "fixture-segmenter" + assert segmentations[0].class_name == "vegetation" + assert segmentations[0].confidence == 0.88 + assert segmentations[0].mask_path.endswith(f"mask_{segmentations[0].id}.png") + + +def test_non_raster_dataset_request_is_rejected() -> None: + db, project_id, dataset_id = _project_and_dataset(dataset_type="vector") + + with pytest.raises(Exception) as exc_info: + SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="segmentation-placeholder", + confidence_threshold=0.5, + ) + + assert getattr(exc_info.value, "code", None) == "INVALID_DATASET_TYPE" + + +def test_invalid_empty_fixture_geometry_is_rejected() -> None: + db, project_id, dataset_id = _project_and_dataset() + + with pytest.raises(Exception) as exc_info: + SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="fixture-segmenter", + confidence_threshold=0.5, + parameters_json={ + "fixture_mode": True, + "fixture_segmentations": [ + {"class_name": "vegetation", "confidence": 0.9, "geometry": {"type": "Polygon", "coordinates": []}}, + ], + }, + ) + + assert getattr(exc_info.value, "code", None) == "INVALID_FIXTURE_GEOMETRY" + + +def test_segmentation_geojson_feature_collection_shape_and_provenance() -> None: + project_id = uuid4() + dataset_id = uuid4() + analysis_run_id = uuid4() + segmentation = _segmentation(project_id, dataset_id, analysis_run_id) + db = FakeSession( + objects={(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, analysis_type="segmentation", status="success", parameters_json={})}, + query_rows={Segmentation: [segmentation]}, + ) + + feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id) + + assert feature_collection["type"] == "FeatureCollection" + feature = feature_collection["features"][0] + assert feature["geometry"]["type"] == "MultiPolygon" + assert feature["properties"]["segmentation_id"] == str(segmentation.id) + assert feature["properties"]["class_name"] == "vegetation" + assert feature["properties"]["confidence"] == 0.81 + assert feature["properties"]["area_m2"] == 123.4 + assert feature["properties"]["model_name"] == "fixture-segmenter" + assert feature["properties"]["analysis_run_id"] == str(analysis_run_id) + assert feature["properties"]["dataset_id"] == str(dataset_id) + assert feature["properties"]["job_id"] == str(segmentation.job_id) + assert feature["properties"]["source_tile_path"] == "storage/tiles/tile_0000.tif" + assert feature["properties"]["tile_index"] == 0 + assert feature["properties"]["mask_path"] == segmentation.mask_path + assert feature["properties"]["bbox_json"] == segmentation.bbox_json + assert feature["properties"]["provenance_json"] == {"fixture_mode": True} + + +def test_segmentation_qa_persists_quality_check_and_metrics() -> None: + project_id = uuid4() + dataset_id = uuid4() + reference_dataset_id = uuid4() + analysis_run_id = uuid4() + segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1)) + reference_dataset = Dataset( + id=reference_dataset_id, + project_id=project_id, + name="reference.geojson", + dataset_type="vector", + source="manual", + dataset_role="reference", + ) + reference_feature = VectorFeature( + id=uuid4(), + dataset_id=reference_dataset_id, + feature_class="vegetation", + geometry=from_shape(box(0, 0, 1, 1), srid=4326), + ) + db = FakeSession( + objects={ + (AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}), + (Dataset, reference_dataset_id): reference_dataset, + }, + query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]}, + ) + + result = SegmentationService.compare_segmentations_with_reference( + db=db, + analysis_run_id=analysis_run_id, + reference_dataset_id=reference_dataset_id, + iou_threshold=0.5, + ) + + quality_checks = [item for item in db.added if isinstance(item, QualityCheck)] + metrics = [item for item in db.added if isinstance(item, Metric)] + assert result["matches"] == 1 + assert result["precision"] == 1.0 + assert result["recall"] == 1.0 + assert result["f1_score"] == 1.0 + assert quality_checks[0].check_type == "segmentations_vs_reference" + assert quality_checks[0].analysis_run_id == analysis_run_id + assert [metric.metric_key for metric in metrics] == [ + "precision", + "recall", + "f1", + "mean_iou", + "false_positive_count", + "false_negative_count", + ] + + +def test_segmentation_qa_no_match_case_persists_zero_scores() -> None: + project_id = uuid4() + dataset_id = uuid4() + reference_dataset_id = uuid4() + analysis_run_id = uuid4() + segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1)) + reference_dataset = Dataset( + id=reference_dataset_id, + project_id=project_id, + name="reference.geojson", + dataset_type="vector", + source="manual", + dataset_role="reference", + ) + reference_feature = VectorFeature( + id=uuid4(), + dataset_id=reference_dataset_id, + feature_class="vegetation", + geometry=from_shape(box(10, 10, 11, 11), srid=4326), + ) + db = FakeSession( + objects={ + (AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}), + (Dataset, reference_dataset_id): reference_dataset, + }, + query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]}, + ) + + result = SegmentationService.compare_segmentations_with_reference( + db=db, + analysis_run_id=analysis_run_id, + reference_dataset_id=reference_dataset_id, + iou_threshold=0.5, + ) + + assert result["matches"] == 0 + assert result["false_positives"] == 1 + assert result["false_negatives"] == 1 + assert result["precision"] == 0.0 + assert result["recall"] == 0.0 + assert result["f1_score"] == 0.0 + + +def test_segmentation_models_api_uses_envelope() -> None: + response = TestClient(app).get("/api/v1/segmentation/models") + + assert response.status_code == 200 + assert "data" in response.json() + assert {model["model_id"] for model in response.json()["data"]["models"]} == { + "segmentation-placeholder", + "fixture-segmenter", + "yolo-seg-configured", + "sam-configured", + } + + +def test_segmentation_geojson_api_uses_canonical_envelope(monkeypatch) -> None: + analysis_run_id = uuid4() + + monkeypatch.setattr( + "app.api.routes.segmentation.SegmentationService.segmentations_to_geojson", + lambda *_args, **_kwargs: {"type": "FeatureCollection", "features": []}, + ) + app.dependency_overrides[get_db] = lambda: FakeSession() + try: + response = TestClient(app).get(f"/api/v1/segmentation/runs/{analysis_run_id}/geojson") + finally: + app.dependency_overrides.pop(get_db, None) + + assert response.status_code == 200 + assert response.json() == {"data": {"type": "FeatureCollection", "features": []}} + + +def test_sprint9_migration_declares_segmentation_foundation() -> None: + migration_path = Path(__file__).parents[1] / "alembic" / "versions" / "202606120900_sprint9_segmentation_foundation.py" + migration_text = migration_path.read_text(encoding="utf-8") + + for required_text in ( + "segmentations", + "project_id", + "dataset_id", + "job_id", + "analysis_run_id", + "model_name", + "model_version", + "class_name", + "confidence", + "MultiPolygon", + "bbox_json", + "area_m2", + "mask_path", + "source_tile_path", + "tile_index", + "properties_json", + "provenance_json", + "ix_segmentations_project_id", + "ix_segmentations_dataset_id", + "ix_segmentations_analysis_run_id", + "ix_segmentations_job_id", + "ix_segmentations_class_name", + "ix_segmentations_geometry", + 'postgresql_using="gist"', + ): + assert required_text in migration_text diff --git a/backend/tests/test_storage_service.py b/backend/tests/test_storage_service.py new file mode 100644 index 00000000..de289492 --- /dev/null +++ b/backend/tests/test_storage_service.py @@ -0,0 +1,28 @@ +from pathlib import Path +from types import SimpleNamespace + +from app.services.storage_service import StorageService + + +def test_persist_dataset_file_records_metadata(monkeypatch, tmp_path) -> None: + monkeypatch.setattr( + "app.services.storage_service.get_settings", + lambda: SimpleNamespace(storage_root=str(tmp_path)), + ) + + metadata = StorageService.persist_dataset_file( + project_id="project-123", + dataset_id="dataset-456", + dataset_type="vector", + original_filename="../weird name!@#.geojson", + content=b"example-bytes", + content_type="application/geo+json", + ) + + assert metadata["original_filename"] == "weird name___.geojson" + assert metadata["stored_filename"] == "dataset-456_weird name___.geojson" + assert metadata["content_type"] == "application/geo+json" + assert metadata["size_bytes"] == 13 + assert len(metadata["checksum_sha256"]) == 64 + assert Path(metadata["storage_path"]).exists() + assert str(Path(tmp_path, "uploads", "project-123", "vector", "dataset-456")) in metadata["storage_path"] diff --git a/backend/tests/test_vector_operations_service.py b/backend/tests/test_vector_operations_service.py new file mode 100644 index 00000000..1f7fd42e --- /dev/null +++ b/backend/tests/test_vector_operations_service.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace +from uuid import UUID, uuid4 + +from app.core.errors import AppError +from app.models import Dataset +from app.services.storage_service import StorageService +from app.services.vector_operations_service import VectorOperationsService + + +class FakeSession: + def __init__(self, datasets): + self.datasets = {dataset.id: dataset for dataset in datasets} + self.added = [] + + def get(self, model, item_id): + return self.datasets.get(item_id) + + def add(self, value): + self.added.append(value) + + def commit(self): + return None + + def refresh(self, _value): + return None + + +def _make_vector_dataset(dataset_id: UUID, project_id: UUID, raw: str) -> Dataset: + path = Path(f"./tests/.tmp_{dataset_id}.geojson") + path.write_text(raw, encoding="utf-8") + return Dataset( + id=dataset_id, + project_id=project_id, + name=f"{dataset_id}.geojson", + dataset_type="vector", + source="test", + storage_path=str(path), + original_filename=f"{dataset_id}.geojson", + stored_filename=f"{dataset_id}.geojson", + content_type="application/geo+json", + ) + + +def test_vector_inspect_extracts_feature_count_and_bbox(tmp_path) -> None: + dataset_id = uuid4() + project_id = uuid4() + source_path = tmp_path / f"{dataset_id}.geojson" + source_path.write_text( + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[4.1,51.2]}},{"type":"Feature","geometry":{"type":"Point","coordinates":[4.2,51.3]}}]}', + encoding="utf-8", + ) + + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="vector.geojson", + dataset_type="vector", + source="test", + storage_path=str(source_path), + original_filename="vector.geojson", + stored_filename="vector.geojson", + content_type="application/geo+json", + ) + db = FakeSession([dataset]) + + payload = VectorOperationsService.inspect(db, dataset_id) + + assert payload.feature_count == 2 + assert payload.geometry_type_summary["Point"] == 2 + assert payload.bounds_json == {"min_x": 4.1, "min_y": 51.2, "max_x": 4.2, "max_y": 51.3} + + +def test_vector_bbox_and_stats_share_summary(tmp_path) -> None: + dataset_id = uuid4() + project_id = uuid4() + source_path = tmp_path / f"{dataset_id}.geojson" + source_path.write_text( + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[4.0,51.0]}}]}', + encoding="utf-8", + ) + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="vector.geojson", + dataset_type="vector", + source="test", + storage_path=str(source_path), + original_filename="vector.geojson", + stored_filename="vector.geojson", + content_type="application/geo+json", + ) + db = FakeSession([dataset]) + + bbox = VectorOperationsService.bbox(db, dataset_id) + stats = VectorOperationsService.stats(db, dataset_id) + + assert bbox["feature_count"] == 1 + assert stats["feature_count"] == 1 + assert stats["geometry_type_summary"]["Point"] == 1 + + +def test_vector_intersect_creates_derived_dataset(monkeypatch, tmp_path) -> None: + source_id = uuid4() + target_id = uuid4() + project_id = uuid4() + source_path = tmp_path / f"{source_id}.geojson" + target_path = tmp_path / f"{target_id}.geojson" + source_path.write_text( + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[4.1,51.2]}}]}', + encoding="utf-8", + ) + target_path.write_text( + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[4.0,51.0],[4.0,52.0],[5.0,52.0],[5.0,51.0],[4.0,51.0]]]}}]}', + encoding="utf-8", + ) + + source = Dataset( + id=source_id, + project_id=project_id, + name="source.geojson", + dataset_type="vector", + source="test", + storage_path=str(source_path), + original_filename="source.geojson", + stored_filename="source.geojson", + content_type="application/geo+json", + ) + target = Dataset( + id=target_id, + project_id=project_id, + name="target.geojson", + dataset_type="vector", + source="test", + storage_path=str(target_path), + original_filename="target.geojson", + stored_filename="target.geojson", + content_type="application/geo+json", + ) + db = FakeSession([source, target]) + + persisted = {} + + def _persist_dataset_file(project_id: str, dataset_id: str, dataset_type: str, original_filename: str, content: bytes, content_type: str | None): + persisted["project_id"] = project_id + persisted["dataset_id"] = dataset_id + output = tmp_path / f"{dataset_id}_{dataset_type}.geojson" + output.write_bytes(content) + return { + "original_filename": original_filename, + "stored_filename": output.name, + "content_type": content_type or "application/geo+json", + "size_bytes": len(content), + "checksum_sha256": "test", + "storage_path": str(output), + } + + monkeypatch.setattr(StorageService, "persist_dataset_file", _persist_dataset_file) + + derived_id = VectorOperationsService.intersect(db, source_id, target_id, "intersect_output") + assert derived_id is not None + assert isinstance(derived_id, UUID) + assert persisted["project_id"] == str(project_id) + + +def test_vector_operations_reject_invalid_geometry(tmp_path) -> None: + dataset_id = uuid4() + project_id = uuid4() + source_path = tmp_path / f"{dataset_id}.geojson" + source_path.write_text( + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":"invalid"}}]}', + encoding="utf-8", + ) + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="invalid.geojson", + dataset_type="vector", + source="test", + storage_path=str(source_path), + original_filename="invalid.geojson", + stored_filename="invalid.geojson", + content_type="application/geo+json", + ) + db = FakeSession([dataset]) + + try: + VectorOperationsService.inspect(db, dataset_id) + except AppError as exc: + assert exc.code == "INVALID_GEOMETRY" + else: + raise AssertionError("Invalid geometry should raise INVALID_GEOMETRY") diff --git a/checklists/DAY_1_OPERATOR_CHECKLIST.md b/checklists/DAY_1_OPERATOR_CHECKLIST.md new file mode 100644 index 00000000..71c59791 --- /dev/null +++ b/checklists/DAY_1_OPERATOR_CHECKLIST.md @@ -0,0 +1,28 @@ +# Day 1 Operator Checklist + +Use this checklist while running Codex. + +## Before starting + +- [ ] Extract latest full ZIP. +- [ ] Open repo in development environment. +- [ ] Ensure Docker/Postgres/PostGIS are available if implementation reaches DB stage. +- [ ] Provide Codex with `prompts/codex/day-1/00_START_HERE.md`. + +## During each pass + +- [ ] Codex read required docs. +- [ ] Codex changed only pass-relevant files. +- [ ] Codex ran validation or explained why not. +- [ ] Codex updated execution log. +- [ ] Codex reported changed files. +- [ ] Codex did not silently change architecture. + +## After Day 1 + +- [ ] Backend starts or documented blocker exists. +- [ ] Frontend starts/builds or documented blocker exists. +- [ ] API contracts match actual behavior. +- [ ] TODO reflects reality. +- [ ] CHANGELOG updated. +- [ ] Next pass is clear. diff --git a/checklists/SPRINT_1_OPERATOR_CHECKLIST.md b/checklists/SPRINT_1_OPERATOR_CHECKLIST.md new file mode 100644 index 00000000..1aa7a90a --- /dev/null +++ b/checklists/SPRINT_1_OPERATOR_CHECKLIST.md @@ -0,0 +1,31 @@ +# SPRINT 1 OPERATOR CHECKLIST + +Use this before and after the first Codex implementation run. + +## Before Codex starts + +- [ ] Unzip latest full repo. +- [ ] Open `CODEX_START.md`. +- [ ] Confirm M14 is current milestone. +- [ ] Run `make readiness`. +- [ ] Give Codex `prompts/codex/m14/CODEX_FIRST_DAY_MASTER_PROMPT.md`. +- [ ] Tell Codex to build Sprint 1 only. + +## During Codex run + +- [ ] Reject scope expansion beyond Sprint 1. +- [ ] Require tests or smoke checks after each pass. +- [ ] Ask Codex to record blockers, not work around architecture. +- [ ] Keep GRB/Sentinel/LiDAR out of Sprint 1 implementation. + +## After Codex run + +- [ ] Backend starts. +- [ ] Frontend starts. +- [ ] Database/PostGIS works. +- [ ] Health endpoint works. +- [ ] GeoJSON fixture still validates. +- [ ] Map renders. +- [ ] Dataset metadata workflow works. +- [ ] Changelog/status updated. +- [ ] Next pass is clear. diff --git a/contracts/api/examples/area_create.geojson b/contracts/api/examples/area_create.geojson new file mode 100644 index 00000000..5e9f0f4a --- /dev/null +++ b/contracts/api/examples/area_create.geojson @@ -0,0 +1,8 @@ +{ + "type": "Feature", + "properties": {"name": "Geel demo area"}, + "geometry": { + "type": "Polygon", + "coordinates": [[[4.98,51.16],[4.99,51.16],[4.99,51.17],[4.98,51.17],[4.98,51.16]]] + } +} diff --git a/contracts/api/examples/error_feature_disabled.json b/contracts/api/examples/error_feature_disabled.json new file mode 100644 index 00000000..d04fe74b --- /dev/null +++ b/contracts/api/examples/error_feature_disabled.json @@ -0,0 +1,7 @@ +{ + "error": { + "code": "FEATURE_DISABLED", + "message": "This feature is disabled in the current milestone.", + "details": {"feature": "LIDAR_LAB", "target_milestone": "V4"} + } +} diff --git a/contracts/api/examples/project_create.json b/contracts/api/examples/project_create.json new file mode 100644 index 00000000..3baa5e15 --- /dev/null +++ b/contracts/api/examples/project_create.json @@ -0,0 +1,5 @@ +{ + "name": "Geel Building QA Demo", + "description": "Detect buildings and compare against reference polygons.", + "region": "Kempen" +} diff --git a/contracts/api/examples/qaqc_result.json b/contracts/api/examples/qaqc_result.json new file mode 100644 index 00000000..6efb176f --- /dev/null +++ b/contracts/api/examples/qaqc_result.json @@ -0,0 +1,6 @@ +{ + "analysis_run_id": "demo-qaqc-run", + "metrics": {"precision": 0.92, "recall": 0.88, "f1": 0.90, "mean_iou": 0.71}, + "counts": {"true_positive": 23, "false_positive": 2, "false_negative": 3}, + "layers": {"matches": "exports/demo_matches.geojson", "false_positives": "exports/demo_fp.geojson", "false_negatives": "exports/demo_fn.geojson"} +} diff --git a/contracts/api/response-envelope.md b/contracts/api/response-envelope.md new file mode 100644 index 00000000..8bf79335 --- /dev/null +++ b/contracts/api/response-envelope.md @@ -0,0 +1,53 @@ +# API Response and Error Contract + +## Standard error + +All API errors must use: + +```json +{ + "error": { + "code": "STRING_CODE", + "message": "Human readable message.", + "details": {} + } +} +``` + +## Job response + +Long-running operations return: + +```json +{ + "job_id": "uuid", + "analysis_run_id": "uuid", + "status": "queued", + "status_url": "/analysis-runs/{id}" +} +``` + +## Resource metadata + +Resources should include: + +```json +{ + "id": "uuid", + "created_at": "ISO-8601", + "updated_at": "ISO-8601" +} +``` + +## Pagination + +List endpoints use: + +```json +{ + "items": [], + "total": 0, + "limit": 50, + "offset": 0 +} +``` diff --git a/contracts/database/domain-model.md b/contracts/database/domain-model.md new file mode 100644 index 00000000..c8e3e9e6 --- /dev/null +++ b/contracts/database/domain-model.md @@ -0,0 +1,31 @@ +# Domain Model Contract + +## Core aggregates + +### Project +A project is an investigation workspace. It contains areas, datasets, analyses, and exports. + +### Area +A spatial polygon or selected administrative unit used to clip and analyze datasets. + +### Dataset +A source artifact or external reference. Datasets can be raster, vector, reference, model output, mask, or export. + +### AnalysisRun +A tracked execution of a pipeline such as detection, segmentation, QA, clipping, tiling, or index computation. + +### Detection +A model-derived feature, usually a georeferenced bounding box or polygon. + +### Segmentation +A mask-derived polygon or raster mask output. + +### QualityCheck +A comparison between model outputs and reference data such as GRB. + +## Geometry rules + +- Use SRID 4326 for API-level GeoJSON interchange unless explicitly documented. +- Use projected CRS for area/length calculations when required. +- Store original CRS metadata for every dataset. +- Never calculate area in degrees. diff --git a/contracts/events/event-contracts.md b/contracts/events/event-contracts.md new file mode 100644 index 00000000..fd1cfc46 --- /dev/null +++ b/contracts/events/event-contracts.md @@ -0,0 +1,37 @@ +# Event Contracts + +GeoIntel uses events to document state transitions, even if V1 implements them as database records/log entries rather than a full event bus. + +## Event names + +- ProjectCreated +- AreaCreated +- DatasetUploaded +- DatasetMetadataExtracted +- RasterTiled +- VectorImported +- DetectionQueued +- DetectionStarted +- DetectionCompleted +- DetectionFailed +- SegmentationCompleted +- QualityCheckCompleted +- ExportGenerated + +## Event shape + +```json +{ + "event_id": "uuid", + "event_type": "DatasetUploaded", + "project_id": "uuid", + "dataset_id": "uuid", + "analysis_run_id": null, + "timestamp": "ISO-8601", + "payload": {} +} +``` + +## Implementation rule + +Every long-running workflow must create at least one event at start and one at completion/failure. diff --git a/datasets/.gitkeep b/datasets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/datasets/cache/.gitkeep b/datasets/cache/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/datasets/processed/.gitkeep b/datasets/processed/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/datasets/raw/.gitkeep b/datasets/raw/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/demo/geel/README.md b/demo/geel/README.md new file mode 100644 index 00000000..2fc2fac0 --- /dev/null +++ b/demo/geel/README.md @@ -0,0 +1,6 @@ +# Demo — Geel Building Detection + +Purpose: +Demonstrate the core GeoAI workflow: select area, load imagery/reference data, detect buildings, compare with GRB/fixture reference, export GeoJSON. + +V1 can use small fixtures instead of large real imagery. diff --git a/demo/geel/area_geel_center.geojson b/demo/geel/area_geel_center.geojson new file mode 100644 index 00000000..ff1538db --- /dev/null +++ b/demo/geel/area_geel_center.geojson @@ -0,0 +1,39 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "id": "area_geel_center", + "name": "Geel Centrum Demo" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.98, + 51.16 + ], + [ + 5.0, + 51.16 + ], + [ + 5.0, + 51.18 + ], + [ + 4.98, + 51.18 + ], + [ + 4.98, + 51.16 + ] + ] + ] + } + } + ] +} diff --git a/demo/geel/demo_detections.geojson b/demo/geel/demo_detections.geojson new file mode 100644 index 00000000..40adc421 --- /dev/null +++ b/demo/geel/demo_detections.geojson @@ -0,0 +1,113 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "id": "det_bld_1", + "class": "building", + "confidence": 0.92, + "model_id": "demo-yolo-buildings-v1" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.98505, + 51.16505 + ], + [ + 4.9860500000000005, + 51.16505 + ], + [ + 4.9860500000000005, + 51.16605 + ], + [ + 4.98505, + 51.16605 + ], + [ + 4.98505, + 51.16505 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "det_bld_2", + "class": "building", + "confidence": 0.88, + "model_id": "demo-yolo-buildings-v1" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.99005, + 51.16605 + ], + [ + 4.99105, + 51.16605 + ], + [ + 4.99105, + 51.167049999999996 + ], + [ + 4.99005, + 51.167049999999996 + ], + [ + 4.99005, + 51.16605 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "det_bld_3", + "class": "building", + "confidence": 0.61, + "model_id": "demo-yolo-buildings-v1" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.997, + 51.172 + ], + [ + 4.998, + 51.172 + ], + [ + 4.998, + 51.172999999999995 + ], + [ + 4.997, + 51.172999999999995 + ], + [ + 4.997, + 51.172 + ] + ] + ] + } + } + ] +} diff --git a/demo/geel/expected_qaqc_metrics.json b/demo/geel/expected_qaqc_metrics.json new file mode 100644 index 00000000..be0bffe9 --- /dev/null +++ b/demo/geel/expected_qaqc_metrics.json @@ -0,0 +1,9 @@ +{ + "iou_threshold": 0.5, + "true_positive": 2, + "false_positive": 1, + "false_negative": 1, + "precision": 0.6667, + "recall": 0.6667, + "f1": 0.6667 +} diff --git a/demo/geel/reference_buildings.geojson b/demo/geel/reference_buildings.geojson new file mode 100644 index 00000000..52b32da2 --- /dev/null +++ b/demo/geel/reference_buildings.geojson @@ -0,0 +1,107 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "id": "ref_bld_1", + "class": "building" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.985, + 51.165 + ], + [ + 4.986000000000001, + 51.165 + ], + [ + 4.986000000000001, + 51.166 + ], + [ + 4.985, + 51.166 + ], + [ + 4.985, + 51.165 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "ref_bld_2", + "class": "building" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.99, + 51.166 + ], + [ + 4.9910000000000005, + 51.166 + ], + [ + 4.9910000000000005, + 51.166999999999994 + ], + [ + 4.99, + 51.166999999999994 + ], + [ + 4.99, + 51.166 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "ref_bld_3", + "class": "building" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.995, + 51.17 + ], + [ + 4.996, + 51.17 + ], + [ + 4.996, + 51.171 + ], + [ + 4.995, + 51.171 + ], + [ + 4.995, + 51.17 + ] + ] + ] + } + } + ] +} diff --git a/demo/mol/README.md b/demo/mol/README.md new file mode 100644 index 00000000..4ec58dda --- /dev/null +++ b/demo/mol/README.md @@ -0,0 +1,4 @@ +# Demo — Mol Vegetation / Water Analysis + +Purpose: +Reserved for Sentinel/segmentation workflows in V2. diff --git a/demo/turnhout/README.md b/demo/turnhout/README.md new file mode 100644 index 00000000..6fc03368 --- /dev/null +++ b/demo/turnhout/README.md @@ -0,0 +1,4 @@ +# Demo — Turnhout Change Detection + +Purpose: +Reserved for vector/raster/AI change detection after V1 foundation. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..3c2b05dc --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,54 @@ +services: + db: + image: postgis/postgis:16-3.4 + environment: + POSTGRES_DB: geointel + POSTGRES_USER: geointel + POSTGRES_PASSWORD: geointel + volumes: + - geointel_postgis:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U geointel -d geointel"] + interval: 5s + timeout: 5s + retries: 10 + + backend: + build: + context: ./backend + environment: + DATABASE_URL: postgresql+psycopg://geointel:geointel@db:5432/geointel + STORAGE_ROOT: /app/storage + CORS_ORIGINS: http://localhost:1202,http://127.0.0.1:1202 + ports: + - "8000:8000" + volumes: + - ./storage:/app/storage + command: sh /app/docker_start.sh + depends_on: + db: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5).read()\""] + interval: 10s + timeout: 5s + retries: 12 + start_period: 30s + + frontend: + build: + context: ./frontend + ports: + - "1202:80" + depends_on: + backend: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1/health | grep -q '\"status\"'"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 10s + +volumes: + geointel_postgis: diff --git a/docs/.gitkeep b/docs/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docs/00-start/START_HERE.md b/docs/00-start/START_HERE.md new file mode 100644 index 00000000..6603df23 --- /dev/null +++ b/docs/00-start/START_HERE.md @@ -0,0 +1,122 @@ +# START HERE — GeoIntel Architect Canonical Entry Point + +This is the single canonical entry point for Codex, reviewers and future contributors. +Older handoff files are historical. If documents conflict, follow the precedence order in this file. + +## Current milestone + +**M14 — Build Launch Package** + +The repository is no longer only a documentation bundle. It is now a specification-controlled engineering repo for building GeoIntel Kempen as a GeoAI Workbench. + +## Product one-liner + +GeoIntel Kempen is a GeoAI Workbench for the Belgian Kempen that processes raster data, vector data and AI outputs into geospatially correct detections, segmentations, QA/QC metrics and exports. + +## Non-negotiable product identity + +GeoIntel is: + +- a GeoAI Workbench; +- a geospatial data processing product; +- a platform for raster/vector/AI/QA workflows; +- a portfolio-grade implementation of GIS, remote sensing, computer vision and data engineering. + +GeoIntel is not primarily: + +- a generic dashboard; +- a reporting-only tool; +- a chatbot; +- a QGIS clone; +- a mock demo app. + +## Canonical read order for Codex + +Read these files in order before coding: + +1. `docs/00-start/START_HERE.md` +2. `docs/governance/GEOINTEL_CONSTITUTION.md` +3. `docs/governance/ARCHITECTURE_INVARIANTS.md` +4. `docs/governance/FORBIDDEN_DECISIONS.md` +5. `docs/governance/DECISION_PRECEDENCE.md` +6. `docs/specs/CANONICAL_DOMAIN_MODELS.md` +7. `docs/specs/GIS_STANDARDS.md` +8. `docs/specs/RASTER_STANDARDS.md` +9. `docs/specs/STATE_MACHINES.md` +10. `docs/workflows/GOLDEN_PATHS.md` +11. `docs/build/BUILD_ORDER_DEPENDENCY_GRAPH.md` +12. `docs/build/CODEX_OPERATING_SYSTEM.md` +13. `docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md` +14. `docs/40-build-launch/BUILD_SUCCESS_DEFINITION.md` +15. `docs/40-build-launch/CODEX_STOP_RULES.md` +16. `prompts/codex/m14/CODEX_FIRST_DAY_MASTER_PROMPT.md` + +## Canonical first implementation target + +The first vertical slice is: + +**Project + Area + Dataset + Raster/Vector metadata + Reference polygons + Detection result import + QA/QC + GeoJSON export.** + +Do not start with full AI inference if the foundation is not stable. The first goal is to prove the data lifecycle and geospatial correctness. + +## Canonical V1 golden path + +1. Create project. +2. Create/select an Area in the Kempen. +3. Upload or load a raster/vector dataset. +4. Extract and persist metadata. +5. Load reference polygons, initially demo GRB-like buildings. +6. Import or generate predicted building detections. +7. Convert outputs to valid geospatial features. +8. Run QA/QC against the reference layer. +9. Show results on the map and in metrics panels. +10. Export GeoJSON. + +## What Codex may improve autonomously + +Codex may improve: + +- implementation quality; +- test coverage; +- type safety; +- error handling; +- UI clarity; +- documentation clarity; +- internal helper abstractions; +- performance within defined budgets. + +Codex may not change: + +- product identity; +- core stack; +- CRS policy; +- database choice; +- async job architecture; +- API envelope shape; +- state machine names; +- golden path priority; +- forbidden decisions. + +## Conflict resolution + +If any older document conflicts with this M14 launch layer, follow this order: + +1. Constitution and architecture invariants. +2. Forbidden decisions. +3. State machines and canonical models. +4. API/database contracts. +5. Build order dependency graph. +6. M14 build-launch docs for first-run scope and stop rules. +7. Older milestone handoff documents. + +## Required pass ending + +Every Codex pass must end with: + +- files changed; +- commands run; +- tests passed/failed; +- known limitations; +- whether golden paths still pass; +- whether any architecture invariant was touched; +- next recommended pass. diff --git a/docs/11-quality/REGRESSION_TRAPS.md b/docs/11-quality/REGRESSION_TRAPS.md new file mode 100644 index 00000000..9cac2617 --- /dev/null +++ b/docs/11-quality/REGRESSION_TRAPS.md @@ -0,0 +1,81 @@ +# Regression Traps + +This document lists common failure modes Codex must actively avoid. + +## Geospatial Traps + +### CRS Loss + +Never store or return geometries without CRS context. GeoJSON is usually WGS84 by convention, but source CRS must still be preserved in dataset metadata. + +### Bounding Box Confusion + +Never mix `[minx, miny, maxx, maxy]` with `[west, south, east, north]` without explicitly naming fields. + +### Area Units + +Never compute area in degrees. Reproject to an appropriate projected CRS before area or distance calculations. For Flanders/Kempen, prefer Belgian Lambert 72 / EPSG:31370 for metric calculations unless a stronger reason is documented. + +### Raster/Vector Alignment + +Never compare raster-derived outputs with vector reference layers without documenting resolution, CRS and alignment assumptions. + +### Invalid Geometry + +Always validate polygons. Attempt safe fixes only when documented; otherwise return a validation error. + +## AI Traps + +### Confidence Is Not Accuracy + +Do not present model confidence as accuracy. Accuracy requires comparison against reference or labels. + +### Fixture Detection Is Not Real AI + +When using fixture/stub output, label it clearly as fixture/demo mode. + +### Silent Model Fallback + +Never silently fall back from real model inference to fixture mode. The response must indicate the mode used. + +### Mask Polygonization Noise + +Segmentation polygonization must include simplification/cleanup parameters and preserve original mask path. + +## Backend Traps + +### Long Work in Request Thread + +Do not run heavy raster/AI operations synchronously inside request handlers. Use a job boundary. + +### Inconsistent Status Values + +Use the frozen status enum only: `queued`, `running`, `completed`, `failed`, `cancelled`. + +### File Path Leakage + +API responses may expose logical storage keys or download URLs, not arbitrary host paths. + +## Frontend Traps + +### UI-Only State + +Do not create project, area, dataset or analysis state only in frontend memory. Persist via API. + +### Empty Success Screens + +Every page must distinguish loading, empty, error, ready and completed states. + +### Map Layer Ambiguity + +Every map layer must show source, timestamp, opacity, visibility and legend where applicable. + +## Documentation Traps + +### TODO Instead of Decision + +Do not use TODO comments for architecture gaps. Either implement, document a limitation, or ask for a decision. + +### Contract Drift + +If code changes API responses, update contract docs and examples in the same pass. diff --git a/docs/11-quality/SELF_REVIEW_CHECKLIST.md b/docs/11-quality/SELF_REVIEW_CHECKLIST.md new file mode 100644 index 00000000..692e79a3 --- /dev/null +++ b/docs/11-quality/SELF_REVIEW_CHECKLIST.md @@ -0,0 +1,58 @@ +# Self-Review Checklist for Codex + +Codex must run this checklist before ending every implementation pass. + +## Product Fit + +- [ ] Does the work still support GeoIntel as a GeoAI Workbench? +- [ ] Did the pass avoid adding unrelated dashboard/chat features? +- [ ] Is the Kempen/GRB-first strategy preserved? + +## API and Backend + +- [ ] Are new endpoints documented? +- [ ] Do endpoints return stable JSON shapes? +- [ ] Are errors structured and useful? +- [ ] Are validation failures explicit? +- [ ] Are long-running tasks behind a job/status boundary? + +## Database + +- [ ] Are migrations included? +- [ ] Are geometry columns documented? +- [ ] Are timestamps and provenance fields included? +- [ ] Is source metadata preserved? + +## Geospatial Correctness + +- [ ] Is CRS captured? +- [ ] Are metric calculations done in a projected CRS? +- [ ] Are geometries validated? +- [ ] Are bounds and area units explicit? + +## AI Pipeline Correctness + +- [ ] Is model mode clear: real, stub or fixture? +- [ ] Is confidence not mislabeled as accuracy? +- [ ] Are generated detections georeferenced or explicitly not georeferenced? +- [ ] Are model parameters stored with the run? + +## Frontend + +- [ ] Are loading, empty, error and success states implemented? +- [ ] Does the UI use API data instead of hardcoded business data? +- [ ] Are map layers inspectable? +- [ ] Can users understand what happened after running an analysis? + +## Testing + +- [ ] Were relevant tests added or updated? +- [ ] Do smoke scripts still pass? +- [ ] Are fixtures deterministic? + +## Documentation + +- [ ] Was `CHANGELOG.md` updated? +- [ ] Was `docs/BUILD_STATUS.md` updated? +- [ ] Were affected specs updated? +- [ ] Are known limitations explicit? diff --git a/docs/12-build-control/BUILD_SEQUENCE_LOCK.md b/docs/12-build-control/BUILD_SEQUENCE_LOCK.md new file mode 100644 index 00000000..7c839ec2 --- /dev/null +++ b/docs/12-build-control/BUILD_SEQUENCE_LOCK.md @@ -0,0 +1,147 @@ +# Build Sequence Lock + +This file freezes the order in which Codex should build GeoIntel V1. Codex can split passes into smaller chunks, but it must not reorder major dependencies. + +## Phase 0 — Repository Verification + +- Verify folder structure. +- Verify documentation set exists. +- Verify `.env.example` and docker-compose exist. +- Verify scripts are executable or document how to run them. +- Verify fixtures are valid GeoJSON. + +Exit criteria: + +- all preflight scripts run or have clear remediation notes. + +## Phase 1 — Backend Foundation + +- FastAPI app shell. +- Health endpoint. +- Settings/config loader. +- Structured error response model. +- CORS configured for local frontend. +- Logging baseline. + +Exit criteria: + +- backend imports successfully. +- `/health` returns service status. +- config is read from environment. + +## Phase 2 — Database Foundation + +- SQLAlchemy or SQLModel models. +- Alembic migrations. +- PostGIS extension migration. +- project, area, dataset, analysis_run base tables. +- geometry storage strategy implemented. + +Exit criteria: + +- migrations run against PostGIS. +- seed script creates one project and one area. + +## Phase 3 — Project, Area and Dataset APIs + +- CRUD for projects. +- CRUD for areas. +- dataset upload endpoint. +- metadata extraction queue boundary. +- file storage path convention. + +Exit criteria: + +- OpenAPI docs expose complete endpoints. +- API tests pass for create/list/read flows. + +## Phase 4 — Geospatial Metadata + +- vector metadata extraction. +- raster metadata extraction. +- CRS validation. +- bounds extraction. +- geometry validation. + +Exit criteria: + +- sample GeoJSON returns feature count, bounds and CRS status. +- sample raster placeholder or documented stub returns safe metadata response. + +## Phase 5 — Frontend Shell + +- React app. +- routing. +- layout. +- API client. +- project list. +- project workspace. +- map workspace placeholder. + +Exit criteria: + +- frontend starts. +- health check visible. +- project list loads from API. + +## Phase 6 — Map and Layer Foundation + +- MapLibre map. +- draw/select area. +- layer manager. +- vector layer rendering. +- dataset detail panel. + +Exit criteria: + +- fixture GeoJSON renders on map. +- drawn area can be saved through API. + +## Phase 7 — QA/QC Foundation + +- load predicted and reference polygons. +- compute IoU-based matching. +- compute precision, recall and F1. +- create QA result payload. +- render QA dashboard. + +Exit criteria: + +- fixture QA returns deterministic metrics. +- false positives and false negatives are exported as GeoJSON. + +## Phase 8 — Detection Interface Boundary + +- detection run model. +- detection service interface. +- model registry stub. +- deterministic fixture-based inference fallback. +- later YOLO integration boundary. + +Exit criteria: + +- detection run can produce geospatial detections using fixture mode. +- output is stored and visible as layer. + +## Phase 9 — Export Pipeline + +- export GeoJSON. +- export metrics JSON. +- prepare report shell. +- export provenance metadata. + +Exit criteria: + +- user can download GeoJSON output from UI. + +## Phase 10 — Stabilization + +- smoke tests. +- contract tests. +- docs update. +- changelog. +- known limitations. + +Exit criteria: + +- V1 vertical slice is demonstrable end-to-end. diff --git a/docs/12-build-control/CODEX_DECISION_BOUNDARIES.md b/docs/12-build-control/CODEX_DECISION_BOUNDARIES.md new file mode 100644 index 00000000..347fa78c --- /dev/null +++ b/docs/12-build-control/CODEX_DECISION_BOUNDARIES.md @@ -0,0 +1,51 @@ +# Codex Decision Boundaries + +This document defines where Codex has freedom and where it must not decide alone. + +## Codex Can Decide + +- internal helper function names +- folder organization within already approved domains +- UI microcopy that clarifies state +- additional tests +- stricter validation when compatible with contracts +- small refactors that reduce duplication +- dependency minor versions when compatible +- CSS implementation details + +## Codex Must Follow Existing Specs + +- primary tech stack +- FastAPI backend +- React frontend +- PostGIS database +- GRB-first reference strategy +- status enums +- API envelopes +- CRS calculation rules +- QA/QC formulas +- V1 scope boundaries +- build sequence + +## Codex Must Ask or Stop + +- replacing FastAPI, PostGIS, React or MapLibre +- changing project direction from GeoAI Workbench +- adding authentication +- adding paid services +- changing reference data strategy +- changing scoring/math definitions +- presenting fixture output as real AI +- introducing non-deterministic tests +- removing docs/tests to make builds pass + +## Improvement Rule + +If Codex sees a better approach, it may implement it only when: + +1. it is backward compatible with specs +2. it improves correctness, reliability or clarity +3. it updates docs and tests +4. it does not expand V1 scope + +If not, document it as a proposal in `docs/PROPOSED_IMPROVEMENTS.md`. diff --git a/docs/12-build-control/M7_IMPLEMENTATION_CONTROL_LAYER.md b/docs/12-build-control/M7_IMPLEMENTATION_CONTROL_LAYER.md new file mode 100644 index 00000000..890ba5e7 --- /dev/null +++ b/docs/12-build-control/M7_IMPLEMENTATION_CONTROL_LAYER.md @@ -0,0 +1,89 @@ +# M7 Implementation Control Layer + +This layer exists to keep Codex productive without allowing it to improvise core architecture. + +## Purpose + +M7 adds strict execution control around the existing GeoIntel specifications. It defines how Codex should sequence work, how each pass should prove completion, which traps to avoid, and when it is allowed to improve the design. + +## Operating Principle + +Codex may improve implementation quality, developer experience, robustness, performance and UX clarity, but it may not silently change product direction, data model semantics, analysis definitions, geospatial meaning or API contracts. + +## Build Priorities + +1. Keep the vertical slice working at all times. +2. Build backend contracts before frontend polish. +3. Prefer small complete modules over broad incomplete scaffolding. +4. Every result must be reproducible from fixtures or documented sample data. +5. Every analysis output must retain source, parameters, units and assumptions. +6. Every geospatial geometry must carry CRS awareness. +7. Every long-running process must expose status, errors and recoverability. + +## Mandatory Per-Pass Output + +At the end of every Codex pass, update or create: + +- `CHANGELOG.md` +- `docs/BUILD_STATUS.md` +- relevant TODO checkboxes +- test notes +- known limitations +- next recommended pass + +If a pass changes API contracts, update `contracts/api/` and all affected frontend service calls. + +If a pass changes database models, update migrations, schema docs and seed data. + +If a pass changes analysis logic, update analysis specifications, tests and expected fixtures. + +## Allowed Improvements + +Codex may add: + +- better validation +- better error messages +- safer defaults +- clearer UI states +- test fixtures +- helper utilities +- small performance improvements +- developer scripts +- documentation clarifications + +Codex may not add without explicit approval: + +- multi-user auth +- paid external services +- unrelated AI chat features +- unrelated dashboards +- LiDAR production implementation before V1 slice +- training studio before detection and QA/QC are stable +- new primary data sources that conflict with GRB-first strategy + +## Stop Conditions + +Codex must stop and report instead of continuing if: + +- a core specification conflicts with another specification +- a required dependency cannot be installed +- tests fail for reasons that require product decision changes +- geospatial output cannot be tied to CRS or source metadata +- generated outputs would be misleading or scientifically invalid + +## M7 Completion Target + +The repository is ready for autonomous implementation when Codex can execute: + +1. preflight checks +2. backend foundation +3. database foundation +4. dataset upload and metadata +5. map workspace +6. raster/vector preview +7. demo detection stub or real YOLO integration boundary +8. QA/QC fixture comparison +9. GeoJSON export +10. smoke tests + +without needing manual architecture decisions. diff --git a/docs/12-build-control/MODULE_COMPLETION_MATRIX.md b/docs/12-build-control/MODULE_COMPLETION_MATRIX.md new file mode 100644 index 00000000..de52ce57 --- /dev/null +++ b/docs/12-build-control/MODULE_COMPLETION_MATRIX.md @@ -0,0 +1,16 @@ +# Module Completion Matrix + +This matrix defines what counts as complete for each V1 module. + +| Module | Backend | Frontend | Tests | Docs | V1 Complete When | +|---|---|---|---|---|---| +| Project Manager | CRUD endpoints, models | list/detail/create | API tests | API + UI docs | User can create/open project | +| Area Manager | geometry validation, save area | draw/save/list areas | geometry fixture tests | geospatial rules updated | User can draw and persist polygon | +| Dataset Manager | upload, metadata, storage | upload form, dataset table | upload/metadata tests | storage + data docs | Dataset can be uploaded and inspected | +| Raster Lab | metadata boundary, clip/preview stubs | metadata panel, preview state | metadata tests | raster spec | Raster dataset has readable metadata | +| Vector Lab | GeoJSON import, feature count, bounds | map overlay, details | fixture render/API tests | vector spec | Vector layer renders and can be inspected | +| Detection Lab | analysis run + fixture inference | run form, output layer | deterministic fixture tests | detection spec | Fixture detection output is stored and shown | +| Segmentation Lab | analysis run boundary | planned UI state | contract tests | segmentation spec | V1 may show prepared state unless detection is stable | +| QA/QC Lab | IoU, precision, recall, F1 | metrics panel, error layers | golden QA tests | QA spec | Fixture predictions compare to reference | +| Export Center | GeoJSON/metrics export | download buttons | export tests | export spec | User downloads GeoJSON result | +| Build Status | changelog/status docs | optional UI link | smoke docs test | build status | Status remains truthful | diff --git a/docs/13-implementation-traps/API_RESPONSE_RULES.md b/docs/13-implementation-traps/API_RESPONSE_RULES.md new file mode 100644 index 00000000..d45ea838 --- /dev/null +++ b/docs/13-implementation-traps/API_RESPONSE_RULES.md @@ -0,0 +1,79 @@ +# API Response Rules + +All API responses must be stable and implementation-friendly. + +## Success Envelope + +For single resources: + +```json +{ + "data": {}, + "meta": {} +} +``` + +For lists: + +```json +{ + "data": [], + "meta": { + "count": 0, + "limit": 50, + "offset": 0 + } +} +``` + +## Error Envelope + +```json +{ + "error": { + "code": "VALIDATION_ERROR", + "message": "Human readable message", + "details": {}, + "trace_id": "optional" + } +} +``` + +## Required Resource Fields + +Most persisted resources should include: + +- `id` +- `created_at` +- `updated_at` + +Geospatial resources should also include: + +- `crs` +- `bounds` +- `geometry_type` where applicable + +Analysis resources should include: + +- `status` +- `parameters` +- `outputs` +- `metrics` +- `error_message` when failed + +## Pagination + +Use `limit` and `offset` for V1. Cursor pagination can be added later if needed. + +## Sorting + +Default sort: newest first for projects, datasets, analyses and exports. + +## Contract Drift Rule + +If implementation changes any response shape, update: + +- `contracts/api/` +- API docs +- frontend API client types +- tests diff --git a/docs/13-implementation-traps/FRONTEND_STATE_RULES.md b/docs/13-implementation-traps/FRONTEND_STATE_RULES.md new file mode 100644 index 00000000..d0939e55 --- /dev/null +++ b/docs/13-implementation-traps/FRONTEND_STATE_RULES.md @@ -0,0 +1,70 @@ +# Frontend State Rules + +GeoIntel frontend must be predictable, API-driven and resistant to partial build regressions. + +## State Layers + +Use three state categories: + +1. Server state: projects, datasets, areas, analysis runs, metrics, exports. +2. UI state: selected tab, open panel, layer opacity, map camera. +3. Draft state: unsaved polygon, upload form, threshold slider before run. + +Server state must be loaded through API client functions. Do not duplicate server truth in Zustand except as cached references managed by query tooling. + +## Required Page States + +Every data-driven page must render: + +- initial loading +- empty state +- error state +- ready state +- processing state when jobs exist + +## Map State + +Map layer state must include: + +- id +- name +- source +- visibility +- opacity +- style +- legend label +- feature count if known + +## Analysis Run State + +Analysis run UI must display: + +- status +- started_at +- completed_at when available +- parameters +- model mode if AI-related +- output layers +- metrics +- errors if failed + +## Form Validation + +Client validation improves UX but must not replace backend validation. + +## Navigation + +The navigation should preserve the mental model: + +- Projects +- Workspace +- Map +- Datasets +- Raster +- Vector +- Detection +- Segmentation +- QA/QC +- Exports + +Do not hide core modules behind unrelated dashboard labels. diff --git a/docs/13-implementation-traps/GEOSPATIAL_CALCULATION_RULES.md b/docs/13-implementation-traps/GEOSPATIAL_CALCULATION_RULES.md new file mode 100644 index 00000000..dc0fdf42 --- /dev/null +++ b/docs/13-implementation-traps/GEOSPATIAL_CALCULATION_RULES.md @@ -0,0 +1,87 @@ +# Geospatial Calculation Rules + +These rules define how GeoIntel must handle geospatial calculations. + +## Coordinate Reference Systems + +Default display CRS: EPSG:4326. + +Default metric calculation CRS for Flanders/Kempen: EPSG:31370. + +Every dataset must store: + +- source CRS +- normalized/display CRS if converted +- metric calculation CRS used for area/distance outputs + +## Geometry Validation + +Before inserting vector features: + +1. check geometry exists +2. check geometry type +3. check validity +4. check empty geometry +5. compute bounds +6. compute source feature count + +Invalid geometries should be recorded in dataset metadata. Auto-fix may be attempted using buffer(0) or make_valid only if the metadata records this correction. + +## Area Calculation + +Area values must include units. + +Preferred units: + +- `m2` for feature-level area +- `ha` for summary land cover areas +- `km2` for large area summaries + +Never calculate area from EPSG:4326 degrees. + +## Distance Calculation + +Distance values must include units. + +Preferred units: + +- `m` for local distances +- `km` for totals and densities + +## Density Calculation + +Densities must define denominator: + +- buildings per km2 +- road km per km2 +- vegetation ha per km2 + +## IoU Calculation + +For polygons A and B: + +`IoU = area(intersection(A, B)) / area(union(A, B))` + +Both geometries must be projected to metric CRS before area calculation. + +## Matching Rule + +Default object matching threshold for building QA/QC: + +`IoU >= 0.5` + +Alternative thresholds may be exposed in UI but must default to 0.5 for first V1 implementation. + +## Precision, Recall and F1 + +- TP: predicted feature matched to one reference feature above threshold +- FP: predicted feature without reference match +- FN: reference feature without predicted match + +`precision = TP / (TP + FP)` + +`recall = TP / (TP + FN)` + +`f1 = 2 * precision * recall / (precision + recall)` + +If denominator is zero, return null and include explanatory reason. diff --git a/docs/15-tomorrow-execution/CODEX_HANDOFF_BRIEFING.md b/docs/15-tomorrow-execution/CODEX_HANDOFF_BRIEFING.md new file mode 100644 index 00000000..f7536980 --- /dev/null +++ b/docs/15-tomorrow-execution/CODEX_HANDOFF_BRIEFING.md @@ -0,0 +1,43 @@ +# Codex Handoff Briefing + +You are inheriting GeoIntel Kempen, a GeoAI Workbench for the Belgian Kempen. + +## Read first + +1. `README.md` +2. `AGENTS.md` +3. `docs/V1_SCOPE_FREEZE.md` +4. `docs/SERVICE_ARCHITECTURE.md` +5. `docs/REPOSITORY_CONVENTIONS.md` +6. `docs/15-tomorrow-execution/M8_TOMORROW_EXECUTION_PACK.md` +7. `docs/15-tomorrow-execution/DAY_1_EXECUTION_TIMELINE.md` +8. the current pass prompt under `prompts/codex/day-1/` + +## Build philosophy + +Build thin but real vertical slices. Do not hide missing functionality behind convincing UI. A small working API with tests is better than a beautiful mock. + +## The first usable vertical slice + +The first vertical slice is: + +```text +Project -> Area -> Dataset registration -> Metadata -> Map/Workbench display -> Export-ready internal structure +``` + +Object detection, segmentation and QA/QC are important, but they must sit on a stable foundation. + +## Common failure modes to avoid + +- Creating frontend-only mock data that bypasses API state. +- Mixing geometry parsing into random route handlers. +- Adding AI dependencies before the data model is stable. +- Hardcoding local file paths. +- Ignoring CRS metadata. +- Creating database models that cannot handle future PostGIS geometries. +- Implementing upload without storage policy. +- Writing TODOs instead of completing the requested pass. + +## Improvement freedom + +You may add helper modules, stricter validation, better tests, and cleaner component structure. You may not change the product scope or stack. diff --git a/docs/15-tomorrow-execution/DAY_1_EXECUTION_TIMELINE.md b/docs/15-tomorrow-execution/DAY_1_EXECUTION_TIMELINE.md new file mode 100644 index 00000000..0ac7e932 --- /dev/null +++ b/docs/15-tomorrow-execution/DAY_1_EXECUTION_TIMELINE.md @@ -0,0 +1,124 @@ +# Day 1 Execution Timeline + +This is a practical day plan for Codex. Times are indicative, not strict. + +## Block 0 — Repository audit + +Goal: understand the repo, identify existing docs, confirm no missing foundation files. + +Deliverables: + +- updated `docs/CODEX_EXECUTION_LOG.md`; +- short implementation plan; +- no product scope changes. + +Do not implement features in this block. + +## Block 1 — Backend foundation + +Goal: create the minimal FastAPI application architecture. + +Deliverables: + +- app factory or main app; +- health endpoint; +- config module; +- logging setup; +- consistent response/error envelope; +- backend test harness. + +Acceptance: + +- backend imports cleanly; +- health test passes; +- no database dependency required for health endpoint. + +## Block 2 — Database and domain + +Goal: add SQLAlchemy/Alembic/PostGIS-ready domain foundation. + +Deliverables: + +- database config; +- migration folder; +- core models for projects, areas, datasets, analysis_runs, exports; +- geometry strategy documented in code comments where needed. + +Acceptance: + +- migrations run on local Postgres/PostGIS; +- tests can run with a safe test DB or mocked DB session layer; +- no raw geometry hacks in API layer. + +## Block 3 — Project, Area and Dataset APIs + +Goal: implement first real domain APIs. + +Deliverables: + +- create/list/read projects; +- create/list/read areas; +- dataset registration/upload scaffold; +- validation schemas; +- contract tests. + +Acceptance: + +- documented API contract examples match actual responses; +- invalid GeoJSON produces structured validation error; +- areas store geometry metadata. + +## Block 4 — Frontend shell + +Goal: create a navigable frontend shell matching the workbench model. + +Deliverables: + +- Vite/React/TypeScript app; +- route layout; +- left navigation; +- workspace pages; +- API client with typed methods; +- empty/loading/error states. + +Acceptance: + +- frontend runs; +- routes do not crash; +- API base URL is environment-driven; +- no random fake product flow. + +## Block 5 — Raster/vector metadata + +Goal: add the first geospatial processing services. + +Deliverables: + +- vector metadata parser for GeoJSON; +- raster metadata service scaffold using Rasterio when available; +- dataset metadata endpoint; +- fixture-driven tests. + +Acceptance: + +- fixtures return deterministic metadata; +- unsupported file type returns controlled error; +- processing outputs are stored according to storage spec. + +## Block 6 — Vertical slice stabilization + +Goal: make the foundation coherent. + +Deliverables: + +- smoke script; +- updated docs; +- TODO checked/updated; +- changelog entry; +- known limitations list. + +Acceptance: + +- one command or documented sequence verifies backend + frontend basics; +- no broken imports; +- no undocumented architectural shortcuts. diff --git a/docs/15-tomorrow-execution/M8_TOMORROW_EXECUTION_PACK.md b/docs/15-tomorrow-execution/M8_TOMORROW_EXECUTION_PACK.md new file mode 100644 index 00000000..040b39a1 --- /dev/null +++ b/docs/15-tomorrow-execution/M8_TOMORROW_EXECUTION_PACK.md @@ -0,0 +1,84 @@ +# M8 Tomorrow Execution Pack + +This package exists so Codex can start tomorrow with minimal manual steering. + +## Operating mode + +Codex must work in controlled autonomous passes. Each pass must: + +1. read the relevant docs before editing; +2. implement one coherent layer only; +3. run the documented validation commands; +4. write a concise completion report; +5. update TODO, CHANGELOG and CODEX_EXECUTION_LOG; +6. stop when a blocker requires product/architecture judgement. + +## Non-negotiable product direction + +GeoIntel Kempen is a **GeoAI Workbench**, not a generic dashboard and not a CRUD demo. +The V1 vertical slice must prove: + +- geospatial project/area/dataset management; +- raster/vector metadata extraction; +- map-based spatial workflow; +- PostGIS-ready geometry handling; +- controlled AI detection pipeline scaffolding; +- QA/QC against reference geodata; +- geospatial export. + +## Day-1 success condition + +A successful first Codex day should end with a running foundation that can be started locally and demonstrates: + +- backend health endpoint; +- database connection and migrations; +- project/area/dataset APIs; +- frontend shell with routes; +- map workspace placeholder wired to API state; +- deterministic fixtures and smoke checks; +- no uncontrolled mock-only business logic. + +## Recommended execution order + +1. `prompts/codex/day-1/00_START_HERE.md` +2. `prompts/codex/day-1/01_REPO_AUDIT_AND_PLAN.md` +3. `prompts/codex/day-1/02_BACKEND_FOUNDATION.md` +4. `prompts/codex/day-1/03_DATABASE_AND_DOMAIN.md` +5. `prompts/codex/day-1/04_PROJECT_AREA_DATASET_API.md` +6. `prompts/codex/day-1/05_FRONTEND_SHELL.md` +7. `prompts/codex/day-1/06_RASTER_VECTOR_METADATA.md` +8. `prompts/codex/day-1/07_VERTICAL_SLICE_STABILIZATION.md` + +## What Codex may improve without asking + +Codex may improve naming, folder hygiene, small helper abstractions, test coverage, typing, validation, error messages, and developer ergonomics if the changes preserve the documented contracts. + +## What Codex may not change without explicit approval + +Codex may not change: + +- product positioning; +- selected stack; +- V1 scope boundaries; +- database aggregate names; +- API envelope conventions; +- storage layout; +- QA/QC metric definitions; +- GRB as primary reference strategy; +- incremental build discipline. + +## Required end-of-pass response format + +Every pass must end with: + +```text +PASS COMPLETED: +CHANGED FILES: +- ... +VALIDATION RUN: +- command: result +OPEN ISSUES: +- ... or none +NEXT RECOMMENDED PASS: +- ... +``` diff --git a/docs/15-tomorrow-execution/NEXT_PASS_AFTER_DAY_1.md b/docs/15-tomorrow-execution/NEXT_PASS_AFTER_DAY_1.md new file mode 100644 index 00000000..e7537dd1 --- /dev/null +++ b/docs/15-tomorrow-execution/NEXT_PASS_AFTER_DAY_1.md @@ -0,0 +1,31 @@ +# Next Pass After Day 1 + +If Day 1 succeeds, the next priority is not broad feature expansion. The next priority is the first true GeoAI/GIS capability. + +## Preferred Day 2 sequence + +1. GRB/reference adapter scaffold. +2. Raster tiling interface. +3. Detection model registry scaffold. +4. Deterministic detection fixture adapter. +5. Detection output as GeoJSON. +6. QA/QC overlap metrics against reference fixture. +7. Map overlay display. + +## Why this sequence + +It creates the first portfolio-relevant technical loop: + +```text +Raster/Dataset -> Detection -> Geospatial output -> Reference comparison -> QA metrics -> Export +``` + +## Do not jump directly to + +- full SAM integration; +- real Sentinel downloads; +- LiDAR processing; +- training studio; +- complex report generation. + +Those need the Day 1 foundation and Day 2 detection/QA loop first. diff --git a/docs/16-autonomy-governance/AUTONOMY_BOUNDARIES.md b/docs/16-autonomy-governance/AUTONOMY_BOUNDARIES.md new file mode 100644 index 00000000..c632c613 --- /dev/null +++ b/docs/16-autonomy-governance/AUTONOMY_BOUNDARIES.md @@ -0,0 +1,45 @@ +# Autonomy Boundaries for Codex + +Codex should be proactive, but not uncontrolled. + +## Green zone — Codex can decide + +- function and class names when consistent with docs; +- internal helper extraction; +- validation improvements; +- error message clarity; +- test fixture additions; +- component decomposition; +- minor styling improvements; +- logging improvements; +- dependency pinning within the approved stack. + +## Yellow zone — Codex can decide but must document + +- replacing a library with an equivalent only if dependency installation fails and the replacement stays within the stack intent; +- changing endpoint internals while preserving contracts; +- adding new tables that support existing aggregates; +- adding background job scaffolding earlier than planned; +- improving storage folder structure while preserving published paths. + +Must be documented in: + +- `docs/CODEX_EXECUTION_LOG.md`; +- `CHANGELOG.md`; +- relevant ADR/RFC if architectural. + +## Red zone — Codex must stop and ask + +- changing FastAPI/React/PostGIS stack; +- removing GRB-centered QA/QC direction; +- turning the product into a generic GIS dashboard; +- removing AI pipeline readiness; +- changing V1 scope boundaries; +- adding authentication/multi-user as core V1; +- adding paid/cloud-only dependencies as mandatory; +- implementing real external downloads without source strategy and license notes; +- changing output formats away from GeoJSON/COCO/YOLO/masks without approval. + +## Safe fallback principle + +When a dependency, data source or model cannot be used yet, implement a controlled adapter interface and deterministic fixture-backed behavior. Mark it as a scaffold, not as production-complete. diff --git a/docs/16-autonomy-governance/FAILURE_RECOVERY_PLAYBOOK.md b/docs/16-autonomy-governance/FAILURE_RECOVERY_PLAYBOOK.md new file mode 100644 index 00000000..aba332f0 --- /dev/null +++ b/docs/16-autonomy-governance/FAILURE_RECOVERY_PLAYBOOK.md @@ -0,0 +1,51 @@ +# Failure Recovery Playbook + +Codex must not spiral when a build fails. Use this playbook. + +## Backend import failure + +1. Run the smallest import command available. +2. Fix circular imports first. +3. Verify package `__init__.py` files. +4. Ensure config does not require unavailable services at import time. +5. Add or update a smoke test. + +## Database failure + +1. Check environment variables. +2. Check whether PostGIS extension is required at migration time. +3. Separate pure unit tests from DB integration tests. +4. Do not remove geometry capability to make tests pass. +5. Document required local Postgres/PostGIS command. + +## Frontend build failure + +1. Run TypeScript check. +2. Fix missing exports/imports. +3. Do not silence errors with `any` unless documented and temporary. +4. Ensure API types match contract fixtures. +5. Add an empty/error state rather than fake success. + +## Rasterio/GDAL dependency failure + +1. Keep service interface intact. +2. Add graceful unavailable-state handling. +3. Keep GeoJSON/vector paths working. +4. Document local dependency requirement. +5. Do not fake raster metadata as real metadata. + +## YOLO/SAM unavailable + +1. Keep model registry and detection service interface. +2. Implement deterministic fixture inference adapter. +3. Mark adapter as demo/scaffold. +4. Preserve output shape expected by QA/QC. +5. Do not block foundation work. + +## External data source unavailable + +1. Fall back to fixtures. +2. Preserve source adapter contract. +3. Add retry/error state. +4. Do not hardcode one live response. +5. Document the failure in execution log. diff --git a/docs/16-autonomy-governance/IMPROVEMENT_POLICY.md b/docs/16-autonomy-governance/IMPROVEMENT_POLICY.md new file mode 100644 index 00000000..99ebb75d --- /dev/null +++ b/docs/16-autonomy-governance/IMPROVEMENT_POLICY.md @@ -0,0 +1,43 @@ +# Improvement Policy + +GeoIntel should be strict enough for autonomous execution but flexible enough for good engineering. + +## Desired improvements + +Codex is encouraged to improve: + +- typed API clients; +- service boundaries; +- validation specificity; +- reusable geospatial utilities; +- test determinism; +- frontend state handling; +- developer commands; +- logging and diagnostics; +- small UX clarity improvements. + +## Undesired improvements + +Do not add: + +- unrelated dashboards; +- user accounts before V1; +- payment/billing; +- social features; +- generic file manager replacing dataset manager; +- raw LLM chatbot as a central feature; +- unsupported live data scraping; +- excessive styling frameworks beyond the chosen frontend stack. + +## Improvement report format + +When making improvements beyond the exact prompt, add: + +```text +IMPROVEMENT: +- what changed +- why it helps GeoIntel +- why it does not change scope +``` + +inside `docs/CODEX_EXECUTION_LOG.md`. diff --git a/docs/16-autonomy-governance/QUALITY_GATE_MATRIX.md b/docs/16-autonomy-governance/QUALITY_GATE_MATRIX.md new file mode 100644 index 00000000..658cee09 --- /dev/null +++ b/docs/16-autonomy-governance/QUALITY_GATE_MATRIX.md @@ -0,0 +1,24 @@ +# Quality Gate Matrix + +Every implementation pass must satisfy the relevant gates. + +| Gate | Backend | Frontend | Data/GIS | AI | Required before merge | +|---|---|---|---|---|---| +| Import/build | app imports | TS builds | processing modules import | model adapters import | yes | +| Contract | response envelope | API client typed | metadata shape stable | detection output shape stable | yes | +| Validation | request schemas | form errors | CRS/file errors | threshold/model errors | yes | +| Tests | unit/contract | component where feasible | fixture processing | adapter test | yes for touched area | +| Docs | route/service docs | UI states docs | source/operation docs | model notes | yes | +| No fake success | errors visible | empty states real | unsupported explicit | unavailable explicit | yes | + +## Minimum gates for Day 1 + +- Backend health passes. +- Project/area/dataset contracts pass. +- Frontend shell builds. +- Fixture metadata tests pass. +- Smoke script documents exact failures if any remain. + +## Gate failure response + +If a gate fails, Codex must either fix it or mark the pass incomplete. Do not claim completion when validation was skipped. diff --git a/docs/17-max-prep/M9_API_VALIDATION_EXAMPLES.md b/docs/17-max-prep/M9_API_VALIDATION_EXAMPLES.md new file mode 100644 index 00000000..d45d99e3 --- /dev/null +++ b/docs/17-max-prep/M9_API_VALIDATION_EXAMPLES.md @@ -0,0 +1,91 @@ +# M9 API Validation Examples + +All errors must use the documented API envelope. + +## Invalid Project Name + +Request: + +```json +{ "name": "" } +``` + +Response: + +```json +{ + "success": false, + "data": null, + "error": { + "code": "validation_error", + "message": "Project name is required.", + "details": { "field": "name" } + }, + "meta": {} +} +``` + +## Invalid GeoJSON Polygon + +```json +{ + "success": false, + "data": null, + "error": { + "code": "invalid_geometry", + "message": "Area geometry must be a valid Polygon or MultiPolygon in EPSG:4326.", + "details": { + "reason": "self_intersection" + } + }, + "meta": {} +} +``` + +## Unsupported Dataset Type + +```json +{ + "success": false, + "data": null, + "error": { + "code": "unsupported_dataset_type", + "message": "This file type is not supported in V1.", + "details": { + "allowed_extensions": [".geojson", ".json", ".tif", ".tiff", ".gpkg"] + } + }, + "meta": {} +} +``` + +## External Source Unavailable + +```json +{ + "success": false, + "data": null, + "error": { + "code": "external_service_unavailable", + "message": "The GRB service is unavailable. Use cached data or retry later.", + "details": { "source": "GRB" } + }, + "meta": {} +} +``` + +## Job Failed + +```json +{ + "success": true, + "data": { + "job_id": "uuid", + "status": "failed", + "error_code": "model_not_available", + "error_message": "No compatible detection model is configured." + }, + "error": null, + "meta": {} +} +``` diff --git a/docs/17-max-prep/M9_AUTONOMOUS_BUILD_DOCTRINE.md b/docs/17-max-prep/M9_AUTONOMOUS_BUILD_DOCTRINE.md new file mode 100644 index 00000000..c353a6a6 --- /dev/null +++ b/docs/17-max-prep/M9_AUTONOMOUS_BUILD_DOCTRINE.md @@ -0,0 +1,89 @@ +# M9 Autonomous Build Doctrine + +## Purpose + +This document defines how Codex should behave when building GeoIntel with minimal human intervention. + +## Build Philosophy + +GeoIntel must grow like a professional engineering system: + +1. Contracts first. +2. Backend services second. +3. Frontend integration third. +4. AI pipelines only after stable data flow. +5. QA/QC after outputs exist. +6. Polish only after functionality is testable. + +## Strictness Levels + +### Frozen + +Codex must not change these: + +- FastAPI backend. +- React TypeScript frontend. +- PostgreSQL/PostGIS database. +- Docker Compose local stack. +- Project/Area/Dataset/Analysis domain model. +- API response envelope. +- Storage root conventions. +- V1 scope. + +### Guided + +Codex may choose implementation details within these: + +- exact Python package split, +- React component granularity, +- internal helper names, +- validation library patterns, +- test fixture organization, +- queue abstraction internals. + +### Open Improvement Area + +Codex may improve freely if documented: + +- UI microcopy, +- accessibility, +- loading states, +- logging clarity, +- test coverage, +- developer command quality, +- type safety. + +## Self-Driving Loop + +Every build pass must follow this loop: + +1. Read relevant specs. +2. Identify scope for the pass. +3. Implement the smallest complete vertical slice. +4. Run tests/lint/type checks where available. +5. Update execution log. +6. Update build status. +7. List next pass. + +## Anti-Patterns + +Do not: + +- create huge untested code dumps, +- implement UI without API contracts, +- invent fake geospatial values, +- build YOLO UI before dataset IO works, +- create multiple competing state stores, +- store geospatial data as plain strings when PostGIS geometry is required, +- ignore CRS handling, +- call external services without adapter boundaries. + +## Autonomy Boundary + +Codex can keep working independently as long as: + +- tests are passing or failures are honestly documented, +- no frozen decision is changed, +- V1 scope is preserved, +- every new file belongs to an approved module, +- build logs are updated. diff --git a/docs/17-max-prep/M9_BUILD_BLOCKERS_AND_RECOVERY.md b/docs/17-max-prep/M9_BUILD_BLOCKERS_AND_RECOVERY.md new file mode 100644 index 00000000..fc32187c --- /dev/null +++ b/docs/17-max-prep/M9_BUILD_BLOCKERS_AND_RECOVERY.md @@ -0,0 +1,101 @@ +# M9 Build Blockers and Recovery + +## Database connection failure + +Symptoms: + +- backend cannot connect to PostgreSQL, +- migrations fail, +- PostGIS extension missing. + +Recovery: + +1. Check Docker Compose service names. +2. Check environment variables. +3. Confirm database is reachable from backend container or local process. +4. Run a minimal connection test. +5. Do not replace PostGIS with SQLite except for explicitly isolated unit tests. + +## PostGIS geometry error + +Symptoms: + +- invalid geometry, +- SRID missing, +- geometry column cannot be created. + +Recovery: + +1. Store all app geometry in EPSG:4326 unless a processing-specific CRS is required. +2. Validate GeoJSON before persistence. +3. Use Shapely for geometry validation. +4. Use PostGIS geometry column for persistent area/reference features. +5. Document any CRS transformation. + +## Frontend API mismatch + +Symptoms: + +- UI expects raw data but API returns envelope, +- errors not shown, +- undefined data states. + +Recovery: + +1. Update frontend API client, not individual components. +2. Normalize envelope handling centrally. +3. Ensure every component handles loading, empty, error and ready states. + +## Dependency installation failure + +Symptoms: + +- GDAL/Rasterio install errors, +- PyTorch package issue, +- platform binary mismatch. + +Recovery: + +1. Do not remove the feature from docs. +2. Add dependency note to `docs/DEPENDENCY_LOCK_PLAN.md`. +3. Implement interfaces and tests around pure-Python parts first. +4. Defer heavy binary package execution if needed, but leave adapter boundaries. + +## External data unavailable + +Symptoms: + +- WFS unavailable, +- Sentinel catalog unavailable, +- credentials missing. + +Recovery: + +1. Use demo fixtures. +2. Keep adapter disabled but present. +3. Return explicit `external_service_unavailable` status. +4. Do not fake that live data was fetched. + +## AI model unavailable + +Symptoms: + +- YOLO weights missing, +- SAM unavailable, +- GPU unavailable. + +Recovery: + +1. Build model registry and adapter interface. +2. Add CPU-safe mock inference only if marked as demo mode. +3. Keep output schema identical to real inference. +4. Do not present demo inference as production inference. + +## Test failures + +Recovery: + +1. Fix tests if implementation is wrong. +2. Fix implementation if test reflects contract. +3. Update specs only if they are clearly inconsistent. +4. Document unresolved failures in execution log. diff --git a/docs/17-max-prep/M9_DATA_CONTRACTS_DETAILED.md b/docs/17-max-prep/M9_DATA_CONTRACTS_DETAILED.md new file mode 100644 index 00000000..dd26fc54 --- /dev/null +++ b/docs/17-max-prep/M9_DATA_CONTRACTS_DETAILED.md @@ -0,0 +1,110 @@ +# M9 Detailed Data Contracts + +## Project + +```json +{ + "id": "uuid", + "name": "Geel Building Detection Demo", + "description": "Building detection and QA against reference features.", + "region": "Kempen", + "created_at": "2026-06-11T12:00:00Z", + "updated_at": "2026-06-11T12:00:00Z" +} +``` + +Validation: + +- name is required, +- region defaults to Kempen, +- description optional. + +## Area + +```json +{ + "id": "uuid", + "project_id": "uuid", + "name": "Geel center test area", + "geometry": { "type": "Polygon", "coordinates": [] }, + "crs": "EPSG:4326", + "area_m2": 12345.67, + "bounds": [4.98, 51.15, 5.02, 51.18] +} +``` + +Validation: + +- geometry must be Polygon or MultiPolygon, +- geometry must be valid, +- geometry must not be empty, +- area_m2 must be computed server-side, +- CRS is EPSG:4326 for API IO unless explicitly documented. + +## Dataset + +```json +{ + "id": "uuid", + "project_id": "uuid", + "name": "reference_buildings.geojson", + "dataset_type": "vector", + "source_mode": "demo", + "source_name": "fixture", + "storage_path": "storage/projects/.../raw/reference_buildings.geojson", + "status": "uploaded|processing|ready|failed", + "crs": "EPSG:4326", + "bounds": [4.98, 51.15, 5.02, 51.18], + "feature_count": 100, + "metadata": {} +} +``` + +## Analysis Run + +```json +{ + "id": "uuid", + "project_id": "uuid", + "area_id": "uuid", + "analysis_type": "object_detection|segmentation|qaqc|raster_metadata|vector_metadata", + "status": "queued|running|completed|failed|cancelled", + "parameters": {}, + "started_at": null, + "finished_at": null, + "error": null +} +``` + +## Detection Feature + +```json +{ + "id": "uuid", + "analysis_run_id": "uuid", + "class_name": "building", + "confidence": 0.91, + "geometry": { "type": "Polygon", "coordinates": [] }, + "bbox": [4.99, 51.16, 4.991, 51.161], + "source_tile": "tile_001.tif", + "metadata": {} +} +``` + +## QA/QC Result + +```json +{ + "analysis_run_id": "uuid", + "reference_dataset_id": "uuid", + "predicted_dataset_id": "uuid", + "match_threshold_iou": 0.5, + "precision": 0.94, + "recall": 0.91, + "f1": 0.925, + "true_positives": 94, + "false_positives": 6, + "false_negatives": 9, + "findings": [] +} +``` diff --git a/docs/17-max-prep/M9_FINAL_PRE_CODE_CHECKLIST.md b/docs/17-max-prep/M9_FINAL_PRE_CODE_CHECKLIST.md new file mode 100644 index 00000000..a1138b5b --- /dev/null +++ b/docs/17-max-prep/M9_FINAL_PRE_CODE_CHECKLIST.md @@ -0,0 +1,46 @@ +# M9 Final Pre-Code Checklist + +Before starting a long Codex build session, verify: + +## Repository + +- [ ] Full zip extracted cleanly. +- [ ] Git initialized. +- [ ] Initial commit made before Codex changes. +- [ ] `.env.example` exists. +- [ ] Docker Compose file exists. +- [ ] README explains the project. + +## Codex Input + +- [ ] Master prompt available. +- [ ] Build order available. +- [ ] Definition of Done available. +- [ ] Scorecards available. +- [ ] Failure recovery available. + +## Scope + +- [ ] V1 scope freeze read. +- [ ] No LiDAR in first pass. +- [ ] No training studio in first pass. +- [ ] No QGIS plugin in first pass. +- [ ] No advanced Sentinel implementation in first pass. + +## Expected First Output + +- [ ] Pass 0 audit. +- [ ] Execution log. +- [ ] Build status. +- [ ] Gap report. + +## Local Environment + +- [ ] Docker available. +- [ ] Node available. +- [ ] Python available. +- [ ] PostgreSQL/PostGIS preferably via Docker. + +## Human Intervention Rules + +Only intervene if Codex asks to change a frozen decision or cannot proceed due to environment failure. diff --git a/docs/17-max-prep/M9_GAP_TO_TASK_CONVERSION.md b/docs/17-max-prep/M9_GAP_TO_TASK_CONVERSION.md new file mode 100644 index 00000000..bb1bf616 --- /dev/null +++ b/docs/17-max-prep/M9_GAP_TO_TASK_CONVERSION.md @@ -0,0 +1,54 @@ +# M9 Gap-to-Task Conversion Rules + +When Codex finds a gap, it must convert it into an actionable task instead of leaving vague notes. + +## Task Format + +```md +### TASK-ID: Short title + +Type: backend | frontend | docs | test | devops | data | ai +Priority: P0 | P1 | P2 | P3 +Status: open | in_progress | done | blocked +Owner: Codex + +Context: +... + +Required changes: +- ... + +Acceptance criteria: +- [ ] ... + +Validation commands: +- ... + +Blocked by: +- none or exact blocker +``` + +## Priority Rules + +### P0 + +Blocks app startup, database, API envelope, project/area/dataset foundation. + +### P1 + +Blocks V1 core workflow but not app startup. + +### P2 + +Improves quality, tests, UX, docs. + +### P3 + +Nice-to-have or future extension. + +## Gap Handling + +- If gap is architecture-related: update docs before code. +- If gap is implementation-related: create ticket and implement if in current pass. +- If gap is dependency-related: document in dependency plan and add fallback interface. +- If gap is scope expansion: move to proposed improvements. diff --git a/docs/17-max-prep/M9_GEOSPATIAL_EDGE_CASES.md b/docs/17-max-prep/M9_GEOSPATIAL_EDGE_CASES.md new file mode 100644 index 00000000..1afd788d --- /dev/null +++ b/docs/17-max-prep/M9_GEOSPATIAL_EDGE_CASES.md @@ -0,0 +1,90 @@ +# M9 Geospatial Edge Cases + +Codex must account for these cases early to avoid later rewrites. + +## Invalid polygon + +Examples: + +- self-intersection, +- unclosed ring, +- empty coordinates, +- wrong coordinate nesting. + +Required behavior: + +- reject with validation error, +- return specific message, +- do not persist. + +## MultiPolygon + +Required behavior: + +- accept for areas and vector features, +- compute total area across all parts, +- display as a single layer. + +## CRS mismatch + +Scenario: + +- reference data is EPSG:31370, +- API uses EPSG:4326, +- raster processing may use native CRS. + +Required behavior: + +- store original CRS in dataset metadata, +- transform API geometries to EPSG:4326 for frontend, +- use projected CRS for area/length calculations when needed, +- document transformations. + +## Very large file + +Required behavior: + +- reject files above configured limit with clear error, +- do not load huge files into memory at once, +- expose future chunking/tile strategy. + +## Raster without georeference + +Required behavior: + +- accept only as non-georeferenced image if supported, +- mark geospatial operations unavailable, +- do not pretend map alignment exists. + +## Feature outside selected area + +Required behavior: + +- clip if operation requested, +- otherwise keep original and display warning if used for area analysis. + +## Geometry collection + +Required behavior: + +- reject for V1 unless explicitly converted, +- explain supported geometry types. + +## Zero-area feature + +Required behavior: + +- keep for line/point layers, +- reject or ignore for polygon-area metrics, +- record warning in analysis result. + +## Duplicate features + +Required behavior: + +- do not automatically delete without audit trail, +- expose duplicate count in data quality metadata. + +## Antimeridian/global edge cases + +Not relevant for Kempen V1. Document as out of scope. diff --git a/docs/17-max-prep/M9_IMPLEMENTATION_REVIEW_SCRIPT.md b/docs/17-max-prep/M9_IMPLEMENTATION_REVIEW_SCRIPT.md new file mode 100644 index 00000000..086e3cf4 --- /dev/null +++ b/docs/17-max-prep/M9_IMPLEMENTATION_REVIEW_SCRIPT.md @@ -0,0 +1,59 @@ +# M9 Implementation Review Script + +Use this manual review script after Codex produces a build. + +## 1. Repository Hygiene + +- [ ] No random root-level files. +- [ ] No duplicate docs with conflicting instructions. +- [ ] No generated caches committed. +- [ ] No secrets committed. +- [ ] `.env.example` updated if env vars changed. + +## 2. Backend + +- [ ] App starts. +- [ ] Health endpoint returns envelope or documented health format. +- [ ] API routes grouped logically. +- [ ] Validation errors are structured. +- [ ] Database models match schema docs. +- [ ] Geometry fields are not plain unvalidated strings. +- [ ] Tests exist for implemented endpoints. + +## 3. Frontend + +- [ ] App starts. +- [ ] No blank route screens. +- [ ] API client centralizes envelope parsing. +- [ ] Loading, empty, error states present. +- [ ] Demo workflow visible. +- [ ] Map shell does not depend on unavailable live data. + +## 4. Geospatial Logic + +- [ ] CRS metadata preserved. +- [ ] Area calculations are server-side. +- [ ] Invalid geometries rejected. +- [ ] Demo fixtures load correctly. +- [ ] No fake geospatial metrics presented as real. + +## 5. Documentation + +- [ ] Changelog updated. +- [ ] Build status updated. +- [ ] Execution log updated. +- [ ] Known limitations updated if needed. + +## 6. Commands + +Run what is available: + +```bash +python -m pytest +npm test +npm run build +npm run lint +docker compose config +``` + +If commands are not available yet, Codex must document why. diff --git a/docs/17-max-prep/M9_LONG_FORM_CODEX_PROMPT_VARIANTS.md b/docs/17-max-prep/M9_LONG_FORM_CODEX_PROMPT_VARIANTS.md new file mode 100644 index 00000000..df8e3779 --- /dev/null +++ b/docs/17-max-prep/M9_LONG_FORM_CODEX_PROMPT_VARIANTS.md @@ -0,0 +1,27 @@ +# M9 Long-Form Codex Prompt Variants + +Use these prompts when Codex needs a more specific instruction after the master prompt. + +## Prompt: Backend Foundation Only + +Build only the backend foundation for GeoIntel. Do not work on frontend or AI. Implement FastAPI app startup, settings, health endpoint, response envelope, structured error handling, and basic tests. Follow `docs/API_CONTRACTS.md`, `docs/ERROR_HANDLING_AND_STATUSES.md`, and `docs/17-max-prep/M9_PASS_SCORECARDS.md`. Update execution log, build status and changelog. + +## Prompt: Database Foundation Only + +Implement database foundation only. Configure PostgreSQL/PostGIS using the documented Docker Compose environment. Add ORM models and migrations for projects, areas, datasets and analysis_runs. Geometry must use PostGIS-compatible fields. Do not implement AI or advanced datasets. Add tests where possible and update docs. + +## Prompt: Dataset Manager Only + +Implement the V1 Dataset Manager skeleton. Support upload metadata, storage path conventions, dataset status lifecycle and list/read endpoints. Use demo fixtures for tests. Do not fetch live GRB or Sentinel yet. Ensure source_mode is present. Update execution log and build status. + +## Prompt: Frontend Shell Only + +Implement the frontend shell with React TypeScript. Add routing, layout, project list, workspace shell, dataset manager shell and map workbench placeholder with correct states. Use API client envelope parsing. Do not hardcode fake analysis results. + +## Prompt: QA/QC Skeleton Only + +Implement pure geometry QA utilities and tests: IoU, precision, recall, F1 using fixture polygons. Add a backend service interface and endpoint skeleton if foundation exists. Do not require YOLO outputs yet; use fixture reference/predicted layers. + +## Prompt: Stabilization Pass + +Do not add features. Run all available tests/builds/lints. Fix failures. Update docs, changelog, known limitations and execution log. Remove dead code and root-folder clutter. Ensure app startup paths are documented. diff --git a/docs/17-max-prep/M9_MAX_PREPARATION_PACK.md b/docs/17-max-prep/M9_MAX_PREPARATION_PACK.md new file mode 100644 index 00000000..45e36581 --- /dev/null +++ b/docs/17-max-prep/M9_MAX_PREPARATION_PACK.md @@ -0,0 +1,83 @@ +# M9 Max Preparation Pack + +Status: specification expansion after M8. +Purpose: make GeoIntel as close as possible to a self-driving Codex project while still allowing Codex to make local implementation improvements. + +## Goal + +GeoIntel must be prepared so that Codex can: + +1. read the repository, +2. understand the product and constraints, +3. build the foundation, +4. validate itself, +5. recover from common failures, +6. report honestly what was completed, +7. avoid architectural drift. + +## Principle + +Strict on architecture. Flexible on implementation details. + +Codex may improve: + +- component structure inside the approved route/component map, +- service internals if IO contracts remain stable, +- validation messages if API envelope remains stable, +- tests if they increase coverage, +- UI polish if it does not change the workflow. + +Codex may not change without explicit approval: + +- backend framework, +- frontend framework, +- database choice, +- PostGIS requirement, +- core entity names, +- API response envelope, +- dataset storage layout, +- project positioning as a GeoAI Workbench, +- V1 scope freeze. + +## M9 Additions + +This pack adds: + +- final autonomous build doctrine, +- exact day-one Codex master prompt, +- build pass scorecards, +- gap-to-task conversion rules, +- real-versus-demo data policy, +- geospatial edge case handling, +- failure mode catalog, +- module data contracts, +- UI empty/loading/error states, +- API validation examples, +- seed fixtures policy, +- regression map, +- implementation review scripts, +- handoff checklist. + +## Expected Use Tomorrow + +1. Extract the full zip. +2. Open the repository in Codex. +3. Paste `prompts/codex/M9_DAY_ONE_MASTER_PROMPT.md`. +4. Let Codex run Pass 0 first. +5. Require Codex to update `docs/CODEX_EXECUTION_LOG.md` after each pass. +6. Do not allow feature expansion until Pass 1-4 are green. + +## Success Definition + +The preparation is successful if Codex can start from an empty implementation and produce: + +- a working FastAPI app, +- a working React app, +- Docker Compose services, +- PostGIS models/migrations, +- project/area/dataset APIs, +- demo fixture loading, +- initial map UI shell, +- validation and health checks, +- tests for the implemented pieces, +- a changelog and execution log. diff --git a/docs/17-max-prep/M9_MODULE_DATAFLOW_CHECKLIST.md b/docs/17-max-prep/M9_MODULE_DATAFLOW_CHECKLIST.md new file mode 100644 index 00000000..bf3132ba --- /dev/null +++ b/docs/17-max-prep/M9_MODULE_DATAFLOW_CHECKLIST.md @@ -0,0 +1,133 @@ +# M9 Module Dataflow Checklist + +Every module must declare its dataflow. + +## Dataset Manager + +Input: + +- file upload, +- project id, +- optional area id. + +Processing: + +- validate extension, +- store raw file, +- inspect metadata, +- persist dataset record, +- schedule optional processing job. + +Output: + +- dataset record, +- status, +- metadata. + +## Raster Lab + +Input: + +- raster dataset id, +- optional area geometry. + +Processing: + +- read raster metadata, +- compute bounds, +- inspect bands, +- clip if requested, +- generate preview/tiles later. + +Output: + +- metadata, +- preview descriptor, +- derived dataset if clipped. + +## Vector Lab + +Input: + +- vector dataset id, +- optional area geometry. + +Processing: + +- load features, +- validate CRS, +- compute feature count, +- compute bounds, +- clip if requested. + +Output: + +- vector metadata, +- clipped layer, +- operation report. + +## Detection Lab + +Input: + +- raster dataset id, +- model id, +- threshold, +- classes. + +Processing: + +- tile raster, +- run model adapter, +- merge detections, +- georeference outputs, +- persist detection layer. + +Output: + +- detection dataset/layer, +- analysis run, +- metrics. + +## Segmentation Lab + +Input: + +- raster dataset id, +- model id, +- classes/prompts. + +Processing: + +- generate masks, +- polygonize, +- clean geometry, +- compute areas, +- persist layer. + +Output: + +- mask path, +- polygon layer, +- metrics. + +## QA/QC Lab + +Input: + +- reference vector layer, +- predicted vector layer, +- matching threshold. + +Processing: + +- spatial match, +- IoU calculation, +- precision/recall/F1, +- false positive/negative features. + +Output: + +- QA metrics, +- findings, +- QA overlay layers. diff --git a/docs/17-max-prep/M9_PASS_SCORECARDS.md b/docs/17-max-prep/M9_PASS_SCORECARDS.md new file mode 100644 index 00000000..1b07b870 --- /dev/null +++ b/docs/17-max-prep/M9_PASS_SCORECARDS.md @@ -0,0 +1,91 @@ +# M9 Build Pass Scorecards + +Use these scorecards after every Codex pass. A pass is not considered complete until its scorecard is mostly green. + +## Pass 0 — Audit + +- [ ] Repository tree inspected. +- [ ] Existing docs summarized. +- [ ] Missing implementation listed. +- [ ] Build blockers listed. +- [ ] Next pass selected. +- [ ] No feature code added. + +## Pass 1 — Backend Foundation + +- [ ] FastAPI app starts. +- [ ] Health endpoint exists. +- [ ] Settings load from environment. +- [ ] CORS configured for local frontend. +- [ ] API envelope helper exists. +- [ ] Error handler exists. +- [ ] Backend tests run. + +## Pass 2 — Database Foundation + +- [ ] SQLAlchemy or approved ORM configured. +- [ ] Alembic migrations initialized. +- [ ] PostGIS extension migration exists. +- [ ] Project model exists. +- [ ] Area model exists with geometry. +- [ ] Dataset model exists. +- [ ] Local database connection documented. + +## Pass 3 — Project/Area APIs + +- [ ] Create/list/read project. +- [ ] Create/list/read area. +- [ ] GeoJSON polygon validation. +- [ ] Area size calculation. +- [ ] Envelope responses. +- [ ] Error responses for invalid geometry. +- [ ] Tests for happy and failure paths. + +## Pass 4 — Dataset Manager Skeleton + +- [ ] Dataset upload endpoint. +- [ ] File stored under documented storage root. +- [ ] Metadata extraction placeholder with real file inspection when possible. +- [ ] Dataset list/read APIs. +- [ ] Status lifecycle present. +- [ ] Tests with fixtures. + +## Pass 5 — Frontend Foundation + +- [ ] React app starts. +- [ ] Route shell exists. +- [ ] API client uses envelope. +- [ ] Error/loading/empty states exist. +- [ ] Project list page exists. +- [ ] Workspace shell exists. + +## Pass 6 — Map Workbench Shell + +- [ ] MapLibre or approved map abstraction installed. +- [ ] Area polygon can be displayed. +- [ ] Demo GeoJSON can be loaded. +- [ ] Layer panel shell exists. +- [ ] No hardcoded fake analysis results. + +## Pass 7 — Raster/Vector Core Skeleton + +- [ ] Raster metadata endpoint skeleton. +- [ ] Vector metadata endpoint skeleton. +- [ ] CRS field exposed. +- [ ] Bounds field exposed. +- [ ] UI metadata panels exist. + +## Pass 8 — QA/QC Skeleton + +- [ ] QA service interface exists. +- [ ] IoU function unit-tested with fixtures. +- [ ] Precision/recall formulas implemented. +- [ ] QA endpoint accepts reference and predicted layer IDs or fixture IDs. +- [ ] QA result card exists. + +## Pass 9 — Stabilization + +- [ ] All smoke commands documented. +- [ ] Changelog updated. +- [ ] Known limitations updated. +- [ ] Next advanced module recommendation written. diff --git a/docs/17-max-prep/M9_REAL_VS_DEMO_DATA_POLICY.md b/docs/17-max-prep/M9_REAL_VS_DEMO_DATA_POLICY.md new file mode 100644 index 00000000..378f449e --- /dev/null +++ b/docs/17-max-prep/M9_REAL_VS_DEMO_DATA_POLICY.md @@ -0,0 +1,59 @@ +# Real vs Demo Data Policy + +GeoIntel may use demo fixtures during early development, but the UI and backend must clearly distinguish demo data from live data. + +## Data Categories + +### Real Data + +Data fetched from or uploaded by a real source: + +- GRB WFS/cache, +- user-uploaded GeoTIFF, +- user-uploaded GeoJSON/Shapefile/GPKG, +- Sentinel scene, +- DHMV product. + +### Demo Fixture Data + +Small repository-contained examples used for development and tests: + +- `demo/geel/reference_buildings.geojson`, +- `demo/geel/demo_detections.geojson`, +- `fixtures/geojson/*`. + +### Synthetic Test Data + +Minimal generated data used only in unit tests. + +## Rules + +- Demo data may be used to build UI states and verify pipelines. +- Demo data must be labeled as demo in API responses. +- Real-data adapters must not silently fall back to demo data. +- A failed external fetch must return an error/status, not demo data. +- Synthetic data must not appear in production UI unless under a test/demo route. + +## Dataset Metadata Field + +Every dataset must include: + +```json +{ + "source_mode": "real|demo|synthetic", + "source_name": "GRB|OSM|user_upload|fixture|generated_test", + "license": "string or unknown", + "retrieved_at": "ISO date or null" +} +``` + +## UI Requirement + +The dataset table must display a source badge: + +- Real +- Demo +- Synthetic +- Unknown + +Synthetic should never be shown in normal user workflows. diff --git a/docs/17-max-prep/M9_REGRESSION_MAP.md b/docs/17-max-prep/M9_REGRESSION_MAP.md new file mode 100644 index 00000000..a8146df7 --- /dev/null +++ b/docs/17-max-prep/M9_REGRESSION_MAP.md @@ -0,0 +1,81 @@ +# M9 Regression Map + +These are the most likely regressions during autonomous builds. + +## API Envelope Drift + +Risk: some endpoints return raw data while others return envelopes. + +Prevention: + +- central response helper, +- API client tests, +- sample responses. + +## Geometry Stored Incorrectly + +Risk: GeoJSON stored as text everywhere. + +Prevention: + +- PostGIS geometry columns, +- schema tests, +- validation utilities. + +## Demo Data Masquerades as Real Data + +Risk: UI shows fixture metrics without labeling them. + +Prevention: + +- source_mode field, +- demo badge, +- no fallback-to-fixture for external failures. + +## Frontend State Duplication + +Risk: every component fetches differently. + +Prevention: + +- central API client, +- shared hooks, +- state contracts. + +## Heavy AI Implemented Too Early + +Risk: Codex spends time on YOLO/SAM before storage and datasets work. + +Prevention: + +- pass scorecards, +- V1 ordering, +- disabled state for model features. + +## CRS Ignored + +Risk: area/length values wrong. + +Prevention: + +- CRS field mandatory, +- projected calculations documented, +- geospatial edge cases. + +## Docs Not Updated + +Risk: implementation diverges from specs. + +Prevention: + +- Definition of Done includes docs, +- execution log mandatory. + +## Root Folder Pollution + +Risk: scripts and generated outputs appear at root. + +Prevention: + +- repository conventions, +- review script. diff --git a/docs/17-max-prep/M9_UI_STATE_SPEC.md b/docs/17-max-prep/M9_UI_STATE_SPEC.md new file mode 100644 index 00000000..425261a4 --- /dev/null +++ b/docs/17-max-prep/M9_UI_STATE_SPEC.md @@ -0,0 +1,108 @@ +# M9 UI State Specification + +Every page and major component must implement these states. + +## Required States + +### Initial + +No user action yet. + +Example copy: + +> Start by creating a project or opening the Geel demo. + +### Loading + +Data is being fetched or processed. + +Requirements: + +- spinner or skeleton, +- clear label, +- no layout jump where avoidable. + +### Empty + +Request succeeded but no records exist. + +Example: + +> No datasets have been added to this project yet. + +### Ready + +Data exists and actions are available. + +### Error + +Request failed or validation failed. + +Requirements: + +- readable error, +- retry action where safe, +- technical details hidden behind details/expand if useful. + +### Disabled/Future + +Feature is planned but not implemented in V1. + +Requirements: + +- show disabled state, +- explain why, +- do not show fake results. + +## Page Requirements + +### Home + +- Initial: explain GeoIntel. +- Empty: no recent projects. +- Ready: project cards and demo card. + +### Project Workspace + +- Loading project. +- Missing project error. +- Empty datasets/areas states. +- Ready dashboard. + +### Dataset Manager + +- Empty upload area. +- Upload progress. +- Upload failed. +- Processing. +- Ready metadata. + +### Map Workbench + +- No area selected. +- Area loaded. +- Layer loading. +- Layer error. +- Unsupported layer type. + +### Detection Lab + +- No raster selected. +- No model configured. +- Running job. +- Completed detections. +- Failed inference. + +### QA/QC Lab + +- Missing reference dataset. +- Missing predicted dataset. +- Running comparison. +- Metrics ready. +- No matches found. + +## Do Not + +- Do not leave blank panels. +- Do not show random demo metrics in ready state. +- Do not hide errors in console only. diff --git a/docs/18-ultra-prep/AUTONOMOUS_BUILD_CHARTER.md b/docs/18-ultra-prep/AUTONOMOUS_BUILD_CHARTER.md new file mode 100644 index 00000000..ab461865 --- /dev/null +++ b/docs/18-ultra-prep/AUTONOMOUS_BUILD_CHARTER.md @@ -0,0 +1,41 @@ +# Autonomous Build Charter + +## Mission +Codex must build GeoIntel Kempen as a production-shaped GeoAI Workbench, not as a demo-only UI. Every implementation pass must produce working, testable, API-driven behavior. + +## Frozen Product Identity +GeoIntel is a GeoAI Workbench for the Kempen focused on raster/vector processing, AI detection, segmentation, QA/QC against reference data, and geospatial exports. + +## Allowed Autonomy +Codex may improve: +- internal code organization when it keeps documented contracts intact; +- UI microcopy and layout when it improves clarity; +- validation and error messages; +- tests, fixtures, logging, and developer tooling; +- performance optimizations that do not alter outputs; +- accessibility and keyboard navigation; +- extra helper utilities that support documented workflows. + +## Not Allowed Without Explicit User Approval +Codex must not: +- replace FastAPI, React, TypeScript, PostgreSQL/PostGIS, or Python GIS stack; +- introduce a different product direction such as a generic chatbot or CRUD dashboard; +- remove GRB/QA/QC as a first-class concept; +- make LiDAR, training studio, or MLOps part of V1 core; +- hardcode fake results while presenting them as real processing; +- break existing documented API contracts; +- silently change geometry formats or CRS assumptions; +- ignore tests because a dependency is missing. + +## Output Expectations Per Pass +Every Codex pass must end with: +1. changed files list; +2. completed tasks; +3. skipped tasks with reason; +4. commands run; +5. test results; +6. known limitations; +7. next recommended pass. + +## Quality Principle +If a feature cannot be fully implemented in the pass, Codex must implement the durable skeleton plus honest status handling, not a fake success path. diff --git a/docs/18-ultra-prep/BUILD_PASS_TEMPLATE.md b/docs/18-ultra-prep/BUILD_PASS_TEMPLATE.md new file mode 100644 index 00000000..8c8f1e3b --- /dev/null +++ b/docs/18-ultra-prep/BUILD_PASS_TEMPLATE.md @@ -0,0 +1,66 @@ +# Build Pass Template + +Use this exact structure for every Codex build pass. + +## Pass Name +Example: `Pass 03 — Dataset Manager Backend` + +## Goal +One sentence describing the feature outcome. + +## Inputs +- Required docs: +- Required contracts: +- Required fixtures: +- Previous pass dependencies: + +## Scope +### Must implement +- [ ] Item + +### May improve +- [ ] Item + +### Must not touch +- [ ] Item + +## Implementation Steps +1. Inspect existing repo state. +2. Confirm relevant contracts. +3. Implement backend/domain changes. +4. Implement frontend/API client changes if applicable. +5. Add or update tests. +6. Add fixtures if needed. +7. Update docs and TODO status. +8. Run validation commands. +9. Produce pass summary. + +## Validation Commands +```bash +# backend +pytest +ruff check backend || true + +# frontend +npm run build +npm run typecheck || true +npm run lint || true +``` + +## Acceptance Criteria +- [ ] Feature works through API or UI. +- [ ] Errors are explicit and typed. +- [ ] No mock success path is presented as real. +- [ ] Tests or fixtures cover the happy path and at least one failure path. +- [ ] Documentation is updated. + +## Handoff Format +```md +# Pass Summary +## Completed +## Changed Files +## Commands Run +## Test Results +## Known Issues +## Next Step +``` diff --git a/docs/18-ultra-prep/CODEX_START_HERE.md b/docs/18-ultra-prep/CODEX_START_HERE.md new file mode 100644 index 00000000..11064a17 --- /dev/null +++ b/docs/18-ultra-prep/CODEX_START_HERE.md @@ -0,0 +1,24 @@ +# Codex Start Here + +Read these files first, in this exact order: + +1. `README.md` +2. `AGENTS.md` +3. `docs/18-ultra-prep/AUTONOMOUS_BUILD_CHARTER.md` +4. `docs/V1_SCOPE_FREEZE.md` +5. `docs/SERVICE_ARCHITECTURE.md` +6. `docs/REPOSITORY_CONVENTIONS.md` +7. `docs/API_CONTRACTS.md` +8. `docs/DATABASE_IMPLEMENTATION_PLAN.md` +9. `docs/18-ultra-prep/CRITICAL_PATH_TO_V1.md` +10. `docs/18-ultra-prep/BUILD_PASS_TEMPLATE.md` + +Then execute the first incomplete pass from: + +- `prompts/codex/M10_PASS_SEQUENCE.md` + +Rules: +- Do not skip ahead. +- Do not start UI polish before backend contracts exist. +- Do not fake geospatial processing; return honest pending/unavailable states. +- Update TODO/checklists after every pass. diff --git a/docs/18-ultra-prep/CONNECTOR_IMPLEMENTATION_GUIDE.md b/docs/18-ultra-prep/CONNECTOR_IMPLEMENTATION_GUIDE.md new file mode 100644 index 00000000..5ff2154a --- /dev/null +++ b/docs/18-ultra-prep/CONNECTOR_IMPLEMENTATION_GUIDE.md @@ -0,0 +1,27 @@ +# Connector Implementation Guide + +## Connector Interface +Every external data connector should expose: +- `name` +- `capabilities` +- `is_configured()` +- `health_check()` +- `fetch_by_area(area_geometry, parameters)` +- `normalize(raw_result)` +- `cache_key(area, parameters)` + +## Required Connectors +### GRB Connector +V1 target: fetch or register reference building polygons. If live WFS cannot be implemented immediately, build the interface and support fixture/local import honestly. + +### OSM Connector +V1 target: fetch buildings/roads/water/green where possible using Overpass or local fixture fallback. + +### Sentinel Connector +V2 target. In V1 it should remain disabled with clear roadmap state. + +## Connector Failure Behavior +- timeout -> `CONNECTOR_TIMEOUT` +- missing config -> `CONNECTOR_UNCONFIGURED` +- invalid response -> `CONNECTOR_INVALID_RESPONSE` +- no data -> valid empty result, not error diff --git a/docs/18-ultra-prep/CRITICAL_PATH_TO_V1.md b/docs/18-ultra-prep/CRITICAL_PATH_TO_V1.md new file mode 100644 index 00000000..e6c6d09d --- /dev/null +++ b/docs/18-ultra-prep/CRITICAL_PATH_TO_V1.md @@ -0,0 +1,35 @@ +# Critical Path to V1 + +This is the shortest reliable path to a portfolio-ready V1. + +## V1 Critical Path +1. Repo foundation and dev environment. +2. Database and PostGIS schema. +3. Project and area CRUD. +4. Dataset upload and metadata extraction. +5. Vector import and display. +6. Raster import and metadata display. +7. Map workbench with layer management. +8. OSM/GRB reference ingest interface or stubbed real-data connector with honest unavailable state. +9. Detection pipeline architecture with YOLO adapter. +10. Detection result storage as geospatial features. +11. QA/QC comparison engine against reference polygons. +12. GeoJSON export. +13. Demo scenario page. +14. Smoke tests and documentation. + +## V1 Can Be Portfolio-Ready Without +- full Sentinel automation; +- actual production GRB WFS credentials or final endpoint if unavailable; +- LiDAR processing; +- model training; +- multi-user auth; +- perfect visual polish. + +## V1 Cannot Be Portfolio-Ready Without +- a working map-driven workflow; +- actual geospatial data models; +- real CRS/geometry handling; +- honest dataset status handling; +- at least one end-to-end detection/QA workflow, even if it uses a small local demo fixture/model adapter first; +- clear export output. diff --git a/docs/18-ultra-prep/CRS_POLICY.md b/docs/18-ultra-prep/CRS_POLICY.md new file mode 100644 index 00000000..d14d4b65 --- /dev/null +++ b/docs/18-ultra-prep/CRS_POLICY.md @@ -0,0 +1,24 @@ +# CRS Policy + +## API Boundary +GeoJSON in API responses should be EPSG:4326. + +## Analytical Operations +For Belgium/Kempen projected calculations, use EPSG:31370 where possible. + +## Raster Operations +Raster operations must preserve source CRS unless a reproject operation is explicitly requested. + +## Required Metadata +Every dataset must store: +- source CRS; +- normalized CRS string; +- bounds in source CRS; +- bounds in EPSG:4326; +- transform/affine for rasters where available. + +## Unsupported Cases +If CRS cannot be determined: +- dataset can be stored as `metadata_failed` or `requires_crs`; +- processing requiring geospatial alignment must be blocked; +- UI must ask for CRS or show limitation. diff --git a/docs/18-ultra-prep/ERROR_TAXONOMY.md b/docs/18-ultra-prep/ERROR_TAXONOMY.md new file mode 100644 index 00000000..860cdc6d --- /dev/null +++ b/docs/18-ultra-prep/ERROR_TAXONOMY.md @@ -0,0 +1,36 @@ +# Error Taxonomy + +All API errors must use a consistent shape. + +```json +{ + "error": { + "code": "DATASET_UNSUPPORTED_FORMAT", + "message": "The uploaded file format is not supported for this operation.", + "details": {}, + "trace_id": "optional" + } +} +``` + +## Core Error Codes +- `VALIDATION_ERROR` +- `NOT_FOUND` +- `CONFLICT` +- `FEATURE_DISABLED` +- `DATASET_UPLOAD_FAILED` +- `DATASET_UNSUPPORTED_FORMAT` +- `CRS_MISSING` +- `CRS_UNSUPPORTED` +- `GEOMETRY_INVALID` +- `RASTER_METADATA_FAILED` +- `VECTOR_METADATA_FAILED` +- `PROCESSING_JOB_FAILED` +- `MODEL_UNAVAILABLE` +- `INFERENCE_FAILED` +- `REFERENCE_LAYER_MISSING` +- `QA_MATCHING_FAILED` +- `EXPORT_FAILED` + +## Frontend Requirements +Every error code should render a useful message and suggested next action. diff --git a/docs/18-ultra-prep/FEATURE_FLAG_STRATEGY.md b/docs/18-ultra-prep/FEATURE_FLAG_STRATEGY.md new file mode 100644 index 00000000..e3e4d9c7 --- /dev/null +++ b/docs/18-ultra-prep/FEATURE_FLAG_STRATEGY.md @@ -0,0 +1,34 @@ +# Feature Flag Strategy + +Feature flags prevent future modules from appearing as broken V1 features. + +## Required Flags +- `ENABLE_GRB_CONNECTOR` +- `ENABLE_OSM_CONNECTOR` +- `ENABLE_SENTINEL_LAB` +- `ENABLE_SAM_SEGMENTATION` +- `ENABLE_YOLO_DETECTION` +- `ENABLE_LIDAR_LAB` +- `ENABLE_TRAINING_STUDIO` +- `ENABLE_QGIS_EXPORT` + +## V1 Defaults +```env +ENABLE_GRB_CONNECTOR=true +ENABLE_OSM_CONNECTOR=true +ENABLE_SENTINEL_LAB=false +ENABLE_SAM_SEGMENTATION=false +ENABLE_YOLO_DETECTION=true +ENABLE_LIDAR_LAB=false +ENABLE_TRAINING_STUDIO=false +ENABLE_QGIS_EXPORT=false +``` + +## UI Behavior +Disabled features may be visible as roadmap cards, but must not look like broken tools. They should show: +- why disabled; +- what dependency is missing; +- which milestone enables them. + +## Backend Behavior +Disabled endpoints return a typed `FEATURE_DISABLED` error, not 404 and not silent success. diff --git a/docs/18-ultra-prep/FINAL_PRE_CODEX_CHECKLIST.md b/docs/18-ultra-prep/FINAL_PRE_CODEX_CHECKLIST.md new file mode 100644 index 00000000..f61d8292 --- /dev/null +++ b/docs/18-ultra-prep/FINAL_PRE_CODEX_CHECKLIST.md @@ -0,0 +1,14 @@ +# Final Pre-Codex Checklist + +Before starting tomorrow's Codex build: + +- [ ] Extract latest full zip. +- [ ] Open repo root in Codex environment. +- [ ] Confirm `.env.example` exists. +- [ ] Confirm Docker/PostGIS plan is present. +- [ ] Give Codex `prompts/codex/M10_MASTER_AUTONOMOUS_PROMPT.md`. +- [ ] Tell Codex to execute only one pass at a time. +- [ ] After each pass, require pass report. +- [ ] Reject pass if it changes frozen scope. +- [ ] Reject pass if it hides missing functionality behind fake success. +- [ ] Save output as new update/full zip after meaningful passes. diff --git a/docs/18-ultra-prep/FRONTEND_STATE_MACHINE.md b/docs/18-ultra-prep/FRONTEND_STATE_MACHINE.md new file mode 100644 index 00000000..ffa79c0d --- /dev/null +++ b/docs/18-ultra-prep/FRONTEND_STATE_MACHINE.md @@ -0,0 +1,30 @@ +# Frontend State Machine + +Every async module must model these states explicitly: + +- `idle` +- `loading` +- `empty` +- `ready` +- `error` +- `unavailable` +- `disabled` + +## Dataset Card +- idle: not selected +- loading: metadata extraction running +- empty: no datasets uploaded +- ready: dataset metadata available +- error: upload/extraction failed +- unavailable: operation unsupported for this dataset type +- disabled: feature flag off + +## Analysis Run +- queued +- running +- succeeded +- failed +- cancelled + +## UI Rule +Never leave users with a blank panel. Every panel must explain what is happening or what to do next. diff --git a/docs/18-ultra-prep/GEOMETRY_CONTRACTS.md b/docs/18-ultra-prep/GEOMETRY_CONTRACTS.md new file mode 100644 index 00000000..79304a94 --- /dev/null +++ b/docs/18-ultra-prep/GEOMETRY_CONTRACTS.md @@ -0,0 +1,29 @@ +# Geometry Contracts + +## Internal Geometry Standard +- Database geometries are stored in PostGIS. +- Default storage CRS: EPSG:31370 for Belgian projected operations where appropriate, or EPSG:4326 for API interchange. +- API GeoJSON is always EPSG:4326 unless explicitly stated. +- Area calculations must use projected CRS, not raw WGS84 degrees. + +## Accepted Area Input +- GeoJSON Polygon +- GeoJSON MultiPolygon +- drawn polygon from frontend +- future: municipality selection + +## Validation Rules +- polygon must be closed; +- polygon must be valid; +- polygon must have non-zero area; +- self-intersections must be rejected or repaired explicitly; +- extremely large areas must require confirmation or be rejected by configured max area. + +## Output Rules +- include `crs` metadata if transformed; +- include computed area in square meters; +- include bounds; +- include geometry validity status. + +## Common Trap +Never compute area or distance in degrees. diff --git a/docs/18-ultra-prep/KNOWN_LIMITATIONS_TEMPLATE.md b/docs/18-ultra-prep/KNOWN_LIMITATIONS_TEMPLATE.md new file mode 100644 index 00000000..b30372fa --- /dev/null +++ b/docs/18-ultra-prep/KNOWN_LIMITATIONS_TEMPLATE.md @@ -0,0 +1,16 @@ +# Known Limitations Template + +Use this file format after each major build. + +## Current Limitations +| Area | Limitation | Impact | Workaround | Target Milestone | +|---|---|---|---|---| +| Detection | YOLO weights not configured | Demo adapter only | Configure local weights | V1.x | + +## Data Limitations +- GRB live connector may require endpoint details or local downloads. +- Sentinel automation is V2. +- LiDAR is outside V1. + +## Technical Debt +List only real debt, not unfinished planned scope. diff --git a/docs/18-ultra-prep/MODEL_ADAPTER_GUIDE.md b/docs/18-ultra-prep/MODEL_ADAPTER_GUIDE.md new file mode 100644 index 00000000..2c0054ba --- /dev/null +++ b/docs/18-ultra-prep/MODEL_ADAPTER_GUIDE.md @@ -0,0 +1,34 @@ +# Model Adapter Guide + +## Purpose +Detection and segmentation must be implemented behind stable adapters so V1 can support demo/local model flows while remaining ready for YOLO/SAM integration. + +## Detection Adapter Interface +```python +class DetectionAdapter: + name: str + supported_classes: list[str] + def is_available(self) -> bool: ... + def predict(self, image_tile, parameters) -> list[DetectionResult]: ... +``` + +## Detection Result +Fields: +- class_name +- confidence +- bbox_pixel +- bbox_geo optional after georeferencing +- source_tile +- metadata + +## Segmentation Adapter Interface +```python +class SegmentationAdapter: + name: str + supported_classes: list[str] + def is_available(self) -> bool: ... + def segment(self, image_tile, parameters) -> list[SegmentationResult]: ... +``` + +## V1 Rule +If real YOLO weights are not configured, use a demo adapter only when clearly labelled as demo and never present it as production AI. diff --git a/docs/18-ultra-prep/OBSERVABILITY_PLAN.md b/docs/18-ultra-prep/OBSERVABILITY_PLAN.md new file mode 100644 index 00000000..b12fda29 --- /dev/null +++ b/docs/18-ultra-prep/OBSERVABILITY_PLAN.md @@ -0,0 +1,34 @@ +# Observability Plan + +## Required Logs +- request start/end with trace id; +- dataset upload registration; +- metadata extraction status; +- job lifecycle transitions; +- model adapter selection; +- inference start/end; +- QA/QC matching summary; +- export creation. + +## Required Health Checks +- API health; +- database connection; +- PostGIS extension availability; +- storage write check; +- optional Redis check; +- optional model registry check. + +## Diagnostics Endpoint +`GET /system/diagnostics` + +Should return safe non-secret status: +```json +{ + "api": "ok", + "database": "ok", + "postgis": "ok", + "storage": "ok", + "redis": "unconfigured", + "models": "unconfigured" +} +``` diff --git a/docs/18-ultra-prep/PERFORMANCE_BUDGETS.md b/docs/18-ultra-prep/PERFORMANCE_BUDGETS.md new file mode 100644 index 00000000..1515a062 --- /dev/null +++ b/docs/18-ultra-prep/PERFORMANCE_BUDGETS.md @@ -0,0 +1,25 @@ +# Performance Budgets + +## API +- health endpoint: < 200ms local +- project list: < 500ms for 100 projects +- dataset metadata: < 500ms after extraction +- synchronous upload response: should register file quickly and hand off heavy work to job queue + +## Raster +- never load large rasters fully into memory for preview; +- prefer windowed reads and overviews; +- large tiling must run as background job. + +## Vector +- use spatial indexes for intersection/QA operations; +- simplify only for display, not source-of-truth unless explicitly stored as derived layer. + +## Frontend +- initial app load should not require heavy GIS data; +- map layers should be lazy-loaded; +- large GeoJSON should be paged, tiled, or simplified. + +## Jobs +Long-running operations must have status transitions: +`queued -> running -> succeeded|failed|cancelled`. diff --git a/docs/18-ultra-prep/QA_QC_MATCHING_ALGORITHM.md b/docs/18-ultra-prep/QA_QC_MATCHING_ALGORITHM.md new file mode 100644 index 00000000..696ae30c --- /dev/null +++ b/docs/18-ultra-prep/QA_QC_MATCHING_ALGORITHM.md @@ -0,0 +1,41 @@ +# QA/QC Matching Algorithm + +## Goal +Compare AI detections/segmentations with a reference layer such as GRB buildings. + +## Inputs +- predicted polygons +- reference polygons +- class filter +- IoU threshold, default 0.5 + +## Steps +1. Validate CRS alignment. +2. Reproject to analytical CRS. +3. Build spatial indexes. +4. For every predicted polygon, find candidate reference polygons by bbox intersection. +5. Compute IoU for candidates. +6. Assign best match above threshold greedily, one reference per prediction. +7. Count true positives, false positives, false negatives. +8. Compute precision, recall, F1, mean IoU. +9. Produce unmatched prediction layer and unmatched reference layer. + +## Metrics +```text +precision = TP / (TP + FP) +recall = TP / (TP + FN) +f1 = 2 * precision * recall / (precision + recall) +IoU = intersection_area / union_area +``` + +## Output Layers +- matched predictions +- false positives +- false negatives +- low IoU matches + +## Edge Cases +- empty predictions and empty reference: score should be explicit `no_objects` not perfect success; +- empty predictions with reference: recall 0; +- predictions with empty reference: precision 0; +- invalid polygons must be repaired or excluded with warning. diff --git a/docs/18-ultra-prep/README.md b/docs/18-ultra-prep/README.md new file mode 100644 index 00000000..d946bcf0 --- /dev/null +++ b/docs/18-ultra-prep/README.md @@ -0,0 +1,13 @@ +# M10 Ultra Preparation Pack + +This folder contains the additional control layer for making GeoIntel as autonomous-build-ready as possible for Codex. + +Purpose: +- remove hidden architecture ambiguity; +- give Codex concrete implementation contracts; +- define allowed freedom versus frozen decisions; +- provide validation and recovery procedures; +- prevent common GIS/AI/web-app regressions; +- make each build pass reviewable without manual guessing. + +M10 does not replace the previous documentation. It adds the final execution scaffolding around it. diff --git a/docs/18-ultra-prep/RELEASE_GATE_V1.md b/docs/18-ultra-prep/RELEASE_GATE_V1.md new file mode 100644 index 00000000..4adcc97b --- /dev/null +++ b/docs/18-ultra-prep/RELEASE_GATE_V1.md @@ -0,0 +1,27 @@ +# V1 Release Gate + +GeoIntel V1 is releasable when all gates pass. + +## Product Gate +- [ ] User can create project. +- [ ] User can create/select area. +- [ ] User can upload at least one vector dataset. +- [ ] User can upload/register at least one raster dataset. +- [ ] User can see datasets on map or metadata panel. +- [ ] User can run a detection workflow or labelled demo adapter. +- [ ] User can compare detections to reference polygons. +- [ ] User can export GeoJSON. + +## Engineering Gate +- [ ] Backend starts from clean checkout. +- [ ] Frontend builds from clean checkout. +- [ ] Database migrations apply. +- [ ] Tests run. +- [ ] Smoke script passes. +- [ ] Docs explain setup. + +## Integrity Gate +- [ ] No fake success states. +- [ ] CRS assumptions documented. +- [ ] Feature-disabled states are clear. +- [ ] Known limitations are listed. diff --git a/docs/18-ultra-prep/REPO_HYGIENE_RULES.md b/docs/18-ultra-prep/REPO_HYGIENE_RULES.md new file mode 100644 index 00000000..042fab99 --- /dev/null +++ b/docs/18-ultra-prep/REPO_HYGIENE_RULES.md @@ -0,0 +1,28 @@ +# Repo Hygiene Rules + +## Do Commit +- source code; +- documentation; +- small fixtures; +- config examples; +- migration files; +- test data under size limits. + +## Do Not Commit +- `.env` with secrets; +- large rasters; +- model weights; +- generated cache; +- local database volumes; +- node_modules; +- Python virtualenvs; +- personal paths. + +## Naming +- backend modules use snake_case; +- frontend components use PascalCase; +- docs use uppercase topic names or numbered folders; +- fixtures describe region and purpose. + +## Generated Outputs +Generated exports should go under `exports/` and be gitignored unless they are tiny documented sample fixtures. diff --git a/docs/18-ultra-prep/SECURITY_AND_SECRET_HANDLING.md b/docs/18-ultra-prep/SECURITY_AND_SECRET_HANDLING.md new file mode 100644 index 00000000..688c9d3b --- /dev/null +++ b/docs/18-ultra-prep/SECURITY_AND_SECRET_HANDLING.md @@ -0,0 +1,26 @@ +# Security and Secret Handling + +## Secrets +Never commit API keys, tokens, model credentials, STAC credentials, database passwords, or private URLs. + +## Environment Variables +All secrets must be loaded from `.env` or deployment environment. + +## File Upload Safety +- limit accepted extensions; +- validate MIME/type where possible; +- store uploads outside source directories; +- generate server-side filenames; +- never execute uploaded files; +- reject path traversal. + +## External Connectors +- log endpoint names but not credentials; +- timeout external requests; +- cache responses where appropriate; +- show connector status in UI. + +## AI/Model Safety +- model files must be treated as artifacts; +- do not auto-download arbitrary executable code; +- keep model registry metadata separate from weights. diff --git a/docs/18-ultra-prep/UI_COPY_BANK.md b/docs/18-ultra-prep/UI_COPY_BANK.md new file mode 100644 index 00000000..7aedac37 --- /dev/null +++ b/docs/18-ultra-prep/UI_COPY_BANK.md @@ -0,0 +1,37 @@ +# UI Copy Bank + +## Product Language +Use: +- GeoAI Workbench +- Dataset +- Analysis Run +- Reference Layer +- Detection Result +- QA/QC +- Export + +Avoid: +- magic AI +- automatic truth +- perfect detection +- black box conclusions + +## Empty States +### No Projects +"Start a GeoAI project by selecting an area in the Kempen or opening the prepared demo scenario." + +### No Datasets +"Upload a GeoTIFF, GeoJSON, Shapefile, or use a connector to add reference data." + +### No Analysis Runs +"Run a raster, vector, detection, segmentation, or QA/QC analysis to generate geospatial outputs." + +## Error Suggestions +### CRS Missing +"This dataset has no detected CRS. Add CRS metadata before using it in spatial operations." + +### Model Unavailable +"No detection model is configured. Configure YOLO weights or use the labelled demo adapter." + +### Reference Missing +"QA/QC requires a reference layer such as GRB buildings or a local reference GeoJSON." diff --git a/docs/19-architect-audit/ARCHITECT_AUDIT_REPORT_M11.md b/docs/19-architect-audit/ARCHITECT_AUDIT_REPORT_M11.md new file mode 100644 index 00000000..1210e5a7 --- /dev/null +++ b/docs/19-architect-audit/ARCHITECT_AUDIT_REPORT_M11.md @@ -0,0 +1,81 @@ +# M11 Architect Audit Report + +## Audit summary + +The repo has strong breadth: product docs, architecture docs, API plans, build prompts, fixtures, smoke scripts and milestone handoffs exist. The main risk is no longer lack of documentation; it is competing documentation and missing hierarchy. + +## Findings + +### Finding 1 — Too many competing start points + +There are multiple handoff files and prompt files from M0 through M10. This is useful historically but dangerous for Codex. + +Resolution: + +- added `docs/00-start/START_HERE.md` as canonical entry point; +- added decision precedence; +- older files remain historical. + +### Finding 2 — Governance rules were implied, not constitutional + +Previous docs often said what to build, but not which principles override conflicts. + +Resolution: + +- added GeoIntel Constitution; +- added Architecture Invariants; +- added Forbidden Decisions. + +### Finding 3 — Domain language needed locking + +Terms like Dataset, Layer, Analysis, Detection and Segmentation could be implemented inconsistently. + +Resolution: + +- added Canonical Domain Models. + +### Finding 4 — State machines needed canonical names + +Multiple docs referenced statuses, but a single source of truth was missing. + +Resolution: + +- added `STATE_MACHINES.md`. + +### Finding 5 — Build order needed dependency graph, not only phase lists + +Codex needs to know not only tasks, but why one task must precede another. + +Resolution: + +- added Build Order Dependency Graph. + +### Finding 6 — Golden paths needed stronger protection + +The desired demo was clear, but not protected as regression-critical workflows. + +Resolution: + +- added `GOLDEN_PATHS.md`. + +### Finding 7 — Error handling needed stable codes + +The frontend and backend need a common error taxonomy. + +Resolution: + +- added `ERROR_CATALOG.md`. + +## Remaining recommendations for M12+ + +1. Generate actual starter code skeleton aligned with the docs. +2. Add OpenAPI YAML snapshot once backend exists. +3. Add migration files once database models exist. +4. Convert golden paths into executable tests. +5. Add real small GeoTIFF fixture if licensing/file size allows. +6. Add GRB WFS proof-of-concept adapter after local fixture path is stable. +7. Add QGIS export verification later. + +## Architect verdict + +The repo is now suitable for a disciplined autonomous Codex build, provided Codex starts from `docs/00-start/START_HERE.md` and treats older handoff docs as historical. diff --git a/docs/20-run-readiness/CODEX_TOMORROW_RUNBOOK.md b/docs/20-run-readiness/CODEX_TOMORROW_RUNBOOK.md new file mode 100644 index 00000000..d5dcab1a --- /dev/null +++ b/docs/20-run-readiness/CODEX_TOMORROW_RUNBOOK.md @@ -0,0 +1,86 @@ +# Codex Tomorrow Runbook + +## Before starting + +Use the latest full repo zip, extract it, and open the repository root. + +Read: + +1. `CODEX_START.md` +2. `docs/00-start/START_HERE.md` +3. `docs/20-run-readiness/RUN_READINESS_FINAL.md` +4. `docs/20-run-readiness/PASS_SEQUENCE_FINAL.md` +5. `prompts/codex/final/DAY_1_MASTER_PROMPT.md` + +## Operating rhythm + +For each pass: + +1. Restate the pass goal. +2. List files expected to change. +3. Implement only the pass scope. +4. Run the relevant commands. +5. Fix failures within scope. +6. Update status docs. +7. Produce pass report. + +## Do not allow Codex to drift into + +- redesigning the product; +- adding auth/multi-user early; +- building a chatbot first; +- implementing LiDAR first; +- building a full report generator first; +- replacing FastAPI/PostGIS/React; +- using only demo data without real-data-ready interfaces; +- hiding broken states behind TODO comments. + +## Recommended first commands + +```bash +find . -maxdepth 3 -type f | sort | head -200 +bash scripts/check_repo_structure.sh +python scripts/smoke_docs.py +python scripts/validate_fixtures.py +python scripts/preimplementation_audit.py +``` + +If scripts fail because permissions are missing, run them with `bash scriptname.sh` or `python scriptname.py` rather than changing architecture. + +## Pass report template + +```text +Pass: +Scope: +Files changed: +Commands run: +Tests/smoke checks: +What works now: +Known limitations: +Architecture invariants touched: +Golden path status: +Next pass: +``` + +## Stop conditions + +Stop and create a proposal note instead of coding when: + +- a required architecture decision is missing; +- a chosen dependency conflicts with the dependency policy; +- a requested change violates an invariant; +- API contracts require breaking changes; +- a geospatial assumption is unclear and affects data correctness. + +## Success definition for tomorrow + +A successful day does not require every advanced module. A successful day means: + +- backend foundation exists; +- database/domain foundation exists; +- project/area/dataset flow works; +- fixture vector import works; +- QA/QC can run on fixture buildings; +- GeoJSON export exists; +- minimal UI can display the workflow or at least consume the API; +- docs and tests reflect reality. diff --git a/docs/20-run-readiness/IMPLEMENTATION_READINESS_CHECKLIST.md b/docs/20-run-readiness/IMPLEMENTATION_READINESS_CHECKLIST.md new file mode 100644 index 00000000..2724e8f6 --- /dev/null +++ b/docs/20-run-readiness/IMPLEMENTATION_READINESS_CHECKLIST.md @@ -0,0 +1,40 @@ +# Implementation Readiness Checklist + +## Repo readiness + +- [ ] `CODEX_START.md` exists at repo root. +- [ ] `docs/00-start/START_HERE.md` exists. +- [ ] `docs/20-run-readiness/RUN_READINESS_FINAL.md` exists. +- [ ] `prompts/codex/final/DAY_1_MASTER_PROMPT.md` exists. +- [ ] `scripts/preimplementation_audit.py` runs. +- [ ] Demo fixtures validate. + +## Architecture readiness + +- [ ] Product identity is fixed. +- [ ] Stack is fixed. +- [ ] Database choice is fixed. +- [ ] CRS/GIS standards are fixed. +- [ ] State machines are fixed. +- [ ] Golden paths are fixed. +- [ ] Forbidden decisions are explicit. + +## Build readiness + +- [ ] Pass order is known. +- [ ] First vertical slice is known. +- [ ] V1 exclusions are known. +- [ ] Definition of Done is known. +- [ ] Smoke tests are known. +- [ ] Pass report format is known. + +## Release readiness target + +- [ ] Backend imports. +- [ ] Health endpoint works. +- [ ] Database models exist. +- [ ] Project/Area/Dataset APIs exist. +- [ ] Fixtures can be imported. +- [ ] QA/QC metrics can be computed. +- [ ] GeoJSON can be exported. +- [ ] Minimal UI can show results. diff --git a/docs/20-run-readiness/PASS_SEQUENCE_FINAL.md b/docs/20-run-readiness/PASS_SEQUENCE_FINAL.md new file mode 100644 index 00000000..bea57d45 --- /dev/null +++ b/docs/20-run-readiness/PASS_SEQUENCE_FINAL.md @@ -0,0 +1,206 @@ +# Final Codex Pass Sequence + +Codex must execute these passes in order. Do not skip ahead unless the previous pass is complete and smoke-checked. + +## Pass 0 — Repo audit and bootstrap + +Goal: confirm the repo can be used as an implementation workspace. + +Deliverables: + +- confirm folder structure; +- install/dependency plan selected; +- create missing backend/frontend scaffolding only if absent; +- no product feature work yet; +- run docs smoke scripts. + +Exit criteria: + +- `CODEX_START.md` is acknowledged; +- canonical docs are read; +- no conflicting start path remains unaddressed in notes. + +## Pass 1 — Backend application foundation + +Goal: create a FastAPI app that imports and serves health/status endpoints. + +Deliverables: + +- `backend/app/main.py`; +- settings/config module; +- API router structure; +- health endpoint; +- error envelope helper; +- minimal tests. + +Exit criteria: + +- backend imports successfully; +- health endpoint test passes; +- no database required yet. + +## Pass 2 — Database and domain foundation + +Goal: establish SQLAlchemy/Alembic/PostGIS-ready domain models. + +Deliverables: + +- DB config; +- migration skeleton; +- models for Project, Area, Dataset, AnalysisRun, Detection, Metric, Export; +- schemas for request/response; +- geometry storage strategy documented in code comments and docs. + +Exit criteria: + +- migrations can be generated/applied in local environment; +- models match canonical domain docs; +- no geometry stored as arbitrary string when PostGIS type is available. + +## Pass 3 — Project and Area API + +Goal: implement the first user-managed domain objects. + +Deliverables: + +- project CRUD; +- area CRUD; +- geometry validation; +- area calculation; +- API tests; +- response envelope compliance. + +Exit criteria: + +- create/list/read project works; +- create/list/read area works; +- invalid geometry returns controlled error. + +## Pass 4 — Dataset manager foundation + +Goal: register datasets and extract basic metadata. + +Deliverables: + +- dataset upload/registration endpoint; +- metadata schema; +- storage path convention; +- fixture registration path; +- dataset state machine implemented. + +Exit criteria: + +- fixture vector dataset can be registered; +- dataset moves through valid states; +- failed validation is explicit. + +## Pass 5 — Vector processing core + +Goal: load reference polygons and predicted detection polygons. + +Deliverables: + +- GeoJSON import; +- geometry normalization; +- CRS handling; +- feature count and bounds metrics; +- persistence of reference and predicted layers. + +Exit criteria: + +- reference buildings fixture imports; +- predicted buildings fixture imports; +- invalid GeoJSON is rejected safely. + +## Pass 6 — QA/QC engine foundation + +Goal: compare predicted detections against reference polygons. + +Deliverables: + +- IoU/overlap matching; +- precision, recall, F1; +- false positive/false negative outputs; +- quality check records; +- tests with demo fixtures. + +Exit criteria: + +- expected demo metrics are reproduced or documented; +- algorithm is deterministic; +- matching thresholds are configurable but defaulted. + +## Pass 7 — GeoJSON export + +Goal: export geospatial outputs from the system. + +Deliverables: + +- export endpoint; +- export records; +- GeoJSON FeatureCollection output; +- export validation; +- smoke test. + +Exit criteria: + +- detections export as valid GeoJSON; +- QA/QC outputs can be exported; +- no broken geometry emitted. + +## Pass 8 — Frontend workbench shell + +Goal: create a minimal but coherent UI. + +Deliverables: + +- React/TypeScript app shell; +- route structure; +- project list/detail; +- map workbench placeholder with layer panel; +- API client; +- loading/error/empty states. + +Exit criteria: + +- frontend builds; +- health/status can be displayed; +- no hardcoded permanent fake data except clearly marked demo fixtures. + +## Pass 9 — Map and metrics integration + +Goal: show the vertical slice visually. + +Deliverables: + +- render area/reference/prediction layers; +- metrics panel; +- QA/QC result cards; +- export action; +- basic style guide compliance. + +Exit criteria: + +- demo Geel workflow can be clicked through; +- layer visibility/opacity works at minimum level; +- UI does not obscure the map or results. + +## Pass 10 — Stabilization and release candidate + +Goal: make the first vertical slice releasable. + +Deliverables: + +- smoke tests; +- docs update; +- known limitations; +- changelog; +- run instructions; +- regression checklist. + +Exit criteria: + +- backend tests pass; +- frontend build passes; +- demo workflow documented; +- no architecture invariant violated. diff --git a/docs/20-run-readiness/REPO_CONFLICT_RESOLUTION.md b/docs/20-run-readiness/REPO_CONFLICT_RESOLUTION.md new file mode 100644 index 00000000..20482502 --- /dev/null +++ b/docs/20-run-readiness/REPO_CONFLICT_RESOLUTION.md @@ -0,0 +1,37 @@ +# Repo Conflict Resolution + +The repo contains many milestone documents. This is intentional, but implementation must not follow conflicting instructions. + +## Conflict categories + +### Product conflict + +Example: one document says GeoIntel is a dashboard, another says GeoIntel is a GeoAI Workbench. + +Resolution: follow `GEOINTEL_CONSTITUTION.md`. + +### Stack conflict + +Example: one document suggests another backend framework. + +Resolution: follow ADRs and `ARCHITECTURE_INVARIANTS.md`. + +### Scope conflict + +Example: one older document prioritizes reports before QA/QC. + +Resolution: follow `PASS_SEQUENCE_FINAL.md` and `V1_SCOPE_FREEZE.md`. + +### API conflict + +Example: endpoint naming differs between older docs. + +Resolution: follow `API_CONTRACTS.md`, `API_CONTRACT_FREEZE_M2.md`, and response envelope contracts. + +### CRS/geospatial conflict + +Resolution: follow `GIS_STANDARDS.md`, `CRS_POLICY.md`, and canonical geometry contracts. + +## Rule + +When implementing, newer canonical control docs are binding. Older milestone docs are explanatory only. diff --git a/docs/20-run-readiness/RUN_READINESS_FINAL.md b/docs/20-run-readiness/RUN_READINESS_FINAL.md new file mode 100644 index 00000000..7f19c0df --- /dev/null +++ b/docs/20-run-readiness/RUN_READINESS_FINAL.md @@ -0,0 +1,75 @@ +# M12 Run Readiness Final Audit + +## Purpose + +This document converts the large GeoIntel preparation set into a practical final execution layer for Codex. The repo contains many useful milestone documents, but the implementation run must follow one canonical path. + +## Final readiness status + +GeoIntel is ready to start implementation when Codex follows the canonical path below and does not treat older milestone documents as competing instructions. + +## Canonical control stack + +1. Product identity: `docs/governance/GEOINTEL_CONSTITUTION.md` +2. Architecture invariants: `docs/governance/ARCHITECTURE_INVARIANTS.md` +3. Forbidden decisions: `docs/governance/FORBIDDEN_DECISIONS.md` +4. Decision precedence: `docs/governance/DECISION_PRECEDENCE.md` +5. Domain models: `docs/specs/CANONICAL_DOMAIN_MODELS.md` +6. GIS standards: `docs/specs/GIS_STANDARDS.md` +7. Raster standards: `docs/specs/RASTER_STANDARDS.md` +8. State machines: `docs/specs/STATE_MACHINES.md` +9. Golden paths: `docs/workflows/GOLDEN_PATHS.md` +10. Build dependency graph: `docs/build/BUILD_ORDER_DEPENDENCY_GRAPH.md` +11. Final pass sequence: `docs/20-run-readiness/PASS_SEQUENCE_FINAL.md` +12. Day 1 master prompt: `prompts/codex/final/DAY_1_MASTER_PROMPT.md` + +## Older milestone documents + +Older M0-M11 documents remain valuable as supporting context. They are not deleted because they contain useful details, but they must not override the canonical control stack. + +When in doubt, Codex must follow: + +`CODEX_START.md` → `START_HERE.md` → M12 run-readiness docs → governance/specs → implementation prompts. + +## Final V1 scope + +V1 is the foundation GeoAI Workbench vertical slice. + +V1 includes: + +- backend skeleton; +- database models and migrations; +- project CRUD; +- area CRUD with geometry validation; +- dataset registration/upload metadata; +- vector import using fixtures first; +- raster metadata extraction where libraries are available; +- reference polygon loading; +- predicted detection import; +- QA/QC matching against reference polygons; +- metric persistence; +- GeoJSON export; +- minimal frontend workbench; +- status/error handling; +- smoke tests. + +V1 excludes: + +- live production GRB sync as a blocker; +- heavy model training; +- full SAM/YOLO production inference as a blocker; +- LiDAR processing; +- MLOps registry implementation; +- QGIS plugin; +- multi-user permissions; +- advanced PDF report generation. + +## Implementation policy + +Codex may improve implementation details, UX clarity, tests, types, helper abstractions and documentation. Codex must not alter product identity, stack, state machines, response envelope, database choice, CRS policy or golden path priority without an ADR proposal. + +## Final release target + +The first release target is not a complete GeoAI platform. It is a stable, demonstrable vertical slice proving that GeoIntel can move geospatial data through the core pipeline: + +`data → processing → geospatial output → QA/QC → export`. diff --git a/docs/30-codex-optimization/CODEX_OPTIMIZATION_OVERVIEW.md b/docs/30-codex-optimization/CODEX_OPTIMIZATION_OVERVIEW.md new file mode 100644 index 00000000..36a2a698 --- /dev/null +++ b/docs/30-codex-optimization/CODEX_OPTIMIZATION_OVERVIEW.md @@ -0,0 +1,43 @@ +# M13 Codex Optimization Overview + +M13 adds an optimization layer on top of the M12 run-ready repository. The goal is to improve Codex output quality by making the implementation process more constrained where correctness matters, while still allowing local improvements inside documented boundaries. + +## M13 purpose + +Codex should be able to: + +1. choose the correct entry document without ambiguity; +2. work pass-by-pass without re-planning the whole product; +3. use reusable skills for repeated implementation patterns; +4. respect token and context budgets; +5. avoid secrets leakage; +6. split work safely across parallel agents or worktrees; +7. self-review every implementation pass against explicit gates; +8. escalate only real blockers. + +## New canonical Codex flow + +1. Read `CODEX_START.md`. +2. Read `docs/30-codex-optimization/CODEX_RUN_CHECKLIST.md`. +3. Read `docs/30-codex-optimization/PROMPT_DISCIPLINE.md`. +4. Select the active pass from `docs/20-run-readiness/PASS_SEQUENCE_FINAL.md`. +5. Select only the relevant skill from `skills/`. +6. Implement the smallest coherent pass. +7. Run `make readiness` plus module-specific checks. +8. Fill in `docs/CODEX_EXECUTION_LOG.md`. +9. Produce a pass summary using `prompts/codex/m13/PASS_COMPLETION_REPORT_PROMPT.md`. + +## M13 rule hierarchy + +If instructions conflict, follow this order: + +1. `docs/governance/GEOINTEL_CONSTITUTION.md` +2. `docs/governance/ARCHITECTURE_INVARIANTS.md` +3. `docs/governance/FORBIDDEN_DECISIONS.md` +4. `CODEX_START.md` +5. `docs/30-codex-optimization/*` +6. active pass prompt +7. module documentation +8. historical milestone documents + +Historical M0-M12 documents remain useful, but they must not override governance, invariants, M12 final readiness, or M13 optimization rules. diff --git a/docs/30-codex-optimization/CODEX_RUN_CHECKLIST.md b/docs/30-codex-optimization/CODEX_RUN_CHECKLIST.md new file mode 100644 index 00000000..b720dfb0 --- /dev/null +++ b/docs/30-codex-optimization/CODEX_RUN_CHECKLIST.md @@ -0,0 +1,48 @@ +# Codex Run Checklist + +Use this checklist at the start and end of every Codex session. + +## Before starting + +- [ ] Confirm the current branch/worktree. +- [ ] Read `CODEX_START.md`. +- [ ] Read `docs/00-start/START_HERE.md`. +- [ ] Read `docs/30-codex-optimization/CODEX_OPTIMIZATION_OVERVIEW.md`. +- [ ] Read `docs/30-codex-optimization/PROMPT_DISCIPLINE.md`. +- [ ] Read only the active pass prompt. +- [ ] Read the relevant skill from `skills/`. +- [ ] Run `make readiness` if the repository has shell/python available. +- [ ] Identify the exact files expected to change. + +## During implementation + +- [ ] Keep changes scoped to the active pass. +- [ ] Do not introduce new frameworks without ADR. +- [ ] Do not change API contracts unless the active pass explicitly requires it. +- [ ] Do not create permanent mock-only implementations. +- [ ] Prefer small, verifiable service boundaries. +- [ ] Keep GIS units and CRS assumptions explicit. +- [ ] Record real limitations as limitations, not hidden TODOs. + +## Before finishing + +- [ ] Run relevant tests/smoke checks. +- [ ] Run `make readiness` when possible. +- [ ] Update `docs/CODEX_EXECUTION_LOG.md`. +- [ ] Update `CHANGELOG.md`. +- [ ] List changed files. +- [ ] List what works. +- [ ] List what remains incomplete. +- [ ] List any deliberate deviations from docs. +- [ ] Produce a self-review scorecard. + +## Stop conditions + +Stop and report instead of guessing when: + +- a governance invariant would be violated; +- a required external credential is missing; +- a data source license or endpoint is unclear; +- a destructive migration would be needed; +- tests indicate a core regression; +- implementation requires a new major dependency not already approved. diff --git a/docs/30-codex-optimization/CODEX_SKILLS_INDEX.md b/docs/30-codex-optimization/CODEX_SKILLS_INDEX.md new file mode 100644 index 00000000..92e056d5 --- /dev/null +++ b/docs/30-codex-optimization/CODEX_SKILLS_INDEX.md @@ -0,0 +1,28 @@ +# Codex Skills Index + +Skills are reusable implementation workflows. Use one skill per implementation pass unless the task clearly spans two tightly coupled areas. + +## Available skills + +| Skill | Use when | +|---|---| +| `geoai-backend-build` | creating FastAPI services, routers, schemas, domain logic | +| `postgis-migration` | creating database models, Alembic migrations, geometry fields | +| `raster-pipeline` | implementing Rasterio/GDAL-style raster metadata, clip, tile, indices | +| `vector-processing` | implementing GeoPandas/Shapely vector operations and exports | +| `frontend-maplibre-workbench` | building React/MapLibre pages, layers, UI states | +| `qaqc-review` | implementing IoU, precision/recall, false positives/negatives | +| `codex-pass-review` | final self-review, regression scan and pass completion reports | + +## Skill usage protocol + +1. Read active pass prompt. +2. Select the closest skill. +3. Read `skills//SKILL.md`. +4. Implement using the skill checklist. +5. Run module-specific checks. +6. End with the skill's required report fields. + +## Skill conflict rule + +If a skill conflicts with governance docs, governance wins. Update the skill later; do not violate governance. diff --git a/docs/30-codex-optimization/M13_HANDOFF_SUMMARY.md b/docs/30-codex-optimization/M13_HANDOFF_SUMMARY.md new file mode 100644 index 00000000..f4426292 --- /dev/null +++ b/docs/30-codex-optimization/M13_HANDOFF_SUMMARY.md @@ -0,0 +1,29 @@ +# M13 Handoff Summary — Codex Optimization Pack + +M13 adds the final layer intended to improve tomorrow's Codex execution quality. + +## Added + +- Codex optimization overview. +- Codex run checklist. +- Prompt discipline rules. +- Token/context budget policy. +- Secrets and environment policy. +- Parallel agent strategy. +- Codex skills index. +- Reusable skills under `skills/`. +- M13 day-one optimized master prompt. +- Pass completion report prompt. +- Updated readiness checks for M13 assets. + +## Why this matters + +M12 made the repository run-ready. M13 makes the repository easier for Codex to execute correctly without wasting context, drifting from contracts, leaking secrets, or making undocumented architecture choices. + +## Recommended next action + +Use: + +- `prompts/codex/m13/DAY_1_OPTIMIZED_MASTER_PROMPT.md` + +for the first serious Codex build run. diff --git a/docs/30-codex-optimization/PARALLEL_AGENT_STRATEGY.md b/docs/30-codex-optimization/PARALLEL_AGENT_STRATEGY.md new file mode 100644 index 00000000..1327aa99 --- /dev/null +++ b/docs/30-codex-optimization/PARALLEL_AGENT_STRATEGY.md @@ -0,0 +1,87 @@ +# Parallel Agent Strategy + +GeoIntel can use multiple Codex agents only when their work areas do not conflict. + +## Safe parallel tracks + +### Track A — Backend foundation + +Allowed paths: + +- `backend/` +- `tests/backend/` +- database docs when needed + +Do not touch frontend except API contract comments. + +### Track B — Frontend shell + +Allowed paths: + +- `frontend/` +- `tests/frontend/` +- UI docs when needed + +Do not change API contracts without coordination. + +### Track C — Documentation/runbooks + +Allowed paths: + +- `docs/` +- `prompts/` +- `skills/` +- `checklists/` + +Do not change implementation code. + +### Track D — Fixtures/tests + +Allowed paths: + +- `tests/fixtures/` +- `fixtures/` +- `scripts/` +- test docs + +Do not alter production services except to expose stable test hooks. + +## Unsafe parallel work + +Do not run parallel agents on: + +- database schema plus API schemas unless coordinated; +- API contracts plus frontend client generation unless coordinated; +- storage paths plus dataset manager unless coordinated; +- detection output schemas plus QA/QC engine unless coordinated. + +## Worktree naming convention + +```text +worktrees/ + geointel-backend-foundation + geointel-frontend-shell + geointel-qaqc-engine + geointel-docs-control +``` + +## Merge order + +1. governance/docs updates; +2. database/domain foundation; +3. backend APIs; +4. frontend API client; +5. UI pages; +6. tests/fixtures; +7. polish. + +## Parallel agent completion report + +Every agent must report: + +- branch/worktree name; +- files changed; +- contracts touched; +- tests run; +- merge risks; +- required follow-up from other tracks. diff --git a/docs/30-codex-optimization/PROMPT_DISCIPLINE.md b/docs/30-codex-optimization/PROMPT_DISCIPLINE.md new file mode 100644 index 00000000..53ba00ff --- /dev/null +++ b/docs/30-codex-optimization/PROMPT_DISCIPLINE.md @@ -0,0 +1,94 @@ +# Prompt Discipline for Codex Runs + +Codex performs best when each run has one active objective, a clear source of truth, explicit stop conditions, and a small set of expected outputs. + +## Required prompt shape + +Every implementation prompt should contain: + +1. Active milestone. +2. Active pass. +3. Required documents to read. +4. Forbidden documents to treat as historical only. +5. Expected files or directories to touch. +6. Expected tests/checks. +7. Definition of Done. +8. Reporting format. + +## Good prompt pattern + +```text +You are working on GeoIntel Kempen. +Active pass: PASS_02_DATABASE_DOMAIN. +Read first: CODEX_START.md, docs/30-codex-optimization/CODEX_RUN_CHECKLIST.md, skills/postgis-migration/SKILL.md, docs/DATABASE_IMPLEMENTATION_PLAN.md. +Do not modify frontend files in this pass. +Implement only the database/domain foundation described in the active pass. +Run make readiness and relevant backend checks. +End with changed files, commands run, tests, risks, next pass. +``` + +## Bad prompt pattern + +```text +Build the whole platform. Improve whatever you see. Make it production ready. +``` + +This is forbidden because it causes scope creep, undocumented architecture choices, and conflicting implementations. + +## Improvement boundary + +Codex may improve: + +- naming consistency; +- validation details; +- typing; +- docstrings; +- tests; +- small helper functions; +- error messages; +- UI empty/loading/error states; +- non-breaking internal structure. + +Codex may not independently change: + +- primary stack; +- database choice; +- job queue choice; +- CRS policy; +- API contract shape; +- V1 scope; +- security model; +- storage architecture; +- model governance rules. + +## Context loading rule + +Do not read the entire repository for every pass. Load context in this order: + +1. root start files; +2. governance docs; +3. active pass prompt; +4. relevant skill; +5. directly relevant module docs; +6. code files affected by the pass; +7. tests/fixtures for the affected area. + +## End-of-pass response format + +Codex must end each pass with: + +```md +## Completed + +## Changed files + +## Commands run + +## Test results + +## Known limitations + +## Deviations from docs + +## Next recommended pass +``` diff --git a/docs/30-codex-optimization/SECRETS_AND_ENV_POLICY.md b/docs/30-codex-optimization/SECRETS_AND_ENV_POLICY.md new file mode 100644 index 00000000..b2303a95 --- /dev/null +++ b/docs/30-codex-optimization/SECRETS_AND_ENV_POLICY.md @@ -0,0 +1,50 @@ +# Secrets and Environment Policy + +GeoIntel must be safe to publish as a portfolio repository. + +## Absolute rules + +- Never commit real API keys. +- Never commit credentials, tokens, cookies or private endpoints. +- Never place secrets in docs, fixtures, tests or screenshots. +- `.env.example` may contain placeholder values only. +- Runtime secrets are read from environment variables. +- If a real key is accidentally found, remove it and rotate it outside the repo. + +## Approved environment variables + +- `DATABASE_URL` +- `POSTGRES_HOST` +- `POSTGRES_PORT` +- `POSTGRES_DB` +- `POSTGRES_USER` +- `POSTGRES_PASSWORD` +- `REDIS_URL` +- `STORAGE_ROOT` +- `OPENAI_API_KEY` +- `COPERNICUS_CLIENT_ID` +- `COPERNICUS_CLIENT_SECRET` +- `GRB_WFS_BASE_URL` +- `OSM_OVERPASS_URL` + +## Codex behavior + +When credentials are missing, Codex must: + +1. implement a clear configuration error; +2. document the missing variable; +3. provide an example in `.env.example`; +4. avoid hardcoded fallback secrets; +5. keep external-service calls behind adapters. + +## Local development fallback + +For V1 foundation work, services should be able to run with: + +- local PostGIS; +- local Redis; +- fixture datasets; +- disabled external fetchers; +- deterministic demo outputs. + +This fallback is not fake production behavior. It is a development mode and must be labeled as such. diff --git a/docs/30-codex-optimization/TOKEN_BUDGET_POLICY.md b/docs/30-codex-optimization/TOKEN_BUDGET_POLICY.md new file mode 100644 index 00000000..374be567 --- /dev/null +++ b/docs/30-codex-optimization/TOKEN_BUDGET_POLICY.md @@ -0,0 +1,45 @@ +# Token and Context Budget Policy + +This repository is intentionally documentation-heavy. Codex must not load all documents for every task. + +## Context tiers + +### Tier 0 — Always read + +- `CODEX_START.md` +- `docs/00-start/START_HERE.md` +- `docs/30-codex-optimization/CODEX_RUN_CHECKLIST.md` +- active pass prompt + +### Tier 1 — Read when architecture-sensitive + +- `docs/governance/GEOINTEL_CONSTITUTION.md` +- `docs/governance/ARCHITECTURE_INVARIANTS.md` +- `docs/governance/FORBIDDEN_DECISIONS.md` +- relevant ADRs + +### Tier 2 — Read when module-specific + +- relevant module spec +- relevant skill +- relevant API/database contract +- relevant tests/fixtures + +### Tier 3 — Historical reference only + +- old milestone handoff summaries +- previous pass prompts not active for the current run +- release notes from previous preparation milestones + +## Budget rules + +- Prefer reading indexes before detailed specs. +- Prefer targeted `grep/find` over opening large unrelated docs. +- Do not re-summarize old milestones unless needed. +- Work in small coherent diffs. +- If a task requires touching more than three major subsystems, split it into passes. +- If the active prompt conflicts with governance, stop and report. + +## Large file policy + +Large generated files, model artifacts, tiles, rasters and exports must not be created during documentation-preparation passes unless they are tiny fixtures. Real heavy assets belong outside git or in storage paths documented by `docs/STORAGE_ARCHITECTURE.md`. diff --git a/docs/40-build-launch/BACKLOG_PRIORITIES_MOSCOW.md b/docs/40-build-launch/BACKLOG_PRIORITIES_MOSCOW.md new file mode 100644 index 00000000..b366e27f --- /dev/null +++ b/docs/40-build-launch/BACKLOG_PRIORITIES_MOSCOW.md @@ -0,0 +1,41 @@ +# BACKLOG PRIORITIES — MoSCoW + +## Must have for Sprint 1 + +- Backend app foundation. +- Database/PostGIS foundation. +- Project Manager. +- Area Manager. +- Dataset Manager for GeoJSON. +- Metadata extraction. +- Frontend shell. +- MapLibre map. +- Demo fixture display. +- Readiness checks. + +## Should have for Sprint 1 if musts are complete + +- GeoJSON export baseline. +- Area import from fixture. +- Dataset detail metadata panel. +- Simple layer tree. +- Basic smoke tests for frontend build. + +## Could have later in same early phase + +- Reference/prediction fixture import. +- QA/QC schema validation. +- Simple feature statistics. +- Layer opacity controls. + +## Won't have in Sprint 1 + +- Live YOLO inference. +- Live SAM segmentation. +- GRB live WFS. +- Sentinel automation. +- LiDAR processing. +- Training Studio. +- AI Copilot. +- Full PDF report generator. +- Multi-user auth. diff --git a/docs/40-build-launch/BUILD_ORDER_GRAPH.md b/docs/40-build-launch/BUILD_ORDER_GRAPH.md new file mode 100644 index 00000000..d0ffbd0f --- /dev/null +++ b/docs/40-build-launch/BUILD_ORDER_GRAPH.md @@ -0,0 +1,78 @@ +# BUILD ORDER GRAPH — M14 Launch + +Codex must follow this dependency graph for implementation. + +## Graph + +```text +Repository readiness + -> Environment/config + -> Backend app foundation + -> Database/PostGIS foundation + -> Domain models/schemas + -> Migrations + -> Project API + -> Area API + -> Dataset API + -> Storage service + -> Metadata extraction + -> Frontend shell + -> Frontend API client + -> Map workbench shell + -> Dataset UI + -> Fixture layer display + -> Export baseline + -> Tests and readiness +``` + +## Blocked until foundation is stable + +The following are blocked until the above graph is green: + +- Detection Lab. +- Segmentation Lab. +- QA/QC engine beyond fixture/schema validation. +- Remote Sensing Lab. +- GRB live adapter. +- OSM live adapter. +- LiDAR Workbench. +- Training Studio. + +## Rule + +If a later module requires missing foundation work, Codex must complete the foundation work first instead of building around it. + +## Preferred first implementation passes + +### Pass 1 — Backend and config + +- Create app structure. +- Add config/settings. +- Add health endpoint. +- Add response envelope and error handling. + +### Pass 2 — Database and domain + +- Add database connection. +- Add migrations. +- Add core models. +- Enable PostGIS. + +### Pass 3 — Project/Area/Dataset API + +- Implement CRUD. +- Validate geometry. +- Extract metadata. + +### Pass 4 — Frontend shell and map + +- React app. +- Layout. +- MapLibre. +- API client. + +### Pass 5 — Fixture-driven integration + +- Load demo area/reference layer. +- Display layers. +- Run smoke tests. diff --git a/docs/40-build-launch/BUILD_SUCCESS_DEFINITION.md b/docs/40-build-launch/BUILD_SUCCESS_DEFINITION.md new file mode 100644 index 00000000..da67dcd6 --- /dev/null +++ b/docs/40-build-launch/BUILD_SUCCESS_DEFINITION.md @@ -0,0 +1,106 @@ +# BUILD SUCCESS DEFINITION — Sprint 1 / First Codex Run + +This document defines the exact point at which Codex must stop expanding scope and consider the first implementation run successful. + +## Purpose + +The first build is not successful because many features exist. It is successful when the foundation is stable, testable and ready for the next module. + +## Sprint 1 success statement + +Sprint 1 is successful when GeoIntel can run locally with a backend, frontend, database and a minimal geospatial dataset workflow. + +The vertical slice is: + +```text +Project + -> Area + -> Dataset upload/registration + -> Metadata extraction + -> PostGIS persistence + -> Minimal map display + -> GeoJSON export +``` + +## Required backend success criteria + +- FastAPI application starts without import errors. +- `/health` returns a successful response. +- `/docs` or OpenAPI schema is available in development. +- Database configuration is loaded from environment variables. +- PostgreSQL connection is verified by a health or readiness check. +- PostGIS extension is created or verified by migration/bootstrap logic. +- Project CRUD works through API endpoints. +- Area CRUD works with valid GeoJSON polygon input. +- Dataset registration or upload works for at least GeoJSON. +- Dataset metadata is extracted and persisted. +- API responses use the canonical response envelope. +- Errors use the canonical error format. + +## Required database success criteria + +- Alembic or equivalent migration path exists. +- Core tables exist: + - `projects` + - `areas` + - `datasets` + - `dataset_versions` or documented equivalent + - `analysis_runs` placeholder/table if needed for future compatibility + - `exports` placeholder/table if needed for future compatibility +- Geometry columns use PostGIS types. +- CRS/SRID rules follow `docs/specs/GIS_STANDARDS.md`. +- Demo seed data can be loaded or fixtures can be used in tests. + +## Required frontend success criteria + +- React/Vite application starts. +- Main layout is visible. +- MapLibre map renders. +- Project list/detail state exists. +- Dataset upload/registration UI exists for the Sprint 1 dataset type. +- Area display or drawing/import flow exists in minimal form. +- API client uses configured backend base URL. +- Loading, empty and error states exist for the implemented pages. + +## Required storage success criteria + +- Uploaded/registered dataset files are stored under a controlled storage path. +- Storage paths are not hardcoded to a developer machine. +- Metadata in the database references stored files where applicable. +- Generated exports go to a controlled exports folder. + +## Required testing success criteria + +At minimum: + +- Backend import smoke test passes. +- Health endpoint test passes. +- Database connection/migration smoke test passes or has a documented fallback if no DB is available in CI. +- GeoJSON fixture validation passes. +- Frontend build or typecheck passes. +- `make readiness` passes. + +## Explicitly not required for Sprint 1 + +Do not block Sprint 1 on: + +- YOLO inference. +- SAM inference. +- Sentinel download automation. +- GRB live WFS integration. +- LiDAR processing. +- Training Studio. +- AI Copilot. +- Advanced reporting. +- Multi-user authentication. +- Production deployment. + +## Stop condition + +When all Sprint 1 success criteria are met, Codex must stop feature expansion and produce: + +- changelog entry; +- commands run; +- test results; +- known limitations; +- next pass recommendation. diff --git a/docs/40-build-launch/CODEX_STOP_RULES.md b/docs/40-build-launch/CODEX_STOP_RULES.md new file mode 100644 index 00000000..e9e9f919 --- /dev/null +++ b/docs/40-build-launch/CODEX_STOP_RULES.md @@ -0,0 +1,45 @@ +# CODEX STOP RULES + +Codex must stop or pause expansion when these conditions occur. + +## Hard stop conditions + +Stop implementation and report if: + +- backend cannot import; +- frontend cannot build due to own changes; +- migrations cannot be generated or applied due to unclear schema conflict; +- API contract conflict is found; +- architecture invariant would need to be broken; +- dependency choice conflicts with ADRs; +- secrets or API keys are accidentally introduced; +- external service is required for a Sprint 1 must-have. + +## Soft stop conditions + +Pause expansion and finish cleanup if: + +- tests fail after the intended module is implemented; +- a feature starts requiring a non-Sprint-1 module; +- implementation requires more than one new abstraction not already documented; +- generated code duplicates existing logic; +- TODO comments are being used to hide incomplete logic. + +## What Codex must do at stop + +Report: + +- exact blocker; +- files affected; +- commands run; +- failing output summary; +- recommended fix; +- whether rollback is needed. + +## What Codex must not do + +- Do not continue building unrelated modules while the build is broken. +- Do not silence errors by weakening tests. +- Do not replace real functionality with permanent mock logic. +- Do not change the architecture to make one test pass. +- Do not introduce a new dependency without ADR-compatible justification. diff --git a/docs/40-build-launch/DATA_ACQUISITION_PLAYBOOK.md b/docs/40-build-launch/DATA_ACQUISITION_PLAYBOOK.md new file mode 100644 index 00000000..345fdf91 --- /dev/null +++ b/docs/40-build-launch/DATA_ACQUISITION_PLAYBOOK.md @@ -0,0 +1,116 @@ +# DATA ACQUISITION PLAYBOOK + +This playbook tells Codex how to think about data acquisition without spending the first build researching or overengineering live integrations. + +## Principle + +Sprint 1 uses local demo fixtures first. Live external data integrations are later adapters. + +The data hierarchy is: + +1. Golden fixtures for tests and UI development. +2. Local user-uploaded files. +3. Cached reference extracts. +4. Live external data services. + +## Sprint 1 data sources + +### Golden GeoJSON fixtures + +Use existing files in `demo/geel/` as the first source of truth: + +- `area_geel_center.geojson` +- `reference_buildings.geojson` +- `demo_detections.geojson` +- `expected_qaqc_metrics.json` + +These are not meant to be geographically complete. They are contract fixtures. + +### User upload/register + +Support GeoJSON first. Validate: + +- file extension; +- JSON parse; +- FeatureCollection shape; +- geometry existence; +- CRS handling fallback; +- bounds calculation. + +## GRB strategy + +GRB is the professional reference layer for Vlaanderen. It is the preferred future reference for buildings and related vector features. + +### Sprint 1 + +- Do not implement live GRB WFS yet. +- Use GRB-like local fixture data. +- Design a `ReferenceDataAdapter` interface so GRB can be added without rewriting QA/QC. + +### Sprint 2+ + +- Add GRB adapter. +- Prefer bbox/area-scoped retrieval. +- Cache retrieved features in PostGIS. +- Track source, retrieval timestamp and layer name. + +### Fallback + +If GRB is unavailable: + +- use cached extract; +- show external-source-unavailable status; +- do not fake live data. + +## OSM strategy + +OSM can be useful as a broad fallback/reference but must not replace GRB for professional building QA where GRB is available. + +### Sprint 1 + +- No live OSM required. +- Keep adapter boundary ready. + +### Later + +- Use Overpass or local extracts for bounded areas. +- Cache in PostGIS. + +## Sentinel strategy + +Sentinel belongs to the Remote Sensing Lab, not Sprint 1. + +### Sprint 1 + +- No Sentinel automation. +- Do not add Copernicus dependencies. + +### Later + +- Prefer STAC-based lookup where possible. +- Support NDVI/NDWI/NDBI through raster pipeline. +- Cache downloaded scenes/derived rasters. + +## DHMV / height data strategy + +DHMV/DEM/DSM belongs after raster/vector foundations are stable. + +### Sprint 1 + +- No height integration. + +### Later + +- Add DEM/DSM products as raster datasets. +- Reuse raster metadata, clipping and tiling pipelines. + +## Data acquisition acceptance criteria + +A new data source is accepted only when it has: + +- adapter boundary; +- source metadata; +- cache strategy; +- error handling; +- tests or fixture equivalent; +- documentation update. diff --git a/docs/40-build-launch/FOLDER_OWNERSHIP.md b/docs/40-build-launch/FOLDER_OWNERSHIP.md new file mode 100644 index 00000000..79c159eb --- /dev/null +++ b/docs/40-build-launch/FOLDER_OWNERSHIP.md @@ -0,0 +1,75 @@ +# FOLDER OWNERSHIP AND RESPONSIBILITIES + +Codex must keep responsibilities separated. + +## `backend/app/api/` + +HTTP routes only. No heavy business logic. + +## `backend/app/schemas/` + +Pydantic request/response models and API DTOs. + +## `backend/app/models/` + +Database ORM models. + +## `backend/app/services/` + +Business logic and orchestration. + +## `backend/app/repositories/` + +Database access patterns if repository layer is used. + +## `backend/app/workers/` + +RQ/Celery/background job entry points. + +## `backend/app/geo/` + +GIS-specific helpers: CRS, geometry validation, bounds, area calculations. + +## `backend/app/storage/` + +File storage logic. + +## `backend/tests/` + +Backend tests. Tests should not live beside production modules unless project conventions are changed deliberately. + +## `frontend/src/pages/` + +Route-level pages. + +## `frontend/src/components/` + +Reusable UI components. + +## `frontend/src/features/` + +Feature-oriented frontend modules, e.g. projects, datasets, map, exports. + +## `frontend/src/services/` + +API client/services. + +## `frontend/src/stores/` + +Client state only if needed. + +## `docs/` + +Architecture, specs and governance. Update docs when contracts or build rules change. + +## `demo/` + +Small golden fixtures and demo manifests. Not production storage. + +## `datasets/` + +Local development data folders. Do not commit large real datasets. + +## `storage/` + +Runtime storage. Keep `.gitkeep`, do not commit generated artifacts. diff --git a/docs/40-build-launch/GOLDEN_DATASET_PACKAGE.md b/docs/40-build-launch/GOLDEN_DATASET_PACKAGE.md new file mode 100644 index 00000000..180f9549 --- /dev/null +++ b/docs/40-build-launch/GOLDEN_DATASET_PACKAGE.md @@ -0,0 +1,67 @@ +# GOLDEN DATASET PACKAGE + +The golden dataset package defines the stable demo/test inputs that Codex must protect. + +## Purpose + +Golden data allows Codex to build and test without depending on live external services. + +## Golden area + +`demo/geel/area_geel_center.geojson` + +Represents the canonical Sprint 1 area fixture. It must remain valid GeoJSON. + +## Golden reference layer + +`demo/geel/reference_buildings.geojson` + +Represents reference building polygons. In future this will map to GRB-like reference data. + +Required properties: + +- stable feature IDs where possible; +- polygon or multipolygon geometry; +- deterministic feature count; +- valid geometries. + +## Golden prediction layer + +`demo/geel/demo_detections.geojson` + +Represents predicted building detections or imported detection outputs. + +Required properties: + +- class label; +- confidence where available; +- polygon or bbox-derived polygon geometry; +- stable enough for QA/QC fixture checks. + +## Golden expected metrics + +`demo/geel/expected_qaqc_metrics.json` + +Represents expected QA/QC output for the demo fixture. Codex may update this only if: + +- the fixture geometry intentionally changes; +- the QA/QC algorithm version changes; +- changelog explains the reason. + +## Golden data rules + +- Do not delete golden fixtures. +- Do not replace golden fixtures with random generated data. +- Do not make tests depend on external services when golden fixtures are sufficient. +- Keep fixtures small enough for fast CI/smoke runs. +- If new fixture files are added, update `docs/DEMO_FIXTURE_MANIFEST.md`. + +## Sprint 1 usage + +Codex should use golden data for: + +- frontend map layer smoke display; +- backend fixture validation; +- dataset metadata extraction tests; +- future QA/QC regression tests; +- export contract tests. diff --git a/docs/40-build-launch/MODULE_ACCEPTANCE_CRITERIA.md b/docs/40-build-launch/MODULE_ACCEPTANCE_CRITERIA.md new file mode 100644 index 00000000..4a8cf371 --- /dev/null +++ b/docs/40-build-launch/MODULE_ACCEPTANCE_CRITERIA.md @@ -0,0 +1,83 @@ +# MODULE ACCEPTANCE CRITERIA + +This document defines when a module is considered done enough to move forward. + +## Backend foundation + +Done when: + +- app imports without error; +- health endpoint works; +- config is environment-driven; +- response envelope is used; +- error shape is canonical; +- backend smoke test exists. + +## Database foundation + +Done when: + +- database connection is configured; +- PostGIS is enabled or checked; +- migrations exist; +- core tables can be created; +- geometry storage works; +- tests or smoke checks exist. + +## Project Manager + +Done when: + +- project create/list/read works; +- invalid payloads return canonical errors; +- frontend can display projects; +- tests cover at least one create/list/read path. + +## Area Manager + +Done when: + +- valid GeoJSON polygon can be stored; +- invalid geometry is rejected; +- bounds or area metadata is returned where practical; +- geometry is stored in PostGIS; +- frontend can display at least one area on the map. + +## Dataset Manager + +Done when: + +- GeoJSON dataset can be registered/uploaded; +- metadata is extracted; +- metadata is persisted; +- original file or reference is stored; +- dataset state follows the state machine; +- frontend can list datasets. + +## Map Workbench Foundation + +Done when: + +- MapLibre renders; +- fixture layer can be displayed; +- layer loading state exists; +- error state exists; +- map controls do not break layout. + +## Export baseline + +Done when: + +- at least one valid GeoJSON export path exists; +- export file is stored or streamed consistently; +- export metadata is recorded or returned; +- invalid export requests fail gracefully. + +## QA/QC skeleton + +Done when: + +- schema for reference and prediction layers is documented/implemented; +- no live AI inference is required; +- future QA engine can plug into analysis run structure; +- fixture metrics can be validated if implemented. diff --git a/docs/40-build-launch/RELEASE_STRATEGY.md b/docs/40-build-launch/RELEASE_STRATEGY.md new file mode 100644 index 00000000..f6ae87e9 --- /dev/null +++ b/docs/40-build-launch/RELEASE_STRATEGY.md @@ -0,0 +1,68 @@ +# RELEASE STRATEGY + +GeoIntel releases must be small, testable and aligned with the build order graph. + +## Version targets + +### v0.1 — Foundation + +- Backend starts. +- Frontend starts. +- Database/PostGIS works. +- Health/readiness works. + +### v0.2 — Project and Area Manager + +- Project CRUD. +- Area CRUD. +- Geometry persistence. +- Map displays area fixtures. + +### v0.3 — Dataset Manager + +- GeoJSON upload/register. +- Metadata extraction. +- Dataset list/detail UI. + +### v0.4 — Raster/Vector Foundation + +- Raster metadata skeleton if dependencies are available. +- Vector operations baseline. +- Layer display improvements. + +### v0.5 — Detection Import / Detection Lab Skeleton + +- Detection result import. +- Detection layer display. +- Model-adapter boundary, no heavy inference required yet. + +### v0.6 — QA/QC Foundation + +- Reference vs prediction matching. +- Metrics. +- False positive/false negative outputs. + +### v0.7 — GeoJSON Export and Review + +- Stable export workflow. +- Export validation. +- Basic review UI. + +### v1.0 — GeoAI Workbench MVP + +- Stable dataset workflow. +- Raster/vector foundations. +- Detection/segmentation architecture. +- QA/QC workflow. +- Export workflow. +- Portfolio-ready demo. + +## Release rule + +A release cannot be cut if: + +- readiness checks fail; +- golden paths regress; +- docs are stale; +- API contracts drift without documentation; +- known limitations are hidden. diff --git a/docs/40-build-launch/RISK_REGISTER.md b/docs/40-build-launch/RISK_REGISTER.md new file mode 100644 index 00000000..e841fc97 --- /dev/null +++ b/docs/40-build-launch/RISK_REGISTER.md @@ -0,0 +1,82 @@ +# RISK REGISTER + +This register captures known build and product risks before the first implementation run. + +## R1 — Scope creep during first Codex run + +Impact: high. + +Mitigation: + +- Follow `SPRINT_1_SCOPE_FREEZE.md`. +- Use `CODEX_STOP_RULES.md`. +- Add out-of-scope items to backlog. + +## R2 — External data source instability + +Impact: medium/high. + +Mitigation: + +- Use golden fixtures in Sprint 1. +- Add live adapters later. +- Cache retrieved external data. + +## R3 — GIS CRS mistakes + +Impact: high. + +Mitigation: + +- Follow `docs/specs/GIS_STANDARDS.md`. +- Validate geometries. +- Store SRID explicitly. +- Test fixture bounds. + +## R4 — Raster files too large + +Impact: medium. + +Mitigation: + +- No heavy raster processing in Sprint 1. +- Use tiling strategy later. +- Enforce file size limits when upload is implemented. + +## R5 — AI modules introduced too early + +Impact: high. + +Mitigation: + +- Keep live inference out of Sprint 1. +- Implement adapter boundaries only when needed. +- Use imported demo detections before live models. + +## R6 — Frontend becomes dashboard-first + +Impact: medium. + +Mitigation: + +- Follow Constitution: data -> processing -> QA/QC -> export. +- Map supports analysis; it is not the product by itself. + +## R7 — Mock data becomes permanent + +Impact: high. + +Mitigation: + +- Golden fixtures are contract data, not fake product behavior. +- Mark demo/fixture paths clearly. +- Production endpoints must use persisted data. + +## R8 — Too many competing docs + +Impact: medium. + +Mitigation: + +- Follow `CODEX_START.md` and `docs/00-start/START_HERE.md`. +- M14 launch docs supersede older first-run plans where conflicts exist. diff --git a/docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md b/docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md new file mode 100644 index 00000000..c1fc2322 --- /dev/null +++ b/docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md @@ -0,0 +1,97 @@ +# SPRINT 1 SCOPE FREEZE + +This document freezes the first implementation scope. Codex must not expand Sprint 1 beyond this boundary. + +## Sprint 1 goal + +Build the GeoIntel foundation vertical slice: a locally runnable app that proves project, area, dataset, PostGIS and map foundations. + +## Must build + +### Backend foundation + +- FastAPI app skeleton. +- Settings/config module. +- Structured logging baseline. +- Health/readiness endpoint. +- API response envelope. +- Error handling middleware or equivalent. +- CORS configuration for local frontend. + +### Database foundation + +- PostgreSQL/PostGIS connection. +- Migration tooling. +- Core models/schemas for projects, areas and datasets. +- Geometry persistence. +- Minimal seed/fixture support. + +### Project and area workflow + +- Create/list/read project. +- Create/list/read area. +- Store area geometry. +- Validate geometry. +- Return area metadata such as area size where practical. + +### Dataset workflow + +- Register/upload GeoJSON dataset. +- Store original file or registered reference. +- Extract metadata: + - dataset type; + - feature count; + - geometry type; + - CRS if available; + - bounds. +- Persist metadata. + +### Frontend foundation + +- Vite/React/TypeScript app. +- Main shell/layout. +- MapLibre map. +- Project workspace page. +- Dataset manager page/panel. +- Minimal layer display using demo GeoJSON. +- API client. + +### Tooling + +- Docker Compose for DB and services. +- README quickstart updated if implementation changes commands. +- `make readiness` remains green. +- Basic tests/smoke scripts. + +## Should build if Sprint 1 musts are complete + +- Area import from fixture. +- GeoJSON export endpoint for area or dataset features. +- Minimal QA placeholder that validates reference vs prediction fixture schema, but does not run full QA engine yet. +- Frontend layer opacity toggle. + +## May build only if zero risk + +- Demo fixture loader. +- Simple statistics card for feature count and area. +- Simple project status panel. + +## Must not build in Sprint 1 + +- YOLO live inference. +- SAM live segmentation. +- Sentinel/STAC integration. +- GRB live WFS client. +- DHMV integration. +- LiDAR LAS/LAZ processing. +- Training Studio. +- Model registry UI. +- AI Copilot. +- Advanced PDF reports. +- Multi-user authentication. +- Role-based permissions. +- Production Kubernetes/deployment stack. + +## Scope conflict rule + +If a task seems useful but is not listed under `Must`, `Should` or `May`, Codex must not implement it in Sprint 1. Add it to backlog/open issues instead. diff --git a/docs/ACCEPTANCE_CRITERIA.md b/docs/ACCEPTANCE_CRITERIA.md new file mode 100644 index 00000000..8e93fa4e --- /dev/null +++ b/docs/ACCEPTANCE_CRITERIA.md @@ -0,0 +1,66 @@ +# Acceptance Criteria + +## Milestone M1 — Working foundation +M1 is accepted when: +- backend starts successfully; +- frontend starts successfully; +- project can be created; +- area can be added as polygon; +- dataset can be uploaded or fixture-imported; +- metadata is persisted; +- basic map/workbench UI exists; +- tests cover project/area basics. + +## Milestone M2 — Raster/vector foundation +M2 is accepted when: +- vector data can be imported; +- raster metadata can be read; +- CRS/bounds/resolution are visible; +- area/length calculations use metric CRS; +- vector clipping works; +- raster clipping either works or fails with a clear limitation message; +- outputs are persisted as artifacts. + +## Milestone M3 — Detection foundation +M3 is accepted when: +- a detection run can be started; +- detections are persisted with class, confidence and geometry; +- detections can be visualized/listed; +- detections can be exported as GeoJSON; +- provider interface supports future YOLO implementation. + +## Milestone M4 — QA/QC foundation +M4 is accepted when: +- predictions can be compared with a reference layer; +- IoU matching works; +- precision, recall and F1 are calculated; +- false positives and false negatives are available as separate findings/layers; +- unit tests cover edge cases. + +## Milestone M5 — End-to-end demo +M5 is accepted when Demo 1 runs from UI: +1. Open project. +2. Select area. +3. Load raster/reference fixtures. +4. Run detection. +5. Run QA/QC against reference. +6. View map/results. +7. Export GeoJSON and summary. + +## Quality gates +Before any handoff: +- no syntax errors; +- no obvious broken imports; +- backend test suite run; +- frontend build/typecheck run if available; +- docs updated; +- limitations documented. + +## Rejection criteria +Reject a build if: +- core pages are placeholder-only; +- endpoints return hardcoded success without persisted data; +- geospatial outputs lack CRS metadata; +- code ignores service boundaries; +- major errors are hidden from UI; +- a feature claims GRB/YOLO/SAM support without provider separation or explicit development-provider labeling. diff --git a/docs/ACCEPTANCE_MATRIX.md b/docs/ACCEPTANCE_MATRIX.md new file mode 100644 index 00000000..afff79bd --- /dev/null +++ b/docs/ACCEPTANCE_MATRIX.md @@ -0,0 +1,62 @@ +# Acceptance Matrix + +## Foundation + +Accepted when: + +- Backend starts. +- Frontend starts. +- Database connects. +- Health endpoint works. +- Environment variables are documented. + +## Dataset Manager + +Accepted when: + +- Datasets can be registered and listed. +- Upload status is persisted. +- Metadata extraction is visible. +- Unsupported files are rejected cleanly. + +## Map Workbench + +Accepted when: + +- Areas can be displayed as GeoJSON. +- Layers can be toggled. +- Selected features show metadata. + +## Raster Lab + +Accepted when: + +- Raster metadata is extracted. +- Raster bounds are shown. +- Clip job can be queued. +- Raster artifacts are stored predictably. + +## Vector Lab + +Accepted when: + +- GeoJSON fixture imports. +- Clip/intersection operation works. +- Area/length metrics are correct. + +## Detection Lab + +Accepted when: + +- Detection run can be created. +- Results are stored as geospatial features. +- Confidence threshold is respected. +- GeoJSON export works. + +## QA/QC Lab + +Accepted when: + +- Prediction/reference matching works. +- Precision/recall/F1/IoU are calculated. +- False positives and false negatives are visible. diff --git a/docs/ACCEPTANCE_TEST_CATALOG.md b/docs/ACCEPTANCE_TEST_CATALOG.md new file mode 100644 index 00000000..cdaf3b36 --- /dev/null +++ b/docs/ACCEPTANCE_TEST_CATALOG.md @@ -0,0 +1,66 @@ +# Acceptance Test Catalog + +## Foundation +- Health endpoint returns application status. +- API error responses follow the standard envelope. +- CORS is configured for frontend local development. +- Database connection failure is reported but does not crash import-time tests. + +## Projects +- Create project with valid payload. +- Reject empty name. +- List projects sorted by newest first. +- Get missing project returns 404 envelope. + +## Areas +- Create area from valid Polygon GeoJSON. +- Reject invalid geometry. +- Store geometry with expected SRID. +- Return FeatureCollection for project areas. + +## Datasets +- Upload supported vector file. +- Extract geometry type, feature count, bounds and CRS when available. +- Reject unsupported extension with explicit error code. +- Dataset status transitions: uploaded -> metadata_extracted -> ready or failed. + +## Raster +- Read raster metadata when valid raster available. +- Return band count, bounds, CRS, width, height and resolution. +- Clip request creates a processing job. +- Tile request creates deterministic tile manifest. + +## Vector +- Read vector metadata from GeoJSON fixture. +- Clip vector features to area. +- Buffer vector features by distance in meters. +- Intersect two vector layers. + +## Detection +- Create detection analysis run. +- Demo-mode detector creates stable predictions. +- Detections include class, confidence, geometry and source tile. +- Detection outputs are exportable as GeoJSON. + +## Segmentation +- Create segmentation analysis run. +- Demo-mode segmenter creates stable polygons. +- Segmentations include class, area_m2, confidence and optional mask path. + +## QA/QC +- Match detection to reference by IoU threshold. +- Calculate TP, FP, FN. +- Calculate precision, recall and F1. +- Generate false positive and false negative layers. + +## Frontend +- Every route has loading, empty, error and success states. +- Project creation navigates to workspace. +- Map page can render GeoJSON area fixture. +- Dataset detail shows extracted metadata. +- Analysis status updates are visible. + +## Export +- GeoJSON export returns downloadable file. +- CSV metrics export includes metric key, value and unit. +- Export records are listed in project exports. diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md new file mode 100644 index 00000000..8b23be91 --- /dev/null +++ b/docs/AI_PIPELINES.md @@ -0,0 +1,232 @@ +# AI Pipelines + +## 1. Object Detection Pipeline + +```text +Raster dataset +↓ +Clip to analysis area +↓ +Tile raster +↓ +Normalize tiles +↓ +Run YOLO/PyTorch inference +↓ +Filter by confidence +↓ +Convert pixel boxes to geospatial polygons +↓ +Merge overlapping detections +↓ +Store in PostGIS +↓ +Expose as GeoJSON layer +↓ +Run QA/QC if reference data exists +``` + +### Sprint 8 foundation status + +Sprint 8 implements the detection persistence and execution boundary only: + +- `detections` are first-class PostGIS records linked to project, dataset, job and analysis run. +- `analysis_runs` remain separate from jobs and store model metadata, parameters, result summaries and lifecycle status. +- `yolo-placeholder` reports `not_configured`; no YOLO/PyTorch model is downloaded or executed. +- `manual-fixture-detector` is test/demo-only and persists detections only when `fixture_mode=true` and fixture detections are explicitly supplied. +- Normal application behavior must not create fake detections. + +### Sprint 8B configured YOLO status + +Sprint 8B adds an import-safe real YOLO adapter path: + +- `ultralytics` and `torch` are optional backend extras, not default runtime dependencies. +- `yolo-configured` reports `not_configured` until `YOLO_ENABLED=true`, `YOLO_MODEL_PATH` points to an existing local model file and optional AI dependencies are installed. +- GeoIntel never downloads model weights automatically. +- Real YOLO inference uses an existing raster tile manifest generated by the raster tile operation. +- YOLO pixel boxes are converted to EPSG:4326 detection polygons from tile transform or tile bounds metadata. +- Detection runs remain synchronous behind the existing job and analysis-run persistence boundary for Sprint 8B. + +### Sprint 13 YOLO operational preflight + +Sprint 13 adds a local preflight command for configured YOLO operation: + +```bash +python scripts/yolo_preflight.py --model-path /absolute/path/to/model.pt --tile-manifest-path /absolute/path/to/manifest.json +``` + +For machines without optional AI dependencies, path and manifest checks can be exercised without pretending inference is available: + +```bash +python scripts/yolo_preflight.py --model-path /absolute/path/to/model.pt --tile-manifest-path /absolute/path/to/manifest.json --assume-dependencies --json +``` + +The preflight checks: + +- `YOLO_ENABLED` / explicit enabled state; +- optional dependency availability unless `--assume-dependencies` is used; +- local model file existence; +- tile manifest JSON validity; +- tile count against `YOLO_MAX_TILES`; +- referenced tile file existence. + +The preflight does not load the model, does not import Ultralytics unless dependency discovery requires package metadata, does not run inference and never downloads model weights. + +Environment variables: + +- `YOLO_ENABLED` +- `YOLO_MODEL_PATH` +- `YOLO_MODEL_ID` +- `YOLO_MODEL_DISPLAY_NAME` +- `YOLO_MODEL_VERSION` +- `YOLO_DEVICE` +- `YOLO_IMAGE_SIZE` +- `YOLO_MAX_TILES` +- `YOLO_BATCH_SIZE` + +### Sprint 8C detection visualization and QA status + +Sprint 8C makes persisted detections reviewable: + +- Detection runs can be listed and selected. +- Persisted detections can be listed and filtered by run, dataset, class and minimum confidence. +- Persisted detection geometries can be returned as GeoJSON FeatureCollections for MapLibre display. +- Detection QA compares candidate detection geometries against persisted reference `vector_features`. +- QA results reuse `quality_checks` and `metrics`; no parallel QA persistence system is introduced. +- Segmentation remains out of scope for Sprint 8C. + +## 2. Tile Metadata + +Elke tile moet opslaan: + +- tile path +- parent raster id +- pixel window +- geospatial bounds +- transform +- CRS +- tile size +- overlap + +Zonder tile metadata kunnen modeloutputs niet correct teruggeprojecteerd worden. + +## 3. Detection Output Contract + +Elke detectie bevat: + +- class_name +- confidence +- bbox pixel coords +- source tile +- geospatial polygon +- model id/version +- analysis run id + +## 4. Segmentation Pipeline + +```text +Raster dataset +↓ +Clip/tile +↓ +Run segmentation model +↓ +Generate mask +↓ +Georeference mask +↓ +Polygonize mask +↓ +Simplify/clean geometries +↓ +Store polygons + mask path +↓ +Expose as map layer +``` + +### Sprint 9 segmentation foundation status + +Sprint 9 implements the segmentation persistence and review boundary only: + +- `segmentations` are first-class PostGIS records linked to project, dataset, job and analysis run. +- PostGIS MultiPolygon geometry in EPSG:4326 is authoritative for map display, QA and GeoJSON output. +- Mask paths are persisted as artifact/provenance references, not authoritative feature state. +- `segmentation-placeholder`, `yolo-seg-configured` and `sam-configured` report `not_configured`. +- `fixture-segmenter` is test/demo-only and persists segmentations only when `fixture_mode=true` and fixture segmentations are explicitly supplied. +- Segmentation QA compares persisted segmentation geometries against persisted reference `vector_features`. +- QA results reuse `quality_checks` and `metrics`; no parallel QA system is introduced. +- GeoIntel does not install SAM, run YOLO-seg, download model weights or fake production segmentations in Sprint 9. + +## 5. Change Detection Pipeline + +Fase 1: vector/detection based. + +```text +Run A detections ++ +Run B detections +↓ +Spatial matching +↓ +added / removed / changed +↓ +Change polygons +↓ +Metrics +``` + +Fase 2: raster index based. + +```text +Raster A index ++ +Raster B index +↓ +Difference raster +↓ +Threshold +↓ +Polygonize changed zones +``` + +Fase 3: segmentation based. + +```text +Mask A ++ +Mask B +↓ +Class difference +↓ +Change polygons +``` + +## 6. Model Strategy + +V1: + +- gebruik een bestaande YOLO-integratie met configureerbaar modelpad +- demo-model mag lokaal worden geplaatst in `models/` +- code moet ook zonder model kunnen starten, maar detection job moet dan duidelijke fout geven + +V2: + +- SAM/YOLO segmentation + +V3: + +- annotation export +- finetuning + +## 7. Reproduceerbaarheid + +Elke analysis run moet bewaren: + +- model id +- model version +- parameters +- confidence threshold +- tile size +- overlap +- input dataset id +- code path/version indien mogelijk diff --git a/docs/ANALYSIS_ENGINE.md b/docs/ANALYSIS_ENGINE.md new file mode 100644 index 00000000..3d613a5a --- /dev/null +++ b/docs/ANALYSIS_ENGINE.md @@ -0,0 +1,160 @@ +# Analysis Engine + +De Analysis Engine bevat alle reproduceerbare berekeningen. AI mag interpretaties schrijven, maar de cijfers komen uit deze engine. + +## 1. BuildingAnalyzer + +### Input + +- gebouwpolygonen uit GRB, OSM of detecties +- analysegebied + +### Output metrics + +```json +{ + "building_count": 123, + "building_area_total_m2": 45678.9, + "building_density_per_km2": 87.2, + "average_building_area_m2": 371.4, + "built_ratio": 0.23 +} +``` + +## 2. RoadAnalyzer + +### Input + +- wegvectoren +- analysegebied + +### Output + +- totale weglengte +- wegendichtheid +- verdeling per wegtype +- nabijheid tot hoofdwegen + +## 3. RasterAnalyzer + +### Input + +- rasterdataset +- analysegebied + +### Output + +- metadata +- bounds +- resolutie +- bandstatistieken +- histogram +- nodata-percentage + +## 4. VegetationAnalyzer + +### Input + +- NDVI-raster of vegetatiesegmentatie + +### Output + +- vegetatieoppervlakte +- vegetatiepercentage +- gemiddelde NDVI +- lage vegetatiegebieden +- hoge vegetatiegebieden + +## 5. WaterAnalyzer + +### Input + +- waterpolygonen of NDWI-raster + +### Output + +- wateroppervlakte +- waterpercentage +- afstand tot water +- waterverandering later + +## 6. DetectionAnalyzer + +### Input + +- detecties +- analysegebied + +### Output + +- aantal per klasse +- gemiddelde confidence +- confidence distributie +- oppervlakte per klasse indien polygonen beschikbaar +- detectiedichtheid + +## 7. SegmentationAnalyzer + +### Input + +- mask/polygon segmentaties + +### Output + +- oppervlakte per klasse +- segment count +- gemiddelde confidence +- dekking binnen analysegebied + +## 8. QAAnalyzer + +### Input + +- AI-output +- referentielaag +- IoU threshold + +### Output + +```json +{ + "true_positives": 100, + "false_positives": 8, + "false_negatives": 12, + "precision": 0.9259, + "recall": 0.8928, + "f1": 0.9090, + "mean_iou": 0.71 +} +``` + +### Regels + +- Een detectie matcht met een referentieobject als IoU >= threshold. +- Meerdere detecties op één referentieobject moeten gededupliceerd worden. +- Niet-gematchte detecties zijn false positives. +- Niet-gematchte referentieobjecten zijn false negatives. + +## 9. ChangeAnalyzer + +### Input + +- analysis run A +- analysis run B + +### Output + +- added objects +- removed objects +- changed objects +- changed area +- change density + +## 10. ScoreEngine later + +Scores zijn niet de primaire focus voor de vacaturegerichte versie, maar kunnen later worden toegevoegd: + +- Open Space Pressure Score +- Urban Expansion Score +- Nature Connectivity Score +- Water Resilience Score diff --git a/docs/ANALYSIS_SPECIFICATIONS.md b/docs/ANALYSIS_SPECIFICATIONS.md new file mode 100644 index 00000000..0eff5d7d --- /dev/null +++ b/docs/ANALYSIS_SPECIFICATIONS.md @@ -0,0 +1,250 @@ +# GeoIntel Kempen — Analysis Specifications v1.0 + +This file defines exact inputs, outputs and formulas for the first implementation of the analysis engine. + +## Global rules + +- All area-based metrics must be calculated in a projected CRS suitable for Belgium/Flanders, preferably EPSG:31370 internally for metric calculations. +- Store geometries consistently and transform only at API/render boundaries when needed. +- All metrics must include unit, input dataset ids, analysis run id and calculation parameters. +- Never let the AI copilot invent metrics. Metrics must come from the analysis engine. + +## AreaAnalyzer + +### Input + +- Area polygon. + +### Output metrics + +| Key | Unit | Formula | +|---|---|---| +| `area_m2` | m² | `ST_Area(area.geometry)` | +| `area_km2` | km² | `area_m2 / 1_000_000` | +| `perimeter_m` | m | `ST_Perimeter(area.geometry)` | +| `bbox` | geometry/json | calculated bounds | + +## BuildingAnalyzer + +### Input + +- Area polygon. +- Building polygons from GRB, OSM, user vector layer, or AI segmentation/detection polygons. + +### Processing + +1. Clip building geometries to area. +2. Remove invalid geometries or repair with `make_valid`. +3. Calculate per-building clipped area. +4. Aggregate. + +### Output metrics + +| Key | Unit | Formula | +|---|---|---| +| `building_count` | count | number of building features intersecting area | +| `building_area_total_m2` | m² | sum clipped building area | +| `building_area_total_ha` | ha | `building_area_total_m2 / 10000` | +| `building_coverage_ratio` | ratio | `building_area_total_m2 / area_m2` | +| `building_density_per_km2` | count/km² | `building_count / area_km2` | +| `mean_building_area_m2` | m² | `building_area_total_m2 / building_count` | +| `largest_building_area_m2` | m² | max building area | + +### Output layers + +- `buildings_clipped` +- `building_centroids` +- `large_buildings_top_20` + +## RoadAnalyzer + +### Input + +- Area polygon. +- Road line or polygon features. + +### Output metrics + +| Key | Unit | Formula | +|---|---|---| +| `road_length_total_m` | m | sum clipped road lengths | +| `road_length_total_km` | km | `/1000` | +| `road_density_km_per_km2` | km/km² | `road_length_total_km / area_km2` | +| `major_road_length_km` | km | filtered by road class if available | + +### Output layers + +- `roads_clipped` +- `major_roads_clipped` + +## GreenAnalyzer + +### Input options + +- Green polygons from OSM/GRB/landuse. +- NDVI raster threshold result. +- Segmentation polygons classified as vegetation. + +### Output metrics + +| Key | Unit | Formula | +|---|---|---| +| `green_area_total_m2` | m² | sum green polygons clipped to area | +| `green_ratio` | ratio | `green_area_total_m2 / area_m2` | +| `green_patch_count` | count | number of disjoint green patches | +| `largest_green_patch_m2` | m² | max patch area | +| `green_fragmentation_index` | index | `green_patch_count / max(green_area_total_ha, 0.01)` | + +### Interpretation +High fragmentation means green is split into many smaller patches. + +## WaterAnalyzer + +### Input + +- Water polygons/lines from GRB/OSM. +- NDWI threshold polygons later. + +### Output metrics + +| Key | Unit | Formula | +|---|---|---| +| `water_area_total_m2` | m² | sum clipped water polygon area | +| `water_ratio` | ratio | `water_area_total_m2 / area_m2` | +| `watercourse_length_m` | m | sum water line length | +| `distance_to_nearest_water_m` | m | minimum distance from area centroid to water geometry | + +## RasterAnalyzer + +### Input + +- Raster dataset. +- Optional area polygon. + +### Output metrics + +Per band: + +| Key | Unit | +|---|---| +| `band_min` | band unit | +| `band_max` | band unit | +| `band_mean` | band unit | +| `band_std` | band unit | +| `nodata_ratio` | ratio | + +### Required operations + +- Read metadata. +- Clip by area. +- Compute statistics. +- Generate preview tile or PNG. + +## RemoteSensingIndexAnalyzer + +### NDVI + +Formula: + +```text +NDVI = (NIR - Red) / (NIR + Red) +``` + +Output: + +- `ndvi_mean` +- `ndvi_median` +- `ndvi_low_ratio` using threshold configurable, default `< 0.2` +- `ndvi_high_ratio` using threshold configurable, default `> 0.5` +- vectorized high/low vegetation zones later + +### NDWI + +```text +NDWI = (Green - NIR) / (Green + NIR) +``` + +### NDBI + +```text +NDBI = (SWIR - NIR) / (SWIR + NIR) +``` + +## DetectionAnalyzer + +### Input + +- Detection records with class, confidence and geometry. +- Area polygon. + +### Output metrics + +| Key | Unit | Formula | +|---|---|---| +| `detection_count` | count | detections within area | +| `detection_count_by_class` | json | group by class | +| `mean_confidence` | ratio | average confidence | +| `low_confidence_count` | count | confidence below threshold | +| `detected_area_m2_by_class` | json | sum polygon area where available | + +## ScoreEngine v1 + +The score engine must be transparent. Every score returns value, inputs, weights and explanation. + +### Open Space Pressure Score + +Default weights: + +```yaml +building_coverage_ratio: 0.35 +road_density_normalized: 0.25 +green_ratio_inverse: 0.25 +urban_growth_normalized: 0.15 +``` + +Score: + +```text +100 * weighted_sum(normalized_factors) +``` + +### Nature Connectivity Score + +Default weights: + +```yaml +green_ratio: 0.35 +largest_green_patch_ratio: 0.25 +fragmentation_inverse: 0.25 +major_road_barrier_inverse: 0.15 +``` + +### Water Resilience Score + +Default weights: + +```yaml +green_ratio: 0.30 +water_buffer_presence: 0.20 +impervious_inverse: 0.30 +low_point_risk_inverse: 0.20 +``` + +V1 may calculate a simplified score without height data by marking height-dependent factors as unavailable. + +## Metric storage contract + +Each metric row must include: + +```json +{ + "analysis_run_id": "uuid", + "key": "building_density_per_km2", + "value": 123.4, + "unit": "count/km2", + "method": "BuildingAnalyzer.v1", + "inputs": ["dataset_uuid"], + "parameters": {}, + "quality_flags": [] +} +``` diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md new file mode 100644 index 00000000..3cca46d9 --- /dev/null +++ b/docs/API_CONTRACTS.md @@ -0,0 +1,972 @@ +# API Contracts v1 + +This document freezes the first API shape. Codex may add implementation details but must not rename these routes without updating this file and the frontend API client. + +## API principles + +- Base path: `/api/v1`. +- JSON by default. +- GeoJSON accepted for geometries where possible. +- Long processing tasks return a job or analysis run record instead of blocking. +- Error responses use the shared `ApiError` schema. + +## Shared schemas + +### ApiError + +```json +{ + "error": "string", + "message": "human readable message", + "details": {}, + "request_id": "optional string" +} +``` + +### GeoJsonGeometry + +Any valid GeoJSON geometry object. V1 primarily expects `Polygon` and `MultiPolygon` for areas. + +### BoundingBox + +```json +{ + "min_x": 0.0, + "min_y": 0.0, + "max_x": 0.0, + "max_y": 0.0, + "crs": "EPSG:4326" +} +``` + +## Health + +### GET `/health` + +Returns service status. + +```json +{ + "status": "ok", + "service": "geointel-backend", + "version": "0.1.0" +} +``` + +### GET `/api/v1/system/capabilities` + +Returns enabled feature flags and tool availability. + +```json +{ + "postgis": true, + "rasterio": true, + "geopandas": true, + "yolo": false, + "sam": false, + "grb": "planned", + "sentinel": "planned", + "providers": [ + { + "provider_name": "grb", + "display_name": "GRB", + "authority_level": "authoritative", + "supported_layers": ["buildings", "roads", "parcels"], + "supported_geometry_types": ["Polygon", "MultiPolygon", "LineString", "MultiLineString"], + "supported_query_modes": ["area"], + "fetch_signature": "POST /api/v1/external/grb/fetch", + "configured": false, + "status": "not_configured", + "limitation_message": "GRB live WFS/download integration is not configured in Sprint 7B.", + "attribution": "Digitaal Vlaanderen - Basiskaart Vlaanderen (GRB)", + "license_note": "Use must follow Digitaal Vlaanderen open data and attribution terms.", + "not_configured_reason": "Provider integration is not configured yet" + } + ] +} +``` + +## Projects + +### GET `/api/v1/projects` + +Returns all projects. + +### POST `/api/v1/projects` + +Request: + +```json +{ + "name": "Geel building detection demo", + "description": "Detect buildings and validate against GRB", + "region": "Kempen" +} +``` + +Response: `ProjectRead`. + +### GET `/api/v1/projects/{project_id}` + +Returns one project with summary counts. + +### PATCH `/api/v1/projects/{project_id}` + +Updates name/description/region. + +### DELETE `/api/v1/projects/{project_id}` + +Soft-delete in V1 preferred. Hard-delete only if storage cleanup is also implemented. + +## Areas + +### GET `/api/v1/projects/{project_id}/areas` + +Returns areas for a project. + +### POST `/api/v1/projects/{project_id}/areas` + +Request: + +```json +{ + "name": "Geel Centrum AOI", + "geometry": {"type": "Polygon", "coordinates": []}, + "crs": "EPSG:4326" +} +``` + +Backend responsibilities: + +- Validate geometry. +- Repair trivial polygon issues if safe. +- Store geometry in PostGIS. +- Calculate area in square meters using projected CRS. +- Store bbox. + +## Datasets + +### POST `/api/v1/projects/{project_id}/datasets/upload` + +Multipart upload. + +Fields: + +- `file`: dataset file. +- `dataset_type`: `vector`, `geojson` (legacy), `raster`. +- `source`: free text, e.g. `user_upload`, `grb`, `osm`. +- `dataset_role`: `source`, `derived`, or `reference` (default `source`). +- `source_name`: optional source identity, e.g. `manual`, `grb`, `osm`; reference uploads default to `manual` when omitted. +- `reference_layer_name`: optional reference layer label, e.g. `buildings`; only retained for reference datasets. +- `area_id`: optional. + +Response: `DatasetRead` with extracted metadata if supported. + +Vector uploads remain stored as original files and are also persisted into `vector_features` as queryable PostGIS state. + +### GET `/api/v1/projects/{project_id}/datasets` + +List datasets. + +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}` + +Return metadata. + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/metadata/refresh` + +Re-extract metadata. + +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/vector/inspect` + +Return a wrapped vector inspection payload with metadata, storage summary and feature summary. + +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/vector/summary` + +Return vector summary data only. + +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/metadata` + +Return raster metadata profile for supported raster uploads. + +If raster processing is unavailable: + +```text +code: RASTER_PROCESSING_UNAVAILABLE +message: Raster processing unavailable. Install rasterio and GDAL-compatible drivers to enable raster metadata extraction. +``` + +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/inspect` + +Return raster inspect wrapper payload. + +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/stats` + +Return raster band statistics payload. + +If raster processing dependencies are unavailable: + +- code: `RASTER_PROCESSING_UNAVAILABLE` +- message: dependency-specific unavailable message. + +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/preview` + +Preview readiness for raster layers. + +If preview dependencies are unavailable: + +- code: `RASTER_PROCESSING_UNAVAILABLE` +- message: `Raster preview unavailable...` + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/clip` + +Clip raster by selected area. Returns a `202`-style accepted job payload through the job wrapper (`jobs` create/read flow). + +If raster processing dependencies are unavailable: + +- code: `RASTER_PROCESSING_UNAVAILABLE` +- message: `Raster processing unavailable. Install rasterio and GDAL-compatible drivers to enable raster processing operations.` + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/reproject` + +Reproject raster dataset to another CRS. + +Input: + +- `target_crs` (default: `EPSG:31370`) +- `resampling` (`nearest`, `bilinear`, `cubic`; default `nearest`) +- `output_name` + +Returns a job payload with derived dataset id in `result.output_dataset_id`. + +Failure modes: + +- code: `INVALID_PARAMETERS` for bad CRS or resampling +- code: `INVALID_DATASET_CRS` when source raster CRS is missing +- code: `RASTER_PROCESSING_UNAVAILABLE` when rasterio is unavailable + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/indices/ndvi` + +Compute NDVI from raster band pairs. + +Input: + +- `nir_band` (positive integer, 1-based) +- `red_band` (positive integer, 1-based) +- `output_name` (optional) + +Returns a job payload with derived dataset id in `result.output_dataset_id`. + +Failure modes: + +- code: `INVALID_PARAMETERS` for non-positive/non-integer band indices +- code: `INVALID_PARAMETERS` for band index outside source band count +- code: `INVALID_DATASET_TYPE` when source is not raster +- code: `RASTER_PROCESSING_UNAVAILABLE` when rasterio or numpy is unavailable + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/indices/ndwi` + +Compute NDWI from raster band pairs. + +Input: + +- `nir_band` (positive integer, 1-based) +- `green_band` (positive integer, 1-based) +- `output_name` (optional) + +Returns a job payload with derived dataset id in `result.output_dataset_id`. + +Failure modes: + +- code: `INVALID_PARAMETERS` for non-positive/non-integer band indices +- code: `INVALID_PARAMETERS` for band index outside source band count +- code: `INVALID_DATASET_TYPE` when source is not raster +- code: `RASTER_PROCESSING_UNAVAILABLE` when rasterio or numpy is unavailable + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/indices/ndbi` + +Compute NDBI from raster band pairs. + +Input: + +- `nir_band` (positive integer, 1-based) +- `swir_band` (positive integer, 1-based) +- `output_name` (optional) + +Returns a job payload with derived dataset id in `result.output_dataset_id`. + +Failure modes: + +- code: `INVALID_PARAMETERS` for non-positive/non-integer band indices +- code: `INVALID_PARAMETERS` for band index outside source band count +- code: `INVALID_DATASET_TYPE` when source is not raster +- code: `RASTER_PROCESSING_UNAVAILABLE` when rasterio or numpy is unavailable + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/tile` + +Generate raster tiles and a manifest for downstream processing. Returns a job payload with `tile_set_id` and manifest metadata. + +If raster processing dependencies are unavailable: + +- code: `RASTER_PROCESSING_UNAVAILABLE` +- message: `Raster processing unavailable. Install rasterio and GDAL-compatible drivers to enable raster processing operations.` + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/vector/clip` + +Clip vector dataset to selected area. + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/vector/buffer` + +Apply buffer distance to vector features. + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/vector/intersect` + +Intersect source vector dataset with another vector dataset. + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/vector/stats` + +Return vector stats (feature counts and geometry summary). + +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/vector/bbox` + +Return vector bounds and feature count. + +## Jobs + +### POST `/api/v1/projects/{project_id}/jobs` + +Create a job. + +### GET `/api/v1/projects/{project_id}/jobs` + +List jobs. + +### GET `/api/v1/projects/{project_id}/jobs/{job_id}` + +Read job detail. + +### GET `/api/v1/projects/{project_id}/jobs/{job_id}/status` + +Read simplified job status payload. + +## Provider registry + +### GET `/api/v1/external/providers` + +Returns all configured provider capability descriptors. + +### GET `/api/v1/external/providers/capabilities` + +Compatibility alias for listing provider capability descriptors. + +### GET `/api/v1/external/providers/{provider_name}` + +Returns one provider capability descriptor. + +### GET `/api/v1/external/providers/{provider_name}/layers` + +Returns the supported provider layers. + +### GET `/api/v1/external/providers/{provider_name}/status` + +Returns configured/status/limitation fields. + +### POST `/api/v1/external/providers/{provider_name}/import` + +Defines the future provider import contract. Sprint 7B does not perform live imports or write datasets. + +Request: + +```json +{ + "project_id": "uuid-or-local-id", + "area_id": "optional uuid-or-local-id", + "layers": ["buildings"], + "dataset_role": "optional source|reference" +} +``` + +GRB/OSM response: + +```json +{ + "provider_name": "grb", + "status": "not_configured", + "message": "No live GRB import is configured in Sprint 7B.", + "requested_layers": ["buildings"], + "dataset_id": null, + "dataset_role": "reference", + "source_name": "grb" +} +``` + +Manual and fixture providers point callers to existing upload/fixture flows. No provider writes directly to `vector_features`; all future provider output must flow through `DatasetService` and `VectorFeatureService`. + +## External data fetchers + +### POST `/api/v1/external/osm/fetch` + +Request: + +```json +{ + "project_id": "uuid", + "area_id": "uuid", + "layers": ["buildings", "roads", "water", "green"] +} +``` + +### POST `/api/v1/external/grb/fetch` + +Request: + +```json +{ + "project_id": "uuid", + "area_id": "uuid", + "layers": ["buildings"] +} +``` + +V1 may initially implement this as a service interface with a clear `not_configured` response until the exact WFS endpoint is wired. + +Sprint 7B provider contract responses expose capabilities only. Providers must report: + +```json +{ + "provider_name": "osm", + "display_name": "OpenStreetMap", + "authority_level": "contextual", + "supported_layers": ["buildings", "roads", "water", "landuse"], + "supported_geometry_types": ["Polygon", "MultiPolygon", "LineString", "MultiLineString"], + "supported_query_modes": ["area"], + "configured": false, + "status": "not_configured", + "limitation_message": "OSM live Overpass/download integration is not configured in Sprint 7B.", + "attribution": "OpenStreetMap contributors", + "license_note": "OpenStreetMap data is available under ODbL; attribution is required." +} +``` + +No GRB WFS, OSM Overpass or provider downloads are implemented in Sprint 7B. + +## Demo workflow + +### POST `/api/v1/demo/workflow` + +Seeds an explicit offline demo workflow from local fixture files. This endpoint +does not fetch live GRB/OSM data and does not run AI inference. It creates or +returns: + +- one demo project +- one demo AOI +- one fixture reference building dataset +- one fixture candidate/predicted building dataset +- one persisted QA/QC result with metric rows + +The endpoint is idempotent for the named demo project. + +Response: + +```json +{ + "project_id": "uuid", + "area_id": "uuid", + "reference_dataset_id": "uuid", + "candidate_dataset_id": "uuid", + "quality_check_id": "uuid", + "metric_count": 6, + "status": "ready", + "message": "Demo workflow seeded from explicit local fixtures.", + "created": true +} +``` + +## Analysis + +## Detection Lab + +Sprint 8 implements Detection Lab foundation only. YOLO/PyTorch real inference is not enabled, no model is downloaded, and fixture detections require explicit fixture mode. + +### GET `/api/v1/detection/models` + +Returns object-detection model capability descriptors. + +```json +{ + "models": [ + { + "model_id": "yolo-placeholder", + "display_name": "YOLO detector placeholder", + "framework": "ultralytics/pytorch", + "task_type": "object_detection", + "supported_classes": ["building", "road", "water", "landuse"], + "configured": false, + "status": "not_configured", + "limitation_message": "YOLO/PyTorch inference is not configured in Sprint 8; no model is downloaded or executed.", + "version": null + }, + { + "model_id": "yolo-configured", + "display_name": "Configured YOLO detector", + "framework": "ultralytics/pytorch", + "task_type": "object_detection", + "supported_classes": ["building", "road", "water", "landuse"], + "configured": false, + "status": "not_configured", + "limitation_message": "YOLO is disabled. Set YOLO_ENABLED=true and YOLO_MODEL_PATH to a local model file to enable inference.", + "version": null + } + ] +} +``` + +### POST `/api/v1/detection/run` + +Creates a detection job and detection analysis run. If the requested model is unavailable, the job and analysis run are marked `failed` with `DETECTION_MODEL_UNAVAILABLE` or `DETECTION_DEPENDENCY_UNAVAILABLE`. + +Request: + +```json +{ + "project_id": "uuid", + "dataset_id": "uuid", + "model_id": "yolo-placeholder", + "confidence_threshold": 0.5, + "class_filter": ["building"], + "tile_manifest_path": null, + "parameters_json": {} +} +``` + +Sprint 8B configured YOLO mode uses `model_id: "yolo-configured"`. It requires: + +- `YOLO_ENABLED=true` +- `YOLO_MODEL_PATH` pointing to an existing local model file +- backend optional AI dependencies installed with `geointel-backend[ai]` +- `tile_manifest_path` pointing to an existing raster tile manifest generated by the raster tile operation + +GeoIntel does not download model weights automatically. Configured YOLO runs read existing tile files from the manifest, convert YOLO pixel-space boxes to EPSG:4326 detection polygons and persist detections as first-class records. + +Unavailable model response: + +```json +{ + "analysis_run_id": "uuid", + "job_id": "uuid", + "project_id": "uuid", + "dataset_id": "uuid", + "model_id": "yolo-placeholder", + "status": "failed", + "detection_count": 0, + "error_code": "DETECTION_MODEL_UNAVAILABLE", + "message": "YOLO/PyTorch inference is not configured in Sprint 8; no model is downloaded or executed." +} +``` + +Validation errors: + +- `INVALID_DATASET_TYPE` when the dataset is not raster. +- `DETECTION_MODEL_NOT_FOUND` when the model id is unknown. +- `FIXTURE_MODE_REQUIRED` when `manual-fixture-detector` is requested without `parameters_json.fixture_mode=true`. +- `DETECTION_TILE_MANIFEST_REQUIRED` when `yolo-configured` is requested without `tile_manifest_path`. +- `DETECTION_TILE_MANIFEST_NOT_FOUND` when the provided manifest path does not exist. +- `DETECTION_TILE_MANIFEST_INVALID` when the manifest cannot be parsed or lacks tile metadata. +- `DETECTION_TILE_LIMIT_EXCEEDED` when the manifest exceeds `YOLO_MAX_TILES`. +- `DETECTION_DEPENDENCY_UNAVAILABLE` when YOLO dependencies are not installed. +- `DETECTION_MODEL_LOAD_FAILED` when the local model file exists but cannot be loaded. + +Fixture detector mode is test/demo-only. It persists only explicit `parameters_json.fixture_detections` entries and is never invoked automatically. + +### GET `/api/v1/detection/runs/{analysis_run_id}` + +Returns one detection analysis run. + +### GET `/api/v1/detection/runs` + +Returns detection analysis runs, optionally filtered by `project_id` and `dataset_id`. + +### GET `/api/v1/detection/runs/{analysis_run_id}/detections` + +Returns persisted detections for a detection analysis run. Optional filters: + +- `dataset_id` +- `class_name` +- `min_confidence` + +### GET `/api/v1/detection/datasets/{dataset_id}/detections` + +Returns persisted detections for a raster dataset. Optional filters: + +- `analysis_run_id` +- `class_name` +- `min_confidence` + +### GET `/api/v1/detection/detections/{detection_id}` + +Returns one persisted detection. + +### GET `/api/v1/detection/runs/{analysis_run_id}/geojson` + +Returns persisted detections for a run as a GeoJSON FeatureCollection. Geometry comes from persisted PostGIS detection geometry in EPSG:4326. + +Each feature includes: + +- `detection_id` +- `class_name` +- `confidence` +- `model_name` +- `model_version` +- `analysis_run_id` +- `dataset_id` +- `job_id` +- `source_tile_path` +- `bbox_json` + +### GET `/api/v1/detection/datasets/{dataset_id}/geojson` + +Returns persisted detections for a dataset as a GeoJSON FeatureCollection. Optional filters match the detection list endpoint. + +### POST `/api/v1/detection/runs/{analysis_run_id}/qa/reference` + +Compares persisted detection geometries from an analysis run against persisted `vector_features` from a reference vector dataset. + +Request: + +```json +{ + "reference_dataset_id": "uuid", + "iou_threshold": 0.5, + "class_name": "building", + "min_confidence": 0.5 +} +``` + +Response persists a `quality_check` and `metrics` rows through the existing QA/QC persistence architecture and returns: + +- `precision` +- `recall` +- `f1_score` +- `mean_iou` +- `false_positives` +- `false_negatives` +- `quality_check_id` + +If the reference dataset has no persisted vector features, the endpoint returns `REFERENCE_FEATURES_NOT_FOUND`. It does not calculate fake QA metrics. + +### POST `/api/v1/analysis/building-stats` + +Input: area + vector building layer. + +### POST `/api/v1/analysis/object-detection` + +Request: + +```json +{ + "project_id": "uuid", + "area_id": "uuid", + "dataset_id": "uuid", + "model_id": "optional uuid", + "classes": ["building"], + "confidence_threshold": 0.35, + "tile_size": 640, + "overlap": 64 +} +``` + +Response: `AnalysisRunRead`. + +### POST `/api/v1/analysis/segmentation` + +Same pattern as object detection, but output includes masks and polygonized geometries. + +## Segmentation Lab + +Sprint 9 implements Segmentation Lab foundation only. Real SAM and YOLO-seg inference are not enabled, no model is downloaded, and fixture segmentations require explicit fixture mode. + +### GET `/api/v1/segmentation/models` + +Returns segmentation model capability descriptors: + +- `segmentation-placeholder`: `not_configured` +- `fixture-segmenter`: configured for explicit test/demo fixtures only +- `yolo-seg-configured`: `not_configured` +- `sam-configured`: `not_configured` + +### POST `/api/v1/segmentation/run` + +Creates a segmentation job and segmentation analysis run. If the requested model is unavailable, the job and analysis run are marked `failed` with `SEGMENTATION_MODEL_UNAVAILABLE`. + +Request: + +```json +{ + "project_id": "uuid", + "dataset_id": "uuid", + "model_id": "segmentation-placeholder", + "confidence_threshold": 0.5, + "class_filter": ["vegetation"], + "tile_manifest_path": null, + "parameters_json": {} +} +``` + +Fixture segmenter mode is test/demo-only. It persists only explicit `parameters_json.fixture_segmentations` entries when `parameters_json.fixture_mode=true`; it is never invoked automatically and does not represent production inference. + +Validation errors: + +- `INVALID_DATASET_TYPE` when the dataset is not raster. +- `SEGMENTATION_MODEL_NOT_FOUND` when the model id is unknown. +- `FIXTURE_MODE_REQUIRED` when `fixture-segmenter` is requested without `parameters_json.fixture_mode=true`. +- `INVALID_FIXTURE_SEGMENTATIONS` when fixture payloads are not a list. +- `INVALID_FIXTURE_GEOMETRY` when fixture geometry is empty, invalid or not Polygon/MultiPolygon. + +### GET `/api/v1/segmentation/runs` + +Returns segmentation analysis runs, optionally filtered by `project_id` and `dataset_id`. + +### GET `/api/v1/segmentation/runs/{analysis_run_id}` + +Returns one segmentation analysis run. + +### GET `/api/v1/segmentation/runs/{analysis_run_id}/segmentations` + +Returns persisted segmentation records for a segmentation analysis run. Optional filters: + +- `dataset_id` +- `class_name` +- `min_confidence` + +### GET `/api/v1/segmentation/datasets/{dataset_id}/segmentations` + +Returns persisted segmentation records for a raster dataset. Optional filters: + +- `analysis_run_id` +- `class_name` +- `min_confidence` + +### GET `/api/v1/segmentation/segmentations/{segmentation_id}` + +Returns one persisted segmentation record. + +### GET `/api/v1/segmentation/runs/{analysis_run_id}/geojson` + +Returns persisted segmentations for a run as a GeoJSON FeatureCollection. Geometry comes from persisted PostGIS segmentation geometry in EPSG:4326. + +Each feature includes: + +- `segmentation_id` +- `class_name` +- `confidence` +- `area_m2` +- `model_name` +- `model_version` +- `analysis_run_id` +- `dataset_id` +- `job_id` +- `source_tile_path` +- `tile_index` +- `mask_path` +- `bbox_json` +- `provenance_json` + +### GET `/api/v1/segmentation/datasets/{dataset_id}/geojson` + +Returns persisted segmentations for a dataset as a GeoJSON FeatureCollection. Optional filters match the segmentation list endpoint. + +### POST `/api/v1/segmentation/runs/{analysis_run_id}/qa/reference` + +Compares persisted segmentation geometries from an analysis run against persisted `vector_features` from a reference vector dataset. + +Request: + +```json +{ + "reference_dataset_id": "uuid", + "iou_threshold": 0.5, + "class_name": "vegetation", + "min_confidence": 0.5 +} +``` + +Response persists a `quality_check` and `metrics` rows through the existing QA/QC persistence architecture and returns precision, recall, F1, mean IoU and false positive/negative counts. + +If the segmentation run has no persisted geometries, the endpoint returns `SEGMENTATIONS_NOT_FOUND`. If the reference dataset has no persisted vector features, it returns `REFERENCE_FEATURES_NOT_FOUND`. It does not calculate fake QA metrics. + +### POST `/api/v1/analysis/change-detection` + +Request contains source analysis or datasets A/B and method. + +## QA/QC + +### POST `/api/v1/qa/detections-vs-reference` + +Request: + +```json +{ + "candidate_dataset_id": "uuid", + "reference_dataset_id": "uuid", + "iou_threshold": 0.5, + "area_id": "optional uuid" +} +``` + +Response is wrapped in the job envelope. On success, `result_json` includes precision, recall, F1, mean IoU, false positives, false negatives and `quality_check_id`. + +Sprint 7A persists the QA/QC result as: + +- `jobs`: execution state. +- `quality_checks`: domain result. +- `metrics`: individual measurements. + +Future Detection and Segmentation flows may add an `analysis_run_id` path without replacing persisted quality checks. + +### GET `/api/v1/projects/{project_id}/quality-checks` + +Lists persisted QA/QC quality checks for a project with metric rows. + +Response: + +```json +{ + "items": [ + { + "id": "uuid", + "project_id": "uuid", + "job_id": "uuid-or-null", + "analysis_run_id": "uuid-or-null", + "candidate_dataset_id": "uuid-or-null", + "reference_dataset_id": "uuid", + "check_type": "demo_candidate_vs_reference", + "status": "ok", + "score": 0.5, + "parameters_json": {}, + "findings_json": {}, + "metrics": [ + { + "metric_key": "precision", + "metric_value": 0.5 + } + ] + } + ], + "total": 1, + "limit": 50, + "offset": 0 +} +``` + +## Exports + +### POST `/api/v1/exports/geojson` + +Export detections, segmentations or vector layer to GeoJSON. + +Dataset vector export request: + +```json +{ + "export_kind": "dataset", + "dataset_id": "uuid", + "name": "optional-basename" +} +``` + +Detection run export request: + +```json +{ + "export_kind": "detection_run", + "analysis_run_id": "uuid", + "name": "optional-basename" +} +``` + +Segmentation run export request: + +```json +{ + "export_kind": "segmentation_run", + "analysis_run_id": "uuid", + "name": "optional-basename" +} +``` + +Response persists an `exports` row and writes a deterministic JSON artifact: + +```json +{ + "export_id": "uuid", + "path": "storage/exports/{project_id}/datasets/{target}/{name}.geojson", + "status": "ready", + "export_type": "dataset_geojson", + "metadata_json": { + "source": "dataset", + "feature_count": 0 + } +} +``` + +Vector dataset exports use the stored dataset GeoJSON. Detection and +segmentation exports use persisted first-class geometry records and the +existing Detection/Segmentation GeoJSON conversion services. Raster datasets +are rejected for dataset GeoJSON export. + +### POST `/api/v1/exports/metadata` + +Exports project metadata JSON for projects, datasets, persisted QA/QC summary +rows and existing export history. + +```json +{ + "project_id": "uuid", + "name": "optional-basename" +} +``` + +### GET `/api/v1/exports/projects/{project_id}/exports` + +Lists persisted export records for a project. + +### GET `/api/v1/exports/{export_id}` + +Returns one persisted export record. + +### GET `/api/v1/exports/{export_id}/content` + +Returns the stored JSON artifact content through the standard API envelope. + +### GET `/api/v1/exports/{export_id}/download` + +Downloads the stored JSON/GeoJSON export artifact as a raw file response with +`application/json` content type and a `Content-Disposition` attachment +filename. This endpoint intentionally does not use the JSON envelope because +it is a browser/file-download path; callers that need canonical API JSON should +use `/content`. + +### POST `/api/v1/exports/yolo` + +Export annotations/detections to YOLO format. + +### POST `/api/v1/exports/report` + +Creates a lightweight HTML project report artifact from persisted project, +dataset, QA/QC summary and export history state. This does not create a PDF +and does not introduce a report designer. + +```json +{ + "project_id": "uuid", + "name": "optional-basename" +} +``` + +Response persists an `exports` row with `export_type: +project_report_html`. Download the report through: + +```text +GET /api/v1/exports/{export_id}/download +``` + +PDF/report-designer functionality can be added after core GeoAI workflows work. diff --git a/docs/API_CONTRACT_FREEZE_M2.md b/docs/API_CONTRACT_FREEZE_M2.md new file mode 100644 index 00000000..5cc616fb --- /dev/null +++ b/docs/API_CONTRACT_FREEZE_M2.md @@ -0,0 +1,90 @@ +# API Contract Freeze M2 + +## Required V1 endpoints + +### Health + +`GET /health` + +Response: + +```json +{"status":"ok","service":"geointel-backend"} +``` + +### Projects + +`POST /projects` + +Request: + +```json +{"name":"Geel Building Detection","description":"Demo project","region":"Kempen"} +``` + +Response: project object. + +`GET /projects` + +Response: paginated projects. + +### Areas + +`POST /projects/{project_id}/areas` + +Request must contain GeoJSON polygon. + +### Datasets + +`POST /projects/{project_id}/datasets/upload` + +Multipart upload. Returns dataset object and metadata status. + +`GET /projects/{project_id}/datasets` + +Returns datasets. + +### Analysis runs + +`POST /analysis/object-detection` + +Request: + +```json +{ + "project_id": "uuid", + "dataset_id": "uuid", + "area_id": "uuid", + "model_id": "yolov8n-building-demo", + "confidence_threshold": 0.5 +} +``` + +Response: job envelope. + +`GET /analysis-runs/{id}` + +Returns run status, metrics, outputs. + +### QA/QC + +`POST /analysis/qaqc/building-detection-vs-reference` + +Request: + +```json +{ + "project_id": "uuid", + "prediction_run_id": "uuid", + "reference_dataset_id": "uuid", + "iou_threshold": 0.5 +} +``` + +Response: job envelope. + +### Exports + +`POST /exports/geojson` + +Exports detections, segmentations, QA findings, or vector layers. diff --git a/docs/API_EXAMPLE_RESPONSES.md b/docs/API_EXAMPLE_RESPONSES.md new file mode 100644 index 00000000..a71fe1a6 --- /dev/null +++ b/docs/API_EXAMPLE_RESPONSES.md @@ -0,0 +1,113 @@ +# API Example Responses + +## Error Envelope +```json +{ + "error": { + "code": "DATASET_UNSUPPORTED_FORMAT", + "message": "The uploaded file extension is not supported.", + "details": {"extension": ".txt"}, + "request_id": "req_01HY" + } +} +``` + +## Health +```json +{ + "status": "ok", + "services": { + "database": "ok", + "redis": "ok", + "storage": "ok" + }, + "version": "0.1.0" +} +``` + +## Project +```json +{ + "id": "prj_geel_demo", + "name": "Geel Building Detection Demo", + "description": "Portfolio demo for GeoAI object detection and GRB QA.", + "region": "Kempen", + "created_at": "2026-06-10T00:00:00Z" +} +``` + +## Area FeatureCollection +```json +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "id": "area_geel_center", + "name": "Geel Centrum", + "area_m2": 1250000 + }, + "geometry": { + "type": "Polygon", + "coordinates": [[[4.98,51.16],[5.00,51.16],[5.00,51.18],[4.98,51.18],[4.98,51.16]]] + } + } + ] +} +``` + +## Dataset Metadata +```json +{ + "id": "ds_grb_buildings_geel", + "project_id": "prj_geel_demo", + "name": "GRB Buildings Geel Demo", + "dataset_type": "vector", + "source": "demo_fixture", + "status": "ready", + "crs": "EPSG:4326", + "bounds": [4.98, 51.16, 5.00, 51.18], + "metadata": { + "feature_count": 12, + "geometry_types": ["Polygon"] + } +} +``` + +## Detection Run +```json +{ + "id": "run_detection_001", + "project_id": "prj_geel_demo", + "analysis_type": "object_detection", + "status": "completed", + "parameters": { + "model_id": "demo-yolo-buildings-v1", + "confidence_threshold": 0.35, + "tile_size": 512 + }, + "metrics": { + "detection_count": 14, + "mean_confidence": 0.82 + } +} +``` + +## QA/QC Result +```json +{ + "id": "qc_001", + "analysis_run_id": "run_detection_001", + "reference_dataset_id": "ds_grb_buildings_geel", + "iou_threshold": 0.5, + "metrics": { + "true_positive": 11, + "false_positive": 3, + "false_negative": 1, + "precision": 0.7857, + "recall": 0.9167, + "f1": 0.8462 + } +} +``` diff --git a/docs/API_SPECIFICATION.md b/docs/API_SPECIFICATION.md new file mode 100644 index 00000000..bff4a22d --- /dev/null +++ b/docs/API_SPECIFICATION.md @@ -0,0 +1,229 @@ +# API Specification v1.0 + +Basispad: `/api/v1` + +## Projects + +### GET /projects + +Geeft alle projecten terug. + +### POST /projects + +Maakt een project aan. + +Body: + +```json +{ + "name": "Geel gebouwdetectie demo", + "description": "Detectie en QA/QC van gebouwen in Geel", + "region": "Kempen" +} +``` + +### GET /projects/{project_id} + +Geeft projectdetails terug. + +### DELETE /projects/{project_id} + +Verwijdert een project en gekoppelde metadata. Bestanden moeten veilig afgehandeld worden. + +## Areas + +### GET /projects/{project_id}/areas + +Geeft analysegebieden van een project. + +### POST /projects/{project_id}/areas + +Maakt analysegebied aan. + +Body: + +```json +{ + "name": "Geel Centrum", + "geometry": { "type": "Polygon", "coordinates": [] } +} +``` + +### GET /areas/{area_id} + +Geeft gebieddetails. + +## Datasets + +### POST /projects/{project_id}/datasets/upload + +Uploadt raster of vector dataset. + +Multipart: + +- file +- name +- dataset_type +- source + +### GET /projects/{project_id}/datasets + +Lijst datasets. + +### GET /datasets/{dataset_id} + +Datasetdetails. + +### GET /datasets/{dataset_id}/metadata + +Metadata. + +### POST /datasets/{dataset_id}/extract-metadata + +Start metadata extraction job. + +## Reference Data + +### POST /projects/{project_id}/reference/grb/fetch + +Haalt GRB-data op voor een area. + +Body: + +```json +{ + "area_id": "uuid", + "layers": ["buildings"] +} +``` + +### POST /projects/{project_id}/reference/osm/fetch + +Haalt OSM-data op voor een area. + +## Raster + +### POST /datasets/{dataset_id}/raster/clip + +Clipt raster op area. + +### POST /datasets/{dataset_id}/raster/tile + +Maakt tiles voor AI-inference. + +Body: + +```json +{ + "tile_size": 640, + "overlap": 64, + "area_id": "uuid" +} +``` + +### POST /datasets/{dataset_id}/raster/indices/ndvi + +V2: berekent NDVI. + +## Vector + +### POST /datasets/{dataset_id}/vector/clip + +Clipt vectorlaag op area. + +### POST /datasets/{dataset_id}/vector/buffer + +Maakt buffers. + +### POST /datasets/{dataset_id}/vector/validate + +Valideert geometrieën. + +## Analysis + +### POST /analysis/object-detection + +Start objectdetectie. + +Body: + +```json +{ + "project_id": "uuid", + "area_id": "uuid", + "dataset_id": "uuid", + "model_id": "uuid-or-default", + "classes": ["building"], + "confidence_threshold": 0.35 +} +``` + +### POST /analysis/segmentation + +Start segmentatie. + +### POST /analysis/change-detection + +Start change detection. + +### GET /analysis/{analysis_run_id} + +Geeft runstatus en resultaten. + +### GET /analysis/{analysis_run_id}/detections + +Geeft detecties als GeoJSON FeatureCollection. + +### GET /analysis/{analysis_run_id}/segmentations + +Geeft segmentaties als GeoJSON FeatureCollection. + +## QA/QC + +### POST /analysis/{analysis_run_id}/quality-check + +Vergelijkt AI-resultaten met referentiedataset. + +Body: + +```json +{ + "reference_dataset_id": "uuid", + "iou_threshold": 0.5, + "class_mapping": { + "building": "building" + } +} +``` + +### GET /quality-checks/{quality_check_id} + +Geeft QA/QC-resultaten. + +## Exports + +### POST /exports/geojson + +Exporteert analysis output naar GeoJSON. + +### POST /exports/yolo + +Exporteert annotaties naar YOLO-formaat. + +### POST /exports/coco + +Exporteert annotaties naar COCO-formaat. + +### GET /exports/{export_id}/download + +Download exportbestand. + +## Jobs + +### GET /jobs/{job_id} + +Geeft jobstatus. + +### GET /projects/{project_id}/jobs + +Geeft jobs van een project. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 00000000..b63a45e4 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,148 @@ +# Architecture + +## 1. Overzicht + +GeoIntel bestaat uit: + +- React/TypeScript frontend +- FastAPI backend +- PostgreSQL/PostGIS database +- background job queue +- file/object storage +- GIS processing services +- AI inference services + +## 2. Hoofdcomponenten + +```text +Frontend + ↓ REST/WebSocket +FastAPI Backend + ↓ +PostgreSQL + PostGIS + ↓ +Storage: uploads, processed rasters, tiles, masks, exports + ↓ +Workers: GIS processing, AI inference, QA/QC, export +``` + +## 3. Frontend + +Aanbevolen stack: + +- React +- TypeScript +- MapLibre GL +- Deck.gl +- Tailwind +- TanStack Query +- Zustand of vergelijkbare lichte state store +- Recharts voor eenvoudige grafieken + +Belangrijke principes: + +- kaart centraal, maar analysepanelen even belangrijk +- labs per workflow +- duidelijke jobstatus +- outputs altijd exporteerbaar +- geen verborgen mockgedrag + +## 4. Backend + +Aanbevolen stack: + +- FastAPI +- SQLAlchemy 2.x +- GeoAlchemy2 +- Alembic +- Pydantic +- RQ/Celery +- Rasterio +- GeoPandas +- Shapely +- PyProj +- NumPy +- OpenCV +- Ultralytics/PyTorch + +## 5. Database + +PostgreSQL met PostGIS is verplicht voor: + +- projectgebieden +- vectorfeatures +- detectiepolygonen +- segmentatiepolygonen +- spatial joins +- intersects +- IoU berekeningen +- bounds queries + +## 6. Storage + +Bewaar grote bestanden niet in de database. + +Opslagcategorieën: + +- originele uploads +- verwerkte rasters +- raster tiles +- masks +- model outputs +- exports +- rapporten + +Database bewaart metadata en paden. + +## 7. Jobs + +Langlopende processen moeten via background jobs: + +- raster metadata extraction +- raster clipping +- raster tiling +- vector import +- AI inference +- segmentation polygonize +- QA/QC +- change detection +- export generation + +## 8. AI Inference + +Inference pipeline: + +```text +Raster dataset +→ clip to area +→ tile raster +→ normalize/preprocess +→ model inference +→ convert pixel coords to geospatial coords +→ merge/filter outputs +→ save detections/segmentations +→ expose as map layer +``` + +## 9. CRS-regels + +- Alle interne geometrieën worden opgeslagen in PostGIS met bekende SRID. +- Voor metrische berekeningen wordt een geschikte projectie gebruikt. +- API-output naar frontend mag in EPSG:4326 of WebMercator-compatible formaat. +- Elke dataset zonder CRS krijgt status `needs_crs_review`. + +## 10. Developmentstrategie + +Bouwvolgorde: + +1. backend foundation +2. database schema +3. project/area API +4. dataset upload en metadata +5. frontend workspace en kaart +6. vector import +7. raster import +8. processing jobs +9. detection lab +10. QA/QC +11. export diff --git a/docs/BACKEND_PACKAGE_MAP.md b/docs/BACKEND_PACKAGE_MAP.md new file mode 100644 index 00000000..72c798fa --- /dev/null +++ b/docs/BACKEND_PACKAGE_MAP.md @@ -0,0 +1,80 @@ +# Backend Package Map + +Target backend layout: + +```text +backend/ + app/ + main.py + api/ + router.py + routes/ + health.py + projects.py + areas.py + datasets.py + raster.py + vector.py + analysis.py + qaqc.py + exports.py + core/ + config.py + database.py + logging.py + security.py + errors.py + models/ + project.py + area.py + dataset.py + layer.py + analysis.py + detection.py + segmentation.py + quality.py + export.py + job.py + event.py + schemas/ + common.py + project.py + area.py + dataset.py + analysis.py + quality.py + export.py + services/ + project_service.py + area_service.py + dataset_service.py + raster_service.py + vector_service.py + grb_service.py + detection_service.py + segmentation_service.py + qaqc_service.py + export_service.py + event_service.py + workers/ + queue.py + jobs.py + repositories/ + base.py + projects.py + areas.py + datasets.py + analysis_runs.py + scripts/ + seed_demo.py + tests/ +``` + +## Rules + +- API routes should be thin. +- Business logic belongs in services. +- SQLAlchemy query composition belongs in repositories where useful. +- Pydantic schemas are the API contract. +- Models are persistence structures, not API response structures. +- GIS processing functions must be testable outside HTTP handlers. diff --git a/docs/BUILD_GOVERNANCE.md b/docs/BUILD_GOVERNANCE.md new file mode 100644 index 00000000..c546d7d2 --- /dev/null +++ b/docs/BUILD_GOVERNANCE.md @@ -0,0 +1,9 @@ +# Build Governance + +Elke pass heeft doel, inputdocumenten, taken, niet-doen lijst, acceptatiecriteria, testcommando's en handoff-output. + +Codex werkt altijd bij: `CHANGELOG.md`, `docs/TODO.md`, relevante contractdocs en `docs/CODEX_EXECUTION_LOG.md`. + +Stopregels: backend start niet, frontend buildt niet, migrations falen, API-contracten inconsistent, demo-flow breekt. + +Visuele polish pas na werkende dataflow, API en tests/smoke checks. diff --git a/docs/BUILD_STATUS.md b/docs/BUILD_STATUS.md new file mode 100644 index 00000000..d501264d --- /dev/null +++ b/docs/BUILD_STATUS.md @@ -0,0 +1,38 @@ +# GeoIntel Build Status + +Current preparation milestone: M7 Implementation Control Layer. + +## Done + +- Product blueprint. +- Data specifications. +- Architecture specifications. +- API/database/service documentation. +- Codex build plans and prompts. +- Operational readiness docs. +- Autonomy pack. +- M7 build control, regression traps and self-review layer. + +## Ready for Codex + +Codex can begin with repository verification and backend foundation using the locked build sequence. + +## Must Preserve + +- GeoIntel is a GeoAI Workbench for the Kempen. +- GRB-first reference strategy. +- FastAPI + React + PostGIS. +- API-driven frontend. +- CRS-aware geospatial processing. +- Fixture mode must be clearly labeled. + +## Known Limitations Before Code Build + +- Real GRB WFS integration still needs implementation. +- Real YOLO/SAM inference should follow fixture boundary first. +- Sentinel and LiDAR remain post-foundation roadmap items. +- No production authentication in V1. + +## Next Recommended Codex Pass + +Run `prompts/codex/PASS_00_REPO_AUDIT.md`, then implement backend foundation according to `docs/12-build-control/BUILD_SEQUENCE_LOCK.md`. diff --git a/docs/BUILD_TICKETS_M3.md b/docs/BUILD_TICKETS_M3.md new file mode 100644 index 00000000..053fd887 --- /dev/null +++ b/docs/BUILD_TICKETS_M3.md @@ -0,0 +1,203 @@ +# Build Tickets M3 + +This file converts the implementation epics into concrete Codex-ready tickets. + +## Ticket format + +Each ticket must be implemented with: + +- backend changes if applicable +- frontend changes if applicable +- tests where applicable +- documentation updates +- changelog entry + +## T-001 Backend package scaffold + +Create FastAPI package structure: + +```text +backend/app/main.py +backend/app/api/router.py +backend/app/core/config.py +backend/app/core/database.py +backend/app/models/ +backend/app/schemas/ +backend/app/services/ +backend/app/workers/ +backend/tests/ +``` + +Acceptance: + +- `GET /health` returns status ok. +- backend imports cleanly. +- tests can run without external geospatial data. + +## T-002 Database and Alembic scaffold + +Add SQLAlchemy and Alembic setup for PostgreSQL/PostGIS. + +Acceptance: + +- database URL comes from environment. +- migrations directory exists. +- first migration creates PostGIS extension if available. +- migration plan documented. + +## T-003 Project model and API + +Implement project entity. + +Acceptance: + +- create project +- list projects +- read project +- update project +- soft delete or archive project +- response envelope followed + +## T-004 Area model and geometry API + +Implement areas linked to projects. + +Acceptance: + +- create polygon area as GeoJSON +- validate geometry +- store geometry in PostGIS +- calculate area in square meters using projected CRS +- return bounds and centroid + +## T-005 Frontend foundation + +Create React + TypeScript app structure. + +Acceptance: + +- app boots +- route layout exists +- API client exists +- error/loading components exist +- navigation includes Workspace, Map, Datasets, Raster, Vector, Detection, QA/QC, Exports + +## T-006 Map workbench foundation + +Implement map page and area drawing contract. + +Acceptance: + +- map displays Kempen default viewport +- user can draw or load an example polygon +- polygon can be submitted to backend as area +- active area can be selected + +## T-007 Dataset upload API + +Implement dataset registration and upload. + +Acceptance: + +- accepts GeoTIFF, GeoJSON, ZIP shapefile, GPKG placeholder handling +- stores original file under storage/originals +- creates dataset row +- status starts as uploaded +- returns metadata extraction job status + +## T-008 Raster metadata service + +Implement Rasterio metadata extraction. + +Acceptance: + +- CRS +- bounds +- width/height +- band count +- resolution +- nodata +- dtype +- transform +- summary stats for small rasters or sampled stats for large rasters + +## T-009 Vector metadata service + +Implement GeoPandas metadata extraction. + +Acceptance: + +- CRS +- bounds +- feature count +- geometry types +- columns +- invalid geometry count +- area summary where applicable + +## T-010 GRB reference fetcher skeleton + +Implement service contract for GRB WFS fetch. + +Acceptance: + +- service accepts area geometry +- builds BBOX or polygon filter request +- stores retrieved features as dataset/layer +- if live WFS unavailable, returns a clear source_unavailable status without crashing + +## T-011 Detection pipeline interface + +Implement detection run model and interface. + +Acceptance: + +- request creates analysis_run +- job lifecycle status exists +- deterministic fixture inference can populate detections for demo fixtures +- output geometries are stored and exported as GeoJSON + +## T-012 QA/QC engine v1 + +Implement reference-vs-prediction matching. + +Acceptance: + +- IoU threshold configurable, default 0.5 +- precision, recall, F1 calculated +- false positives and false negatives classified +- QA result stored +- fixture test passes with known expected metrics + +## T-013 Export API + +Implement export registry and GeoJSON export. + +Acceptance: + +- export detection results as FeatureCollection +- export QA false positives/negatives as FeatureCollection +- export analysis summary JSON +- exports have stable file paths under storage/exports + +## T-014 Frontend dataset and analysis panels + +Implement visible pages for dataset, detection and QA workflows. + +Acceptance: + +- dataset list shows status and metadata +- detection page starts run and shows result state +- QA page shows metrics and error classes +- map overlays are connected to available GeoJSON outputs + +## T-015 Stabilization pass + +Acceptance: + +- no broken navigation +- no unhandled promise rejections +- backend health green +- core tests green +- README quickstart updated +- CHANGELOG updated diff --git a/docs/CHANGELOG_M4.md b/docs/CHANGELOG_M4.md new file mode 100644 index 00000000..e4b75e2f --- /dev/null +++ b/docs/CHANGELOG_M4.md @@ -0,0 +1,18 @@ +# Changelog M4 + +## Added +- Autonomous build readiness specification. +- M4 sprint board. +- Module build contracts. +- Acceptance test catalog. +- API example response contracts. +- Job lifecycle contract. +- Frontend state and route contracts. +- Backend service IO contracts. +- Model registry seed specification. +- Demo fixture manifest. +- Codex autonomous runbook. +- Codex pass prompts for backend, database, dataset manager, map workspace, AI demo pipelines and QA/QC exports. + +## Intent +M4 prepares the repository for long Codex implementation sessions with minimal human intervention. diff --git a/docs/CHANGE_DETECTION_SPEC.md b/docs/CHANGE_DETECTION_SPEC.md new file mode 100644 index 00000000..5a3088b6 --- /dev/null +++ b/docs/CHANGE_DETECTION_SPEC.md @@ -0,0 +1,116 @@ +# GeoIntel Kempen — Change Detection Specification v1.0 + +Change Detection is a major showcase workflow combining raster, vector, AI and QA/QC. + +## Goal + +Compare two datasets or analysis runs for the same area and detect additions, removals and significant changes. + +## Supported methods + +## Method A — Vector change detection V1 + +Compare two vector layers or two analysis outputs. + +Examples: + +- GRB buildings snapshot A vs snapshot B. +- AI detections from raster A vs AI detections from raster B. +- OSM buildings from run A vs OSM buildings from run B. + +### Inputs + +- layer A +- layer B +- area polygon +- class filter optional +- matching threshold + +### Algorithm + +1. Normalize CRS. +2. Clip both layers to area. +3. Match features using IoU or spatial overlap. +4. Classify: + - added: feature in B without match in A + - removed: feature in A without match in B + - unchanged: matched with stable geometry + - modified: matched but area or geometry changed above threshold + +### Metrics + +- added count +- removed count +- modified count +- added area m² +- removed area m² +- net area change m² +- percentage change + +## Method B — Raster index change V2 + +Compare NDVI/NDWI/NDBI rasters. + +### Inputs + +- index raster A +- index raster B +- threshold + +### Algorithm + +```text +delta = index_B - index_A +classify pixels by threshold +polygonize changed zones +``` + +### Outputs + +- change raster +- changed polygons +- summary statistics + +## Method C — AI segmentation change V3 + +Run segmentation on both images and compare class polygons. + +Examples: + +- vegetation loss +- new buildings +- water change + +## Output layers + +- `change_added` +- `change_removed` +- `change_modified` +- `change_heatmap` +- `change_uncertain` + +## API + +```http +POST /analysis/change-detection +GET /analysis/{id}/changes +POST /analysis/{id}/exports/change-geojson +``` + +## UI requirements + +Change Lab must support: + +- dataset/layer A selector +- dataset/layer B selector +- method selector +- area selector +- threshold controls +- timeline labels +- map overlays for added/removed/modified +- metrics cards +- export + +## V1 target + +Implement vector change detection. Raster and AI-based change detection are later phases. diff --git a/docs/CI_CD_SPECIFICATION.md b/docs/CI_CD_SPECIFICATION.md new file mode 100644 index 00000000..a4479a7b --- /dev/null +++ b/docs/CI_CD_SPECIFICATION.md @@ -0,0 +1,68 @@ +# CI/CD Specification + +## Doel +De CI/CD-pipeline moet elke wijziging snel valideren zonder zware AI- of GIS-jobs verplicht te maken. Zware checks krijgen aparte profielen. + +## Checkprofielen + +### `quick` +Moet lokaal binnen enkele minuten kunnen draaien. + +- Backend import check. +- Python lint/type smoke. +- Frontend install/build smoke. +- API schema consistency check. +- Geen ontbrekende verplichte documentatie. + +### `integration` +Draait met Docker Compose. + +- PostgreSQL/PostGIS start. +- Redis start. +- Backend start. +- Health endpoint geeft OK. +- Alembic migrations kunnen naar laatste versie. +- Test fixtures kunnen worden ingeladen. + +### `geospatial` +Draait alleen wanneer GDAL/Rasterio/GeoPandas beschikbaar zijn. + +- Raster metadata fixture. +- Vector fixture import. +- CRS-transformatie fixture. +- Clip operatie fixture. + +### `ai-light` +Draait zonder groot model. + +- Model registry laadt. +- Detection pipeline accepteert dummy model adapter. +- Outputcontract voor detections klopt. +- GeoJSON exportcontract klopt. + +### `ai-full` +Optioneel en niet verplicht voor elke commit. + +- YOLO/SAM echte modelrun op kleine fixture. +- Output wordt geprojecteerd naar kaartcoördinaten. +- QA/QC tegen referentievector draait. + +## Verplichte CI-stappen voor M1-builds +1. `scripts/check_repo_structure.sh` +2. `scripts/smoke_backend_import.sh` +3. `scripts/smoke_contracts.py` +4. `scripts/smoke_docs.py` + +## Verplichte CI-stappen zodra code bestaat +1. `pytest backend/tests` +2. `npm run typecheck` +3. `npm run build` +4. `alembic upgrade head` +5. `python scripts/validate_fixtures.py` + +## Build failure policy +Een build mag alleen als groen worden beschouwd wanneer: + +- Alle quick checks slagen. +- Bekende failures expliciet in `docs/KNOWN_LIMITATIONS_M3.md` of nieuwere limitation doc staan. +- Geen nieuwe regressies zonder vermelding in changelog. diff --git a/docs/CODEX_AUTONOMOUS_RUNBOOK_M4.md b/docs/CODEX_AUTONOMOUS_RUNBOOK_M4.md new file mode 100644 index 00000000..65309082 --- /dev/null +++ b/docs/CODEX_AUTONOMOUS_RUNBOOK_M4.md @@ -0,0 +1,46 @@ +# Codex Autonomous Runbook M4 + +Use this runbook when starting an autonomous implementation session. + +## Before Coding +1. Read `README.md`. +2. Read `AGENTS.md`. +3. Read `docs/DEVELOPMENT_RULES.md`. +4. Read `docs/M4_AUTONOMOUS_BUILD_READINESS.md`. +5. Read `docs/SPRINT_BOARD_M4.md`. +6. Read module contract for the sprint being implemented. + +## Implementation Rules +- Build in the sprint order unless explicitly instructed otherwise. +- Do not skip backend tests for frontend work. +- Do not add a new dependency without updating dependency documentation. +- Do not invent new architecture where a contract already exists. +- Preserve fixture determinism. +- Update TODO and changelog after each pass. + +## End-of-Pass Report Format +Each Codex pass must end with: + +```md +## Completed +- ... + +## Tests Run +- ... + +## Changed Files +- ... + +## Remaining TODO +- ... + +## Risks / Blockers +- ... +``` + +## Stop Conditions +Stop and ask for human direction only if: +- A required external credential is missing. +- A core architecture decision conflicts across documents. +- A dependency cannot be installed or replaced safely. +- Data license terms are unclear for a live connector. diff --git a/docs/CODEX_BOOTSTRAP_PROMPT.md b/docs/CODEX_BOOTSTRAP_PROMPT.md new file mode 100644 index 00000000..343cd1ef --- /dev/null +++ b/docs/CODEX_BOOTSTRAP_PROMPT.md @@ -0,0 +1,49 @@ +# Codex Bootstrap Prompt — GeoIntel Kempen + +You are building GeoIntel Kempen, a GeoAI Workbench for the Belgian Kempen region. The repository already contains the specification set. Read these documents before editing code: + +1. `docs/SPECIFICATION_FREEZE_M0.md` +2. `docs/V1_SCOPE_FREEZE.md` +3. `docs/DEVELOPMENT_RULES.md` +4. `docs/REPOSITORY_CONVENTIONS.md` +5. `docs/SERVICE_ARCHITECTURE.md` +6. `docs/API_CONTRACTS.md` +7. `docs/DATABASE_IMPLEMENTATION_PLAN.md` +8. `docs/CODEX_EXECUTION_PLAN.md` +9. `docs/TEST_STRATEGY.md` +10. `docs/DEFINITION_OF_DONE.md` + +## Non-negotiable build rules + +- Build backend-first and API-driven. +- Do not silently replace geospatial logic with fake logic. +- Mock data is allowed only as explicit fixtures in `fixtures/` or `tests/fixtures/`. +- Every implemented endpoint must have request/response schemas. +- Every implemented service must have at least one focused test or a documented reason why it cannot yet be tested. +- Every feature must expose error states, loading states and empty states in the frontend. +- Do not introduce new major dependencies without adding them to `docs/DEPENDENCY_POLICY.md` and explaining why they are needed. +- Do not expand V1 scope beyond `docs/V1_SCOPE_FREEZE.md`. +- Preserve existing docs and update them when implementation diverges. + +## First build objective + +Create a runnable foundation with: + +- FastAPI backend skeleton. +- React + TypeScript frontend skeleton. +- Docker Compose with PostGIS, backend and frontend. +- Health endpoints. +- Project and Area CRUD. +- Dataset metadata model. +- Repository/service structure matching `docs/REPOSITORY_CONVENTIONS.md`. +- Initial tests and lint commands. + +## Required handoff after every pass + +Update `docs/CODEX_EXECUTION_LOG.md` with: + +- What changed. +- What was tested. +- What remains open. +- Known limitations. +- Next recommended pass. diff --git a/docs/CODEX_BUILD_PLAN.md b/docs/CODEX_BUILD_PLAN.md new file mode 100644 index 00000000..198f0d4e --- /dev/null +++ b/docs/CODEX_BUILD_PLAN.md @@ -0,0 +1,208 @@ +# Codex Build Plan + +Dit document is de uitvoeringsvolgorde voor Codex. Werk fase per fase. Sla geen fases over tenzij technisch noodzakelijk. + +## Algemene bouwregels + +- Lees eerst README.md, AGENTS.md en alle docs in deze map. +- Bouw documentatiegedreven. +- Geen quick hacks. +- Geen mock-only functionaliteit als echte functionaliteit verwacht wordt. +- Voeg bij elke fase tests toe waar zinvol. +- Update TODO.md na elke afgeronde fase. +- Houd frontend en backend consistent. +- API-contracten volgen API_SPECIFICATION.md. +- Database volgt DATABASE_SCHEMA.md. + +## Fase 1 — Repo Foundation + +Doel: werkende basisrepo. + +Taken: + +1. Backend scaffold maken in `backend/`. +2. Frontend scaffold maken in `frontend/`. +3. Docker compose toevoegen voor backend, frontend, db, redis. +4. `.env.example` toevoegen. +5. Basis README quickstart bijwerken. +6. Health checks toevoegen. + +Acceptatiecriteria: + +- backend start lokaal +- frontend start lokaal +- database container start +- health endpoint werkt + +## Fase 2 — Database Foundation + +Doel: PostGIS schema opzetten. + +Taken: + +1. SQLAlchemy + GeoAlchemy2 toevoegen. +2. Alembic initialiseren. +3. projects, areas, datasets, analysis_runs toevoegen. +4. PostGIS extensie activeren. +5. Spatial indexes voorzien. + +Acceptatiecriteria: + +- migraties draaien succesvol +- tests kunnen project en area aanmaken +- geometrie wordt correct opgeslagen + +## Fase 3 — Project & Area API + +Doel: projecten en gebieden beheren. + +Taken: + +1. CRUD routes voor projects. +2. CRUD routes voor areas. +3. GeoJSON polygon validatie. +4. Area oppervlakte berekenen. +5. Frontend project list/detail. +6. Map draw basis. + +Acceptatiecriteria: + +- gebruiker kan project maken +- gebruiker kan polygon tekenen en opslaan +- area verschijnt op kaart + +## Fase 4 — Dataset Manager + +Doel: datasets uploaden en metadata lezen. + +Taken: + +1. Upload endpoint. +2. Bestandsopslag in storage/uploads. +3. Raster metadata extraction met Rasterio. +4. Vector metadata extraction met GeoPandas/Fiona. +5. Dataset list/detail UI. +6. Statussen tonen. + +Acceptatiecriteria: + +- GeoTIFF upload toont CRS, bounds, bands, resolutie +- GeoJSON/shapefile upload toont CRS, bounds, feature count + +## Fase 5 — Raster Core + +Doel: rasterdata verwerken. + +Taken: + +1. Raster clipping by area. +2. Raster preview generation. +3. Raster tiling met tile metadata. +4. Tile opslag. +5. UI acties voor clip/tile. + +Acceptatiecriteria: + +- raster kan op area geclipt worden +- tiles worden aangemaakt +- tile metadata bevat transform en bounds + +## Fase 6 — Vector Core + +Doel: vectordata verwerken. + +Taken: + +1. Vector import naar PostGIS. +2. CRS transform naar 4326. +3. Geometry validation/repair. +4. Vector clipping by area. +5. Feature count en area metrics. + +Acceptatiecriteria: + +- vectorlaag wordt zichtbaar op kaart +- clipping werkt +- features zitten in PostGIS + +## Fase 7 — Reference Data + +Doel: referentielaag beschikbaar maken. + +Taken: + +1. OSM fetcher als fallback implementeren. +2. GRB connector interface ontwerpen. +3. Reference datasets opslaan als datasets + vector_features. +4. UI voor referentielaag. + +Acceptatiecriteria: + +- OSM buildings kunnen voor area worden opgehaald +- referentielaag zichtbaar op kaart +- GRB-interface is voorbereid zonder app te blokkeren als externe endpoint nog niet volledig is + +## Fase 8 — Detection Lab + +Doel: objectdetectie end-to-end. + +Taken: + +1. Model config en YOLO wrapper. +2. Detection analysis run. +3. Inference op raster tiles. +4. Pixel bbox naar geospatial polygon. +5. Detecties opslaan. +6. Detectielaag tonen. +7. GeoJSON export. + +Acceptatiecriteria: + +- detection job kan gestart worden +- duidelijke fout als model ontbreekt +- detecties worden als polygonen op kaart getoond +- export werkt + +## Fase 9 — QA/QC Lab + +Doel: detecties vergelijken met referentie. + +Taken: + +1. IoU matching. +2. TP/FP/FN berekening. +3. Precision/recall/F1. +4. quality_checks en quality_findings opslaan. +5. QA UI met metrics en foutlagen. + +Acceptatiecriteria: + +- detection run kan vergeleken worden met referentielaag +- false positives/false negatives zichtbaar +- metrics kloppen op testfixtures + +## Fase 10 — Export & Portfolio Demo + +Doel: bruikbare einddemo. + +Taken: + +1. Exportcentrum. +2. Demo project seed. +3. Portfolio workflow documenteren. +4. Smoke tests. +5. README bijwerken. + +Acceptatiecriteria: + +- één demo-workflow werkt end-to-end +- docs beschrijven hoe demo te draaien +- tests groen + +## Niet doen in eerste bouwronde + +- geen volledige LiDAR Workbench +- geen eigen training studio +- geen complexe AI-copilot +- geen multi-user auth tenzij noodzakelijk +- geen zwaar dashboard boven analysefunctionaliteit diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md new file mode 100644 index 00000000..0f004939 --- /dev/null +++ b/docs/CODEX_EXECUTION_LOG.md @@ -0,0 +1,1128 @@ +# Codex Execution Log + +This file must be updated by Codex after each implementation pass. + +## Format + +```md +## Pass X — Title +Date: + +### Completed +- ... + +### Files changed +- ... + +### Tests run +- ... + +### Known limitations +- ... + +### Next recommended pass +- ... +``` + +## Initial status +Specification phase completed through M0. No code foundation has been implemented yet. + +## M2 Engineering Package + +- [x] Add ADR decision records. +- [x] Add RFC placeholders for future modules. +- [x] Add API/database/event contracts. +- [x] Add model registry and class catalogs. +- [x] Add queue architecture. +- [x] Add acceptance matrix and test catalog. +- [x] Add Codex M2 build prompts. +- [ ] Start Codex Pass 01 backend foundation. + +## M8 preparation + +Added the tomorrow execution layer for Codex autonomy: Day 1 master prompt, pass prompts, failure recovery, quality gates, operator checklist and smoke script scaffold. + +## Sprint 1 readiness hardening (2026-06-11) + +Date: 2026-06-11 + +### Completed +- Hardened backend dependency declarations for Sprint 1 runtime modules in `backend/pyproject.toml`. +- Normalized helper scripts for backend/frontend install/test/dev/readiness across `python`/`python3` interpreter availability. +- Fixed PostGIS/geospatial stack compatibility issues (`geojson_service` import path, package config). +- Updated backend/frontend/docs/readme commands for import smoke, setup, and readiness verification. +- Fixed frontend shell script line-ending parsing failures and added `npm` availability checks. +- Added missing frontend type path fixes and TypeScript typing corrections. +- Added/verified minimal Sprint 1 tests for health and GeoJSON parsing/rejection behaviors. + +### Files changed +- `backend/pyproject.toml` +- `backend/app/core/config.py` +- `backend/app/services/geojson_service.py` +- `backend/tests/test_health.py` +- `backend/tests/test_geojson_dataset_service.py` +- `backend/README.md` +- `frontend/package.json` +- `frontend/tsconfig.json` +- `frontend/src/components/GeoMap.tsx` +- `frontend/src/services/api/areas.ts` +- `frontend/src/services/api/projects.ts` +- `frontend/src/services/api/datasets.ts` +- `frontend/src/App.tsx` +- `frontend/README.md` +- `scripts/run_readiness_check.sh` +- `scripts/backend_install.sh` +- `scripts/backend_test.sh` +- `scripts/backend_dev.sh` +- `scripts/codex_pass_end_check.sh` +- `scripts/smoke_backend_import.sh` +- `scripts/frontend_install.sh` +- `scripts/frontend_typecheck.sh` +- `scripts/frontend_build.sh` +- `scripts/frontend_dev.sh` +- `README.md` +- `.env.example` +- `CHANGELOG.md` +- `Makefile` + +### Tests run +- `python -m compileall backend/app` (pass) +- `cd backend; python -m pytest` (pass, 5 tests) +- `cd backend; python -c "from app.main import app; print(app.title)"` (pass) +- `bash scripts/run_readiness_check.sh` (pass) +- `bash scripts/smoke_backend_import.sh` (pass) +- `bash scripts/frontend_install.sh` (pass) +- `bash scripts/frontend_typecheck.sh` (pass) +- `bash scripts/frontend_build.sh` (pass) +- `make`-based targets not runnable in this environment (`make` command missing) +- `docker compose config` not runnable in this environment (`docker` command missing) +- `python3` commands fail in this environment because `python3` maps to Microsoft Store stub; use `python` instead. + +### Known limitations +- `docker` and `make` are not installed in the current local shell environment. +- `python3` is not a usable interpreter in this environment; `python` must be used for all backend runtime/tests. + +### Next recommended pass +- Run `docker compose` validation and DB-backed migration checks in an environment with Docker + PostGIS service available. + +## Pass 12 — Sprint 2 Foundation +Date: 2026-06-11 + +### Completed +- Added vector/raster dataset typing and lifecycle states (`uploaded`, `validating`, `ready`, `failed`) in dataset service. +- Added vector metadata extraction details (feature counts, geometry types, bounds, area, CRS assumptions). +- Added raster metadata service with dependency-aware unavailable behavior and explicit `RASTER_PROCESSING_UNAVAILABLE` handling. +- Added dataset vector inspect/summary and raster metadata endpoints for project-scoped datasets. +- Persisted deterministic storage metadata for uploads (original/stored filename, MIME, size, checksum). +- Extended frontend dataset manager details panel with type/status/file metadata/feature counts and raster summary readiness. +- Added minimal Sprint 2 tests for vector metadata, legacy `geojson` compatibility, storage metadata persistence, and raster dependency fallback. + +### Files changed +- `backend/app/services/dataset_service.py` +- `backend/app/services/raster_service.py` +- `backend/app/services/geojson_service.py` +- `backend/app/schemas/dataset.py` +- `backend/app/api/routes/datasets.py` +- `backend/app/services/storage_service.py` +- `backend/tests/test_geojson_dataset_service.py` +- `backend/tests/test_raster_service.py` +- `backend/tests/test_storage_service.py` +- `backend/README.md` +- `frontend/src/App.tsx` +- `frontend/src/services/api/datasets.ts` +- `frontend/src/types.ts` +- `docs/API_CONTRACTS.md` +- `docs/CODEX_EXECUTION_LOG.md` +- `.env.example` +- `backend/README.md` +- `frontend/README.md` +- `CHANGELOG.md` +- `README.md` + +### Tests run +- Sprint 2 tests to be executed via readiness commands (see below). + +### Known limitations +- `rasterio` is not required by default; raster metadata extraction is unavailable until installed in the environment. +- Docker and PostGIS validation still depends on local availability of Docker/DB runtime. + +### Next recommended pass +- Keep Sprint 1/2 architecture; implement raster clipping/tiling APIs and status jobs before Detection/Segmentation modules. + +## Pass 13 — Sprint 2 validation and legacy compatibility hardening +Date: 2026-06-11 + +### Completed +- Verified Sprint 2 readiness commands in current environment (where tools are available). +- Revalidated backend tests and frontend typecheck/build after a Sprint 2 compatibility patch. +- Added frontend vector-detail compatibility for legacy `geojson` dataset rows in Sprint 1 records. +- Confirmed backend import smoke and dataset/raster/vector service behavior remain intact. + +### Files changed +- `frontend/src/App.tsx` +- `docs/CODEX_EXECUTION_LOG.md` + +### Tests run +- `python -m compileall backend/app` (pass) +- `cd backend && python -m pytest` (pass, 11 tests) +- `bash scripts/run_readiness_check.sh` (pass) +- `bash scripts/smoke_backend_import.sh` (pass) +- `cd frontend && npm run typecheck` (pass) +- `cd frontend && npm run build` (pass) +- `cd backend; python3 -m compileall backend/app` (fails: python3 shim unavailable in this shell) +- `cd backend; python3 -c "from app.main import app; print(app.title)"` (fails: python3 shim unavailable in this shell) +- `docker compose config` (fails: docker command unavailable in this shell) + +### Known limitations +- `python3` is not available in the current environment; use `python` commands for verification. +- `docker` is not available in the current environment. + +### Next recommended pass +- Run the same sprint verification commands in an environment with `python3` and `docker` installed. + +## Pass 14 — Sprint 3 operations + job foundation +Date: 2026-06-11 + +### Completed +- Added lightweight job model/migration and service layer with statuses `queued`, `running`, `success`, `failed`. +- Added job API endpoints for create/list/read/status under project scope. +- Added vector operation service foundation: + - inspect + - bbox + - stats + - clip by area + - buffer + - intersect +- Added raster operation foundation: + - inspect + - metadata + - preview readiness + - clip placeholder (dependency-aware) + - tile placeholder (dependency-aware) +- Added job-based execution wrappers for vector/raster operation endpoints. +- Updated dataset detail UI with available operations, job list/status, and derived output navigation. +- Added/updated Sprint 3 API contract and backend/frontend documentation updates. + +### Files changed +- `backend/app/schemas/operations.py` +- `backend/app/schemas/__init__.py` +- `backend/app/services/vector_operations_service.py` +- `backend/app/services/raster_operations_service.py` +- `backend/app/services/job_service.py` +- `backend/app/api/routes/jobs.py` +- `backend/app/models/entities.py` +- `backend/app/models/__init__.py` +- `backend/alembic/versions/20260611212435_add_jobs_table.py` +- `backend/tests/test_vector_operations_service.py` +- `backend/tests/test_raster_operations_service.py` +- `backend/tests/test_storage_service.py` +- `frontend/src/App.tsx` +- `frontend/src/services/api/datasets.ts` +- `frontend/src/services/api/jobs.ts` +- `frontend/src/types.ts` +- `backend/README.md` +- `frontend/README.md` +- `docs/API_CONTRACTS.md` +- `docs/CODEX_EXECUTION_LOG.md` +- `CHANGELOG.md` + +### Tests run +- `python -m compileall backend/app` (pass) +- `cd backend && python -m pytest` (pass) +- `bash scripts/run_readiness_check.sh` (pass) +- `bash scripts/smoke_backend_import.sh` (pass) +- `cd frontend && npm run typecheck` (pass) +- `cd frontend && npm run build` (pass) + +### Known limitations +- Raster processing beyond readiness/metadata is intentionally dependency-aware placeholder in Sprint 3. +- `docker` remains unavailable in this environment. +- `python3` still resolves to Windows Store stub; use `python` for verification. + +### Next recommended pass +- Install rasterio/GDAL for real raster clipping/tile implementations. +- Add queue-backed worker (Redis/RQ) behind existing synchronous job facade. +- Add deeper operation acceptance tests for output dataset geometry correctness and persisted metadata. + +## Pass 15 — Sprint 4 raster foundation +Date: 2026-06-11 + +### Completed +- Finalized dependency-aware raster processing behavior for clip/tile/preview and improved metadata enrichment. +- Ensured raster metadata and preview responses include deterministic file metadata (`size_bytes`, `checksum_sha256`, `path`) where available. +- Added raster tile manifest generation with deterministic storage paths and returned manifest payload. +- Updated API contracts and execution documentation from placeholder wording to actual raster-op foundations. +- Added targeted Sprint 4 tests for missing-area clip behavior and tile manifest payload shape. + +### Files changed +- `backend/app/services/raster_operations_service.py` +- `backend/app/api/routes/health.py` +- `backend/tests/test_raster_operations_service.py` +- `docs/API_CONTRACTS.md` +- `docs/CODEX_EXECUTION_LOG.md` +- `backend/README.md` +- `frontend/README.md` +- `README.md` +- `CHANGELOG.md` + +### Tests run +- `python -m compileall backend/app` +- `cd backend && python -m pytest` +- `bash scripts/run_readiness_check.sh` +- `bash scripts/smoke_backend_import.sh` +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` + +### Known limitations +- Raster dependency packages remain optional and will report `RASTER_PROCESSING_UNAVAILABLE` if absent. +- Raster clip/tile remain synchronous under the current in-process job wrapper. + +### Next recommended pass +- Run full raster end-to-end tests with real GeoTIFF fixtures and validate output dataset metadata persistence. +## Pass 16 Sprint 4 raster operations foundation hardening +Date: 2026-06-11 + +### Completed +- Realized the Sprint 4 raster foundation readiness in implementation and docs alignment: + - metadata extraction now returns real raster profile fields when rasterio is available + - preview generation writes deterministic PNG artifacts and reuses cached previews + - clip and tile operations persist deterministic outputs and manifest structure + - all raster processing paths now use explicit dependency-aware errors when rasterio/numpy/pillow are missing +- Fixed remaining frontend render/type issues introduced during raster path handling. +- Strengthened raster tests for tile manifest minimum size and dependency-aware behavior. +- Updated sprint milestone docs to reflect Sprint 4 status: + - `backend/README.md` + - `frontend/README.md` + +### Files changed +- `backend/app/services/raster_operations_service.py` +- `backend/tests/test_raster_operations_service.py` +- `backend/app/api/routes/datasets.py` +- `backend/app/schemas/operations.py` +- `frontend/src/App.tsx` +- `backend/README.md` +- `frontend/README.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Tests run +- `python -m compileall backend/app` (pass) +- `cd backend && python -m pytest` (pass, 26 tests) +- `bash scripts/run_readiness_check.sh` (pass) +- `bash scripts/smoke_backend_import.sh` (pass) +- `cd frontend && npm run typecheck` (pass) +- `cd frontend && npm run build` (pass) + +### Known limitations +- `docker` command is unavailable in this environment, so `docker compose config` could not be executed. +- `python3` command is unavailable as an alias in this environment; use `python` instead. +- Full raster dependency stack may be unavailable in some dev environments; those cases intentionally return structured `RASTER_PROCESSING_UNAVAILABLE` responses. + +### Next recommended pass +- Sprint 5: add end-to-end RasterOps coverage for CRS-preserving re-projection, export-quality metadata policy, and tile set artifact cleanup lifecycle. +## Pass 17 — Sprint 5 raster analytics hardening + +Date: 2026-06-11 + +### Completed +- Implemented Sprint 5 raster analytics foundation: + - band statistics endpoint and service with dependency-aware unavailable mode. + - reproject operation with CRS validation and reprojection metadata persistence. + - clip/tile hardening for missing/invalid geometry and empty clip output handling. + - enriched tile manifest fields (`tile_set_id`, `tile_size`, `overlap`, `source_dataset_id`, `source_raster_id`, `bounds`, `count`, `tile_paths`, `ai_inference`, `tile_server`, `created_at`, parameters). +- Updated raster stats/job error tests: + - dependency-aware stats failure coverage. + - invalid CRS request validation for reproject. + - persisted derived dataset assertions for reproject and clip. + - failure persistence test for failed raster jobs (`_run_job_sync`). +- Strengthened raster frontend detail panel rendering: + - job result JSON is visible for raster/vector operations. + - clearer raster metadata/status visibility retained for CRS/bounds/resolution display. +- Updated docs: + - backend/README Sprint 5 section. + - frontend/README Sprint 5 section. + - CHANGELOG entry for Sprint 5. + +### Files changed +- `backend/app/services/raster_operations_service.py` (final reproject and manifest hardening alignment) +- `backend/tests/test_raster_operations_service.py` +- `frontend/src/App.tsx` +- `backend/README.md` +- `frontend/README.md` +- `docs/API_CONTRACTS.md` +- `docs/RASTER_OPERATIONS_SPEC.md` +- `docs/CODEX_EXECUTION_LOG.md` +- `CHANGELOG.md` + +### Tests run +- `python -m compileall backend/app` (pass) +- `cd backend && python -m pytest` (pass) +- `bash scripts/run_readiness_check.sh` (pass) +- `bash scripts/smoke_backend_import.sh` (pass) +- `cd frontend && npm run typecheck` (pass) +- `cd frontend && npm run build` (pass) + +### Known limitations +- Raster statistics and reproject operations still depend on environment availability of `rasterio`/`numpy`. +- Raster dependency checks and operation errors remain dependency-aware when libraries are unavailable. + +### Next recommended pass +- Sprint 6: add raster index/mask workflows (NDVI/NDWI/NDBI), tile serving or export packaging, and AI-ready dataset linking. +## Pass 18 - Sprint 6 spectral indices + +### Completed +- Added local raster index operations (`ndvi`, `ndwi`, `ndbi`) under the existing raster operation architecture. +- Added typed band payload validation and explicit `INVALID_PARAMETERS` handling for missing/invalid band indexes. +- Implemented dependency-aware failure behavior for index execution (`RASTER_PROCESSING_UNAVAILABLE`) when `rasterio` or `numpy` are unavailable. +- Implemented local index output generation with float32 raster derivation and persisted provenance metadata on derived datasets. +- Extended dataset detail UI with spectral index controls (NDVI/NDWI/NDBI), run actions, and output dataset navigation from jobs. +- Updated API contracts, raster operation spec, and project documentation for Sprint 6 behavior. + +### Files changed +- `backend/app/schemas/operations.py` +- `backend/app/services/raster_operations_service.py` +- `backend/app/api/routes/datasets.py` +- `backend/tests/test_raster_operations_service.py` +- `frontend/src/services/api/datasets.ts` +- `frontend/src/App.tsx` +- `frontend/src/types.ts` +- `docs/API_CONTRACTS.md` +- `docs/RASTER_OPERATIONS_SPEC.md` +- `backend/README.md` +- `frontend/README.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Tests run +- `python -m compileall backend/app` +- `cd backend && python -m pytest` +- `bash scripts/run_readiness_check.sh` +- `bash scripts/smoke_backend_import.sh` +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` + +### Known limitations +- Raster dependency checks remain optional; missing raster packages return explicit `RASTER_PROCESSING_UNAVAILABLE` results. +- Job execution remains synchronous under current Sprint 3 job facade. +- `docker` and `python3` availability still depend on developer environment. + +### Next recommended pass +- Keep operation architecture stable, then add threshold/mask workflows and output export packaging in a follow-up pass. + + +## Pass 19 - RC-2 stabilization +Date: 2026-06-12 + +### Completed +- Fixed backend compile/import blockers identified in RC-1: + - corrected dataset upload parameter ordering. + - corrected schema package re-exports for area schemas. +- Fixed frontend typecheck/build blockers in `App.tsx`. +- Added Alembic migration for dataset reference/provenance metadata columns required by current ORM models. +- Fixed QA comparison runtime crash and added focused QA service coverage. +- Corrected envelope response-model mismatches for vector inspect and raster stats endpoints. +- Strengthened `scripts/run_readiness_check.sh` so readiness runs backend compile, backend tests, frontend typecheck and frontend build. +- Improved readiness Python interpreter selection so it chooses an interpreter capable of running pytest. + +### Files changed +- `backend/app/services/dataset_service.py` +- `backend/app/schemas/__init__.py` +- `backend/app/api/routes/datasets.py` +- `backend/app/services/qa_service.py` +- `backend/alembic/versions/202606120001_add_dataset_reference_metadata.py` +- `backend/tests/test_qa_service.py` +- `frontend/src/App.tsx` +- `scripts/run_readiness_check.sh` +- `docs/CODEX_EXECUTION_LOG.md` + +### Tests run +- `python -m compileall backend\\app` (pass) +- `cd backend && python -m pytest` (pass, 40 tests) +- `cd frontend && npm run typecheck` (pass) +- `cd frontend && npm run build` (pass; bundle-size warning only) +- `bash scripts/run_readiness_check.sh` (pass) +- `cd backend && python -m alembic heads` (pass, single head `202606120001`) +- `cd backend && python -m alembic upgrade head --sql` (pass, generated SQL includes required dataset metadata columns) +- `docker compose config` not runnable in this environment because `docker` is not installed. + +### Known limitations +- Fresh online database migration was validated only as generated Alembic SQL in this environment; run `cd backend && python -m alembic upgrade head` against a live PostgreSQL/PostGIS database on a machine with Docker or Postgres available. +- Existing non-RC architecture limitations from RC-1 remain intentionally unfixed: synchronous job facade, file-first vector outputs, monolithic frontend component, and incomplete future AI/storage architecture. + +### Next recommended pass +- Do not start Sprint 7 until RC-2 verification is repeated against a live Docker/PostGIS environment. + +## Pass 20 - Sprint 7A persistence and QA foundation +Date: 2026-06-12 + +### Completed +- Added first-class `vector_features` ORM model and Alembic migration with dataset and GiST geometry indexes. +- Persisted uploaded vector GeoJSON features into PostGIS-backed `vector_features` while keeping original file storage intact. +- Added first-class `quality_checks` and `metrics` ORM models and Alembic migration indexes. +- Added `QualityService` for persisted QA/QC domain records and metric rows. +- Updated QA candidate-vs-reference route so successful QA jobs also persist a `QualityCheck` and metrics, and return `quality_check_id` in `result_json`. +- Hardened provider capability contracts for GRB and OSM as `not_configured` stubs with supported layers, geometry types and query modes. +- Added Sprint 7A tests for vector feature persistence, quality check persistence, metrics persistence, dataset role validation, provider contracts, migration integrity and QA route persistence. +- Updated database/API documentation for Vector Features Architecture, Quality Check Architecture, Metrics Architecture and Provider Architecture. + +### Files changed +- `backend/app/models/entities.py` +- `backend/app/models/__init__.py` +- `backend/app/services/dataset_service.py` +- `backend/app/services/vector_feature_service.py` +- `backend/app/services/quality_service.py` +- `backend/app/api/routes/qa.py` +- `backend/app/providers/base.py` +- `backend/app/providers/grb.py` +- `backend/app/providers/osm.py` +- `backend/app/schemas/health.py` +- `backend/alembic/versions/202606120700_sprint7a_persistence_foundation.py` +- `backend/tests/test_sprint7a_persistence_foundation.py` +- `docs/DATABASE_IMPLEMENTATION_PLAN.md` +- `docs/API_CONTRACTS.md` +- `docs/CODEX_EXECUTION_LOG.md` +- `CHANGELOG.md` + +### Tests run +- `python -m compileall backend/app` (pass) +- `python -m pytest backend -q` (pass, 47 tests) + +### Known limitations +- Sprint 7A intentionally does not implement GRB downloads, OSM downloads, Detection Lab, Segmentation Lab, LiDAR, AI Copilot, Training Studio or Reports. +- Job execution remains the existing synchronous facade. +- Live database migration still needs validation against a running PostgreSQL/PostGIS service in an environment with Docker or Postgres available. + +### Next recommended pass +- Complete full release validation commands, including Alembic heads/SQL generation, readiness script, frontend typecheck/build and Docker config if Docker is available. + +### Validation addendum +Date: 2026-06-12 + +Additional Sprint 7A validation completed after migration index cleanup: + +- `python -m compileall backend/app` (pass) +- `cd backend && python -m pytest` (pass, 47 tests) +- `bash scripts/run_readiness_check.sh` (pass) +- `cd frontend && npm run typecheck` (pass) +- `cd frontend && npm run build` (pass; Vite chunk-size warning only) +- `cd backend && python -m alembic heads` (pass, single head `202606120700`) +- `cd backend && python -m alembic upgrade head --sql` (pass; generated SQL includes `vector_features`, `quality_checks`, `metrics` and the named GiST index `ix_vector_features_geometry`) +- `docker compose config` could not run because Docker is not installed in this shell. + +## Pass 21 - Sprint 7B provider integration skeleton (2026-06-12) + +- Implemented central provider registry for `grb`, `osm`, `manual` and `fixture`. +- Added provider capability, layer, status and future import-contract endpoints using the existing response envelope style. +- Preserved GRB and OSM as explicit `not_configured` providers; no live WFS, Overpass, download or fake provider data was introduced. +- Documented and tested provider-to-dataset mapping rules; future provider output must flow through `DatasetService` / `VectorFeatureService` rather than direct `vector_features` writes. +- Added frontend Provider Capabilities panel without live import buttons for GRB/OSM. +- Added opt-in `scripts/live_migration_smoke.sh` for real PostGIS migration smoke checks. +- Added Sprint 7B tests for provider registry, API envelopes, invalid provider handling, import contract and smoke script presence. + +## Pass 22 - Sprint 8 Detection Lab foundation (2026-06-12) + +### Completed +- Added first-class `detections` ORM model and Alembic migration with project, dataset, analysis run, class and GiST geometry indexes. +- Hardened `analysis_runs` with dataset, job, model, result and created-at fields while keeping jobs conceptually separate from analysis lifecycle. +- Added model registry capability service for `yolo-placeholder` (`not_configured`) and `manual-fixture-detector` (explicit fixture/demo only). +- Added `DetectionService` boundary for model listing, request validation, analysis run creation, job creation, unavailable model responses and explicit fixture persistence. +- Added Detection Lab API endpoints under `/api/v1/detection` using the existing response envelope style. +- Added minimal frontend Detection Lab panel for model capability status, raster dataset selection, confidence threshold and run result/error display. +- Updated database, API, AI pipeline, backend/frontend README, TODO and changelog docs. + +### Known limitations +- Real YOLO/PyTorch inference is not enabled and no model downloads are performed. +- Fixture detector requires `fixture_mode=true` and explicit fixture detections; it is not production inference. +- Segmentation, LiDAR, AI Copilot, Training Studio and Reports remain out of scope. + +### Next recommended pass +- Verify Sprint 8 with full backend/frontend/readiness/Alembic gates, then perform a Sprint 8 verification audit before Sprint 8B real YOLO integration. + +## Pass 23 - Sprint 8B configured YOLO foundation (2026-06-12) + +### Completed +- Added optional backend `ai` dependency group for `ultralytics` and `torch`; normal backend startup remains import-safe without those packages. +- Added YOLO configuration settings: + - `YOLO_ENABLED` + - `YOLO_MODEL_PATH` + - `YOLO_MODEL_ID` + - `YOLO_MODEL_DISPLAY_NAME` + - `YOLO_MODEL_VERSION` + - `YOLO_DEVICE` + - `YOLO_IMAGE_SIZE` + - `YOLO_MAX_TILES` + - `YOLO_BATCH_SIZE` +- Added `yolo-configured` model registry capability with honest `not_configured`, `dependency_unavailable` and `configured` states. +- Added `YoloDetectionAdapter` that imports Ultralytics only in the load path and refuses missing local model files before model construction. +- Added raster tile manifest validation and configured tile-limit enforcement for real YOLO runs. +- Added pixel bbox to EPSG:4326 polygon georeferencing from tile transform or bounds metadata. +- Routed configured YOLO outputs through existing `DetectionService`, `Job`, `AnalysisRun` and first-class `Detection` persistence. +- Added Detection Lab tile manifest path input for the configured YOLO model. +- Added mocked Sprint 8B tests for model registry status, dependency-unavailable behavior, tile manifest validation, georeferencing and persisted detections. +- Updated API, AI pipeline, backend/frontend README and changelog documentation. + +### Known limitations +- Sprint 8B does not add workers/queues; configured YOLO runs remain synchronous behind the existing job abstraction. +- Real model loading is validated at execution time. The registry reports configured when dependencies and local model path are present. +- No model weights are downloaded by GeoIntel. +- Detection visualization/map overlays are deferred. +- Segmentation, LiDAR, AI Copilot, Training Studio and Reports remain out of scope. + +### Next recommended pass +- Run full Sprint 8B validation and then perform a Sprint 8B verification audit before advancing to detection visualization/QA or segmentation planning. + +## Pass 24 - Sprint 8C detection visualization and QA integration (2026-06-12) + +### Completed +- Added detection result review endpoints for listing runs, listing detections by run/dataset, retrieving detection detail and returning persisted detections as GeoJSON FeatureCollections. +- Added lightweight detection filters for class name and minimum confidence. +- Added detection QA against persisted reference `vector_features` using the existing `QualityService`, `quality_checks` and `metrics` persistence path. +- Added frontend Detection Lab run selection, detection table, class/confidence filters and MapLibre detection GeoJSON overlay via the existing map component. +- Added frontend detection QA controls and metric summary display. +- Added Sprint 8C tests for GeoJSON output, list/filter behavior, detection detail, API envelope shape, QA persistence and no-match QA behavior. +- Added direct Sprint 8B tests for missing and invalid tile manifest files. +- Updated API, AI pipeline, backend/frontend README and changelog documentation. + +### Known limitations +- Detection QA requires reference datasets to have persisted `vector_features`; unsupported references return a clear error instead of fake metrics. +- Detection overlays reuse the existing single GeoJSON map layer styling; complex class-based map styling is deferred. +- Segmentation, LiDAR, AI Copilot, Training Studio and Reports remain out of scope. + +### Next recommended pass +- Run full Sprint 8C validation and perform a Sprint 8C verification audit before starting Sprint 9 Segmentation Lab. + +## Pass 25 - Sprint 9 Segmentation Lab foundation (2026-06-12) + +### Completed +- Added first-class `segmentations` ORM model and Alembic migration with project, dataset, job, analysis run, class and GiST geometry indexes. +- Added segmentation model registry capabilities for `segmentation-placeholder`, `fixture-segmenter`, `yolo-seg-configured` and `sam-configured`. +- Added `SegmentationService` boundary for raster validation, job/analysis-run lifecycle, unavailable model responses and explicit fixture-only persistence. +- Added segmentation adapter placeholder module with no SAM, YOLO-seg, torch or ultralytics imports. +- Added persisted segmentation GeoJSON output generated from PostGIS geometry and provenance properties. +- Added segmentation QA against persisted reference `vector_features` using existing `quality_checks` and `metrics`. +- Added minimal frontend Segmentation Lab panel for model states, raster selection, runs/results, GeoJSON map overlay and QA metric display. +- Updated API, AI pipeline, storage, database, backend/frontend README and changelog documentation. + +### Files changed +- `backend/app/models/entities.py` +- `backend/app/models/__init__.py` +- `backend/alembic/versions/202606120900_sprint9_segmentation_foundation.py` +- `backend/app/schemas/segmentation.py` +- `backend/app/schemas/__init__.py` +- `backend/app/services/model_registry_service.py` +- `backend/app/services/segmentation_adapter.py` +- `backend/app/services/segmentation_service.py` +- `backend/app/api/routes/segmentation.py` +- `backend/app/main.py` +- `backend/tests/test_sprint9_segmentation_foundation.py` +- `frontend/src/types.ts` +- `frontend/src/services/api/segmentation.ts` +- `frontend/src/services/api/index.ts` +- `frontend/src/App.tsx` +- `docs/API_CONTRACTS.md` +- `docs/AI_PIPELINES.md` +- `docs/STORAGE_ARCHITECTURE.md` +- `docs/DATABASE_IMPLEMENTATION_PLAN.md` +- `backend/README.md` +- `frontend/README.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Tests run +- `python -m pytest backend/tests/test_sprint9_segmentation_foundation.py -q` (red first: missing `Segmentation` import, then pass) +- `cd backend && python -m pytest` (pass, 88 tests) +- `cd frontend && npm run typecheck` (pass) + +### Known limitations +- Sprint 9 intentionally does not implement real SAM, real YOLO-seg, model downloads, new AI dependencies or production-scale async inference. +- Fixture segmenter requires explicit `fixture_mode=true` and explicit fixture segmentations; it is not production inference. +- Metric area is only persisted when provided by the fixture/output payload; Sprint 9 does not compute authoritative area from masks. + +### Next recommended pass +- Run full Sprint 9 validation and then perform Sprint 9 Verification Audit before considering future real SAM/YOLO-seg integration. + +### Validation addendum +- `python -m compileall backend/app` (pass) +- `cd backend && python -m pytest` (pass, 88 tests) +- `bash scripts/run_readiness_check.sh` (pass) +- `cd frontend && npm run typecheck` (pass) +- `cd frontend && npm run build` (pass; existing Vite chunk-size warning only) +- `cd backend && python -m alembic heads` (pass, single head `202606120900`) +- `cd backend && python -m alembic upgrade head --sql` (pass; generated SQL includes `segmentations` and GiST index) +- `bash -n scripts/live_migration_smoke.sh` (pass) +- `docker compose config` could not run because Docker is not installed in this shell. + +## Pass 26 - Sprint 10 release hardening and frontend modularization (2026-06-13) + +### Completed +- Extracted Provider Capabilities, Detection Lab and Segmentation Lab sections from `frontend/src/App.tsx` into focused frontend components. +- Preserved existing workbench state ownership, API client calls, map overlay behavior and UI copy. +- Hardened `scripts/run_readiness_check.sh` with Alembic head verification and live migration smoke script syntax validation. +- Updated frontend README and changelog documentation for Sprint 10 maintainability work. + +### Files changed +- `frontend/src/App.tsx` +- `frontend/src/components/providers/ProviderPanel.tsx` +- `frontend/src/components/detection/DetectionLab.tsx` +- `frontend/src/components/segmentation/SegmentationLab.tsx` +- `scripts/run_readiness_check.sh` +- `frontend/README.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- Sprint 10 intentionally does not add new backend capabilities, migrations, product features, AI dependencies or live provider fetching. +- `App.tsx` still owns shared workbench state orchestration; further extraction can be considered in a later maintainability pass if needed. + +### Sprint 10 addendum - additional frontend extraction +- Extracted `frontend/src/components/project/ProjectPanel.tsx` and `frontend/src/components/project/AreaPanel.tsx` from `frontend/src/App.tsx`. +- Kept project and area form state owned by `App.tsx`; extracted components receive state and callbacks only. +- `cd frontend && npm run typecheck` passed after the additional extraction. + +## Pass 27 - Sprint 11 Live Docker/PostGIS Runtime Validation (2026-06-13) + +### Completed +- Hardened `scripts/live_migration_smoke.sh` so it runs `SELECT 1`, applies `alembic upgrade head`, then checks `PostGIS_Version()`. +- Added migrated schema-object checks for core tables and geometry indexes after the live migration step. +- Added `backend/tests/test_live_migration_smoke_script.py` to lock the smoke-script ordering and schema-check contract. +- Documented the Docker/PostGIS validation command sequence, expected local `DATABASE_URL` and cleanup commands in `backend/README.md`. + +### Files changed +- `scripts/live_migration_smoke.sh` +- `backend/tests/test_live_migration_smoke_script.py` +- `backend/README.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Runtime status +- Docker is not installed or not available in this shell, so `docker compose config`, `docker compose up -d db` and the live container-backed smoke could not be completed here. +- On a Docker-enabled machine, run: + - `docker compose config` + - `docker compose up -d db` + - `DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel bash scripts/live_migration_smoke.sh` + +### Known limitations +- Sprint 11 did not add product behavior, API contracts, migrations, AI dependencies or provider fetching. +- Live runtime validation is partially blocked until Docker/PostGIS is available in the execution environment. + +## Pass 28 - Sprint 12 QA/QC golden dataset and benchmarking (2026-06-15) + +### Completed +- Added deterministic golden QA/QC fixtures for reference and predicted building polygons. +- Added `fixtures/golden/expected_qa_metrics.json` with the expected partial-match baseline. +- Added `scripts/run_golden_qa_benchmark.py` to run existing `QaService` logic and verify `QualityService` persistence output. +- Added backend tests for expected golden metrics, benchmark command output and persisted metric keys. +- Updated QA/QC specification, backend README and changelog documentation. + +### Files changed +- `fixtures/golden/reference_buildings.geojson` +- `fixtures/golden/predicted_buildings.geojson` +- `fixtures/golden/expected_qa_metrics.json` +- `scripts/run_golden_qa_benchmark.py` +- `backend/tests/test_sprint12_golden_qa_benchmark.py` +- `docs/QA_QC_SPECIFICATION.md` +- `backend/README.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Expected benchmark metrics +- precision: `0.5` +- recall: `0.5` +- F1: `0.5` +- mean IoU: `0.8339768339761133` +- false positives: `1` +- false negatives: `1` + +### Known limitations +- The benchmark uses explicit fixture/demo data and an in-memory persistence session; it does not replace the pending Docker/PostGIS live smoke. +- Sprint 12 does not add product features, API contracts, migrations, live providers, AI model execution or new dependencies. + +## Pass 29 - Sprint 13 Real YOLO operational hardening (2026-06-15) + +### Completed +- Added `YoloPreflightService` for local configured-YOLO readiness checks without model loading, inference or downloads. +- Added `scripts/yolo_preflight.py` for CLI checks of enabled state, dependencies, local model file, tile manifest validity, tile limit and tile paths. +- Added Sprint 13 backend tests for disabled, dependency-unavailable and ready preflight states plus CLI JSON output. +- Updated AI pipeline, backend README and changelog documentation. + +### Files changed +- `backend/app/services/yolo_preflight_service.py` +- `scripts/yolo_preflight.py` +- `backend/tests/test_sprint13_yolo_preflight.py` +- `docs/AI_PIPELINES.md` +- `backend/README.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- Preflight does not prove model compatibility or inference correctness; it intentionally avoids loading YOLO models. +- Optional AI dependencies are still not installed by default. +- Docker/PostGIS live validation remains pending until Docker is available. + +## Pass 30 - Release hardening audit pass (2026-06-15) + +### Completed +- Audited release-readiness signals after Sprint 13, including timestamp warnings, frontend bundle output, migration SQL rendering and readiness coverage. +- Replaced backend `datetime.utcnow()` calls with timezone-aware UTC timestamps in service paths. +- Verified the affected backend tests with `DeprecationWarning` promoted to errors. +- Split frontend production output into app, React vendor and MapLibre vendor chunks, with an explicit chunk warning threshold for the known MapLibre GIS runtime. +- Updated backend/frontend README, TODO and changelog documentation. + +### Files changed +- `backend/app/services/dataset_service.py` +- `backend/app/services/geojson_service.py` +- `backend/app/services/job_service.py` +- `backend/app/services/qa_service.py` +- `backend/app/services/quality_service.py` +- `frontend/vite.config.ts` +- `backend/README.md` +- `frontend/README.md` +- `docs/TODO.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- This pass does not add product features, migrations, API contracts, AI dependencies, provider fetching or model execution. +- Docker/PostGIS live validation still requires a Docker-enabled machine. +- Larger frontend architectural decomposition remains a separate low-risk planning item; this pass only hardened build output. + +## Pass 31 - Extended release hardening sweep (2026-06-15) + +### Completed +- Promoted the backend readiness gate to run `pytest` with `-W error::DeprecationWarning`. +- Added backend tests that verify readiness and pass-end scripts keep the stricter release checks in place. +- Hardened `scripts/codex_pass_end_check.sh` so placeholder scans skip `node_modules`, `dist` and `__pycache__` folders. +- Updated `docs/TODO.md` with a current implementation status layer while preserving older planning context. +- Re-ran pass-end checks and strict backend warning checks. + +### Files changed +- `scripts/run_readiness_check.sh` +- `scripts/codex_pass_end_check.sh` +- `backend/tests/test_readiness_gate.py` +- `backend/README.md` +- `docs/TODO.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- This pass still does not add API contracts, migrations, product features, provider fetching, AI dependencies or model execution. +- Docker/PostGIS live validation remains blocked in this local environment because Docker is unavailable. + +## Pass 32 - Readiness contract gate hardening (2026-06-15) + +### Completed +- Added API contract smoke validation to `scripts/run_readiness_check.sh`. +- Added a regression test that requires the readiness gate to keep running `scripts/smoke_contracts.py`. +- Re-ran the full readiness gate after the change. + +### Files changed +- `scripts/run_readiness_check.sh` +- `backend/tests/test_readiness_gate.py` +- `backend/README.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- Docker/PostGIS live validation remains blocked in this local environment because Docker is unavailable. + +## Pass 33 - Docker runtime build hardening (2026-06-15) + +### Completed +- Investigated Unraid/Tower Docker build failure from pasted server output. +- Fixed backend Docker build ordering so `README.md` and `app/` exist before `pip install .`. +- Removed mandatory root `.env` references from Compose; default local runtime now uses checked-in environment values. +- Added PostGIS healthcheck and backend `depends_on` health condition. +- Made the backend container run `python -m alembic upgrade head` before starting Uvicorn. +- Added backend and frontend `.dockerignore` files to reduce Docker build context and exclude dependency/build/cache outputs. +- Added regression tests for Dockerfile package-source ordering, Compose env behavior, DB health/migration startup and Docker ignore coverage. + +### Files changed +- `backend/Dockerfile` +- `docker-compose.yml` +- `backend/.dockerignore` +- `frontend/.dockerignore` +- `backend/tests/test_docker_runtime_config.py` +- `README.md` +- `backend/README.md` +- `docs/TODO.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- Docker commands still cannot be executed in this local Codex environment because Docker is unavailable here. +- The server should rerun `docker compose build --no-cache && docker compose up -d` to verify the real Docker runtime. + +## Pass 34 - Docker browser port 1202 (2026-06-15) + +### Completed +- Changed Docker Compose frontend host publishing from `5173:5173` to `1202:5173`. +- Added backend Docker CORS defaults for `http://localhost:1202` and `http://127.0.0.1:1202`. +- Updated `.env.example` and local/Docker documentation to point browser users to `http://localhost:1202` for Docker Compose. +- Added a regression assertion to Docker runtime config tests. + +### Files changed +- `docker-compose.yml` +- `.env.example` +- `backend/tests/test_docker_runtime_config.py` +- `README.md` +- `backend/README.md` +- `frontend/README.md` +- `docs/LOCAL_DEVELOPMENT_RUNBOOK.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- Docker commands still cannot be executed in this local Codex environment because Docker is unavailable here. + +## Pass 35 - Docker backend database startup retry (2026-06-16) + +### Completed +- Investigated Tower runtime logs showing backend Alembic startup failed with `connection refused` even after the db container reported healthy. +- Added `backend/docker_start.sh` to retry a real SQLAlchemy `SELECT 1` connection before migrations. +- Updated Compose to run `sh /app/docker_start.sh` for backend startup. +- Added regression tests that require the Docker start script and SQL readiness retry before migrations. + +### Files changed +- `backend/docker_start.sh` +- `docker-compose.yml` +- `backend/tests/test_docker_runtime_config.py` +- `backend/README.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- Docker commands still cannot be executed in this local Codex environment because Docker is unavailable here. + +## Pass 36 - Alembic Docker log formatting fix (2026-06-16) + +### Completed +- Investigated backend Docker logs showing repeated literal `%(levelname)-5.5s [%(name)s] %(message)s` lines during migrations. +- Fixed `backend/alembic.ini` logging formatter from escaped `%%(...)` tokens to runtime interpolation `%(...)` tokens. +- Added a regression test for Alembic logging formatter correctness. +- Verified Alembic SQL rendering no longer emits literal formatter spam. + +### Files changed +- `backend/alembic.ini` +- `backend/tests/test_alembic_logging_config.py` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +## Pass 37 - Frontend same-origin API proxy for Docker LAN access (2026-06-16) + +### Completed +- Audited the running app at `http://192.168.10.150:1202` from Codex: frontend HTML and backend `/health` were reachable, but `/api/v1/projects` on the frontend origin returned the frontend HTML fallback. +- Changed the frontend API client default from `http://localhost:8000` to same-origin requests. +- Added Vite proxy routes for `/api` and `/health`. +- Added Docker Compose `VITE_API_PROXY_TARGET=http://backend:8000` so LAN browsers use `http://192.168.10.150:1202` only and the frontend container proxies API calls internally. +- Added regression tests for same-origin API/proxy behavior. + +### Files changed +- `frontend/src/services/api/client.ts` +- `frontend/vite.config.ts` +- `docker-compose.yml` +- `backend/tests/test_docker_runtime_config.py` +- `README.md` +- `frontend/README.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- The running Tower deployment needs a rebuild/restart before this fix is active. + +## Pass 38 - Browser runtime proxy guard (2026-06-16) + +### Completed +- Added `scripts/verify_browser_runtime.sh` to verify the browser-facing frontend URL, `/api/v1/projects` proxy behavior and optional backend health endpoint. +- Added readiness syntax validation for the browser runtime verification script. +- Updated environment and local development documentation to prefer same-origin frontend API calls with Vite proxying in Docker/LAN deployments. + +### Files changed +- `scripts/verify_browser_runtime.sh` +- `scripts/run_readiness_check.sh` +- `backend/tests/test_docker_runtime_config.py` +- `README.md` +- `docs/ENVIRONMENT_SPEC.md` +- `docs/LOCAL_DEVELOPMENT_RUNBOOK.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- The currently running Tower deployment at `http://192.168.10.150:1202` still returns frontend HTML for `/api/v1/projects` until the frontend image is rebuilt and restarted. + +## Pass 39 - Environment contract cleanup (2026-06-16) + +### Completed +- Corrected `.env.example` and `docs/ENVIRONMENT_SPEC.md` to use the YOLO environment variable names read by backend settings: `YOLO_ENABLED`, `YOLO_MODEL_PATH`, `YOLO_MODEL_VERSION` and `YOLO_MAX_TILES`. +- Updated frontend example settings to keep `VITE_API_BASE_URL` empty by default and expose `VITE_API_PROXY_TARGET` for Vite proxy routing. +- Added regression coverage for example environment names and browser runtime proxy verification. + +### Files changed +- `.env.example` +- `docs/ENVIRONMENT_SPEC.md` +- `backend/tests/test_docker_runtime_config.py` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- Docker cannot be executed in this local Codex environment; Tower still needs an image rebuild/restart to activate the frontend proxy fix. + +## Pass 40 - Frontend Docker nginx reverse proxy (2026-06-16) + +### Completed +- Replaced the Docker frontend runtime with a production Vite build served by nginx. +- Added `frontend/nginx.conf` with explicit reverse proxy rules for `/api/` and `/health` to `backend:8000`. +- Changed Docker Compose frontend publishing from `1202:5173` to `1202:80`. +- Updated Docker runtime regression tests and documentation so the browser-facing API path is no longer dependent on Vite dev-server proxy behavior. + +### Files changed +- `frontend/Dockerfile` +- `frontend/nginx.conf` +- `docker-compose.yml` +- `backend/tests/test_docker_runtime_config.py` +- `README.md` +- `frontend/README.md` +- `docs/ENVIRONMENT_SPEC.md` +- `docs/LOCAL_DEVELOPMENT_RUNBOOK.md` +- `CHANGELOG.md` +- `docs/CODEX_EXECUTION_LOG.md` + +### Known limitations +- Docker still cannot be executed in this local Codex environment. Tower must rebuild the frontend image to activate the nginx runtime. + +## Sprint 14 - Docker GIS runtime enablement (2026-06-16) + +- Added backend `gis` optional dependency group for the approved Rasterio/GeoPandas runtime stack. +- Updated backend Docker image to install `.[gis]` and GDAL/GEOS/PROJ system packages. +- Added `scripts/verify_gis_runtime.sh` to verify PostGIS, Rasterio and GeoPandas capabilities through the browser-facing frontend proxy. +- Added readiness syntax coverage for the GIS runtime verification script. +- Added regression tests for Docker GIS dependency installation and capability verification coverage. +- Updated backend, environment, root README and changelog documentation with local/LAN verification commands. +- No API contracts, migrations, AI dependencies, provider fetching or product features were changed. +- Added `scripts/gis_import_smoke.py` and wired it into the backend Docker build so broken Rasterio/GeoPandas/pyogrio imports fail during image build. +- Added Docker Compose healthchecks for backend and frontend; frontend now waits for backend service health before starting. +- Corrected the GIS import smoke placement so the backend Docker build can access it inside the `./backend` build context; the root script now wraps the backend script. + +## Sprint 15 - Explicit demo workflow seed (2026-06-16) + +- Added `POST /api/v1/demo/workflow` for an explicit offline demo workflow seed. +- Added `DemoWorkflowService` to create or return a demo project, AOI, fixture reference dataset, fixture candidate dataset and persisted QA/QC metrics. +- Added `scripts/seed_demo_workflow.py` for terminal-based demo seeding. +- Added frontend `Load demo workflow` action in the Projects panel. +- Added endpoint/fixture-contract tests and readiness compile coverage for the demo seed script. +- No live GRB/OSM fetching, AI inference, migrations or new dependencies were introduced. + +## Sprint 16 - QA/QC result visibility (2026-06-16) + +- Added `GET /api/v1/projects/{project_id}/quality-checks` for read-only project QA/QC result listing. +- Added `QualityCheckService` to return persisted `quality_checks` with metric rows. +- Added frontend QA/QC Results panel and API client support. +- Demo workflow loading and QA actions now refresh persisted QA/QC results in the UI. +- Added backend tests for quality check listing and canonical envelopes. +- No migrations, live provider fetching, AI inference or new dependencies were introduced. + +## Sprint 17 export foundation (2026-06-16) + +Changed: +- Hardened GeoJSON exports so vector dataset, detection run and segmentation run exports persist `exports` rows and write JSON artifacts. +- Added project metadata JSON export plus export list/read/content endpoints. +- Added a frontend Export Center panel for creating exports, listing export records and previewing JSON content. +- Added backend tests for export persistence, artifact writing, raster rejection and canonical envelope behavior. + +Tested: +- `python -m compileall backend/app` +- `cd backend && python -m pytest -W error::DeprecationWarning` +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` +- `bash scripts/run_readiness_check.sh` +- `cd backend && python -m alembic heads && python -m alembic upgrade head --sql` +- `bash -n scripts/live_migration_smoke.sh && bash -n scripts/verify_browser_runtime.sh && bash -n scripts/verify_gis_runtime.sh` + +Open: +- Docker/live browser validation must be rerun on the deployment host after rebuild. +- YOLO-format export and report export remain documented future work; this pass only implements JSON/GeoJSON export foundation. + +Limitations: +- Export artifacts are returned through API JSON content preview; browser file-download UX is not implemented yet. +- Detection and segmentation exports require existing persisted runs; no inference or fake output generation is introduced. + +Next recommended pass: +- Rebuild/redeploy the Docker stack and verify `/api/v1/exports/*` through the LAN frontend proxy, then consider a lightweight file-download endpoint or report artifact pass. + +## Sprint 17 export download hardening (2026-06-16) + +Changed: +- Added `GET /api/v1/exports/{export_id}/download` as a raw file response for stored JSON/GeoJSON export artifacts. +- Reused the same export artifact existence validation for content preview and downloads. +- Added frontend Export Center download buttons using the configured/same-origin API base URL. +- Added backend tests for missing artifacts and file download response headers/content. + +Tested: +- `python -m compileall backend/app` +- `cd backend && python -m pytest -W error::DeprecationWarning` +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` +- `bash scripts/run_readiness_check.sh` +- `cd backend && python -m alembic heads && python -m alembic upgrade head --sql` +- `bash -n scripts/live_migration_smoke.sh && bash -n scripts/verify_browser_runtime.sh && bash -n scripts/verify_gis_runtime.sh` + +Open: +- Docker/LAN validation should be rerun after redeploy on the Tower host. + +Limitations: +- Download endpoint is intentionally a raw file response, not a canonical JSON envelope, because it is a browser/file artifact path. + +Next recommended pass: +- Rebuild Docker and verify Export Center create/preview/download against `http://192.168.10.150:1202`. + +## Sprint 17 lightweight report artifact export (2026-06-16) + +Changed: +- Added `POST /api/v1/exports/report` to create a lightweight HTML project report artifact from persisted project, dataset and QA/QC summary state. +- Added HTML escaping for report-rendered project and dataset values. +- Updated export downloads to return `text/html` for HTML report artifacts and `application/json` for JSON/GeoJSON artifacts. +- Added a frontend Export Center action for project report HTML export. +- Added backend tests for HTML report artifact creation and HTML download response behavior. + +Tested: +- `python -m compileall backend/app` +- `cd backend && python -m pytest -W error::DeprecationWarning` +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` +- `bash scripts/run_readiness_check.sh` +- `cd backend && python -m alembic heads && python -m alembic upgrade head --sql` +- `bash -n scripts/live_migration_smoke.sh && bash -n scripts/verify_browser_runtime.sh && bash -n scripts/verify_gis_runtime.sh` + +Open: +- Docker/LAN verification should be rerun after deployment rebuild. + +Limitations: +- Report export is intentionally a simple HTML artifact, not a PDF designer or standalone Reports module. +- Report content is summary-only and uses existing persisted project, dataset and QA/QC rows. + +Next recommended pass: +- Rebuild Docker and verify Export Center metadata, GeoJSON, report HTML and download flows through the LAN URL. + +## Sprint 17 export audit trail and LAN demo/export smoke (2026-06-16) + +Changed: +- Added export history to project metadata JSON and lightweight HTML report artifacts. +- Added `scripts/verify_demo_export_workflow.sh` to verify the browser-facing demo workflow, persisted QA/QC listing, metadata export, report export, vector GeoJSON export, export listing and artifact downloads. +- Included the demo/export workflow script in the main readiness syntax gate. +- Added backend tests to lock export-history content and demo/export script coverage. + +Tested: +- `python -m compileall backend/app` +- `cd backend && python -m pytest -W error::DeprecationWarning` +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` +- `bash scripts/run_readiness_check.sh` +- `cd backend && python -m alembic heads && python -m alembic upgrade head --sql` +- `bash -n scripts/live_migration_smoke.sh && bash -n scripts/verify_browser_runtime.sh && bash -n scripts/verify_gis_runtime.sh && bash -n scripts/verify_demo_export_workflow.sh` + +Open: +- `scripts/verify_demo_export_workflow.sh` still needs to be run against the rebuilt Tower deployment URL. + +Limitations: +- The smoke script intentionally uses explicit fixture demo data and does not fetch live GRB/OSM or run AI inference. + +Next recommended pass: +- Rebuild Docker on the Tower host and run `bash scripts/verify_demo_export_workflow.sh http://192.168.10.150:1202`. diff --git a/docs/CODEX_EXECUTION_PLAN.md b/docs/CODEX_EXECUTION_PLAN.md new file mode 100644 index 00000000..311ed3ff --- /dev/null +++ b/docs/CODEX_EXECUTION_PLAN.md @@ -0,0 +1,241 @@ +# Codex Execution Plan — GeoIntel M1 Build + +## Core instruction +Build backend-first, then connect frontend. Do not start with visual polish. The project must become a working GeoAI Workbench foundation. + +## Pass 0 — Repo verification +Before coding: +- read README.md +- read docs/SPECIFICATION_FREEZE_M0.md +- read docs/V1_SCOPE_FREEZE.md +- read docs/SERVICE_ARCHITECTURE.md +- read docs/REPOSITORY_CONVENTIONS.md +- inspect current repo structure +- create/update docs/CODEX_EXECUTION_LOG.md + +Output: +- execution log started +- no product scope changes + +## Pass 1 — Backend foundation +Implement: +- FastAPI app structure +- config management +- database session setup +- SQLAlchemy base +- health endpoint +- error handling +- repository/service folders +- pytest setup + +Acceptance: +- backend starts +- health endpoint returns OK +- tests run + +Do not implement GIS processing yet. + +## Pass 2 — Database schema foundation +Implement models and migrations if migration tooling is present: +- Project +- Area +- Dataset +- Layer +- AnalysisRun +- Detection +- Segmentation +- Metric +- QualityCheck +- Export +- JobLog or AnalysisLog + +Acceptance: +- tables can be created +- model relationships work +- basic repository tests pass + +## Pass 3 — Project and area API +Implement: +- project CRUD +- area CRUD +- GeoJSON polygon input +- area/perimeter calculation using metric CRS +- validation for invalid geometry + +Acceptance: +- create project +- create area +- retrieve project with areas +- tests for simple polygon area + +## Pass 4 — Dataset Manager API +Implement: +- upload endpoint +- storage service +- dataset records +- file checksum +- type detection +- metadata endpoint + +Acceptance: +- upload vector fixture +- upload raster fixture if available +- dataset metadata stored +- unsupported file returns clear error + +## Pass 5 — Vector foundation +Implement: +- GeoJSON import +- zipped shapefile import if feasible +- CRS detection +- geometry validation +- clipping by area +- summary metrics + +Acceptance: +- vector fixture imported +- vector clipped by test area +- area/length stats correct + +## Pass 6 — Raster foundation +Implement: +- raster metadata extraction +- raster bounds/CRS/resolution/bands +- raster clip by area when CRS exists +- simple histogram/statistics +- tile job records or actual tile generation + +Acceptance: +- raster fixture metadata read +- missing CRS warning works +- clip operation returns artifact path or clear unsupported message + +## Pass 7 — Reference layer provider foundation +Implement: +- provider interface +- local fixture provider +- configurable GRB WFS provider shell +- caching reference layer as Dataset/Layer + +Acceptance: +- fixture building reference layer loads +- provider output normalized +- cached layer can be used by QA service + +## Pass 8 — Detection pipeline foundation +Implement: +- DetectionService +- DevelopmentDetectionProvider with deterministic fixture output +- YoloDetectionProvider interface/shell if dependency not ready +- detection analysis run lifecycle +- detection geometry persistence +- GeoJSON export + +Acceptance: +- run detection on fixture/project +- detections are persisted +- detections appear as layer output +- export returns GeoJSON + +Important: +- Use development provider only as a temporary provider, not as fake final logic. +- Keep provider interface ready for YOLO. + +## Pass 9 — QA/QC engine +Implement: +- IoU calculation +- matching algorithm +- precision/recall/F1 +- false positive/false negative layers +- quality check persistence + +Acceptance: +- unit tests for IoU +- unit tests for matching +- QA run compares detection fixture to reference fixture +- metrics persisted and returned + +## Pass 10 — Segmentation foundation +Implement: +- segmentation analysis run lifecycle +- development segmentation provider +- mask artifact record +- polygon output if simple fixture allows + +Acceptance: +- segmentation run can be created +- segmentation result can be listed/exported + +## Pass 11 — Frontend foundation +Implement: +- React app shell +- routing +- API client +- project list/create page +- project workspace page +- status/error/empty components + +Acceptance: +- frontend starts +- user can create/open project through API + +## Pass 12 — Dataset and Map UI +Implement: +- Dataset Manager page +- Map Workbench page +- layer manager +- display vector outputs +- show dataset metadata + +Acceptance: +- user can upload/import dataset +- user can see layer on map or at least its extent/feature list if map library setup is incomplete + +## Pass 13 — Detection and QA UI +Implement: +- Detection Lab page +- run detection with provider selector +- show detections +- QA/QC Lab page +- run QA against reference layer +- show metrics and findings + +Acceptance: +- Demo 1 can be executed end-to-end from UI using fixtures/development provider + +## Pass 14 — Export UI and workflow polish +Implement: +- Exports page +- export history +- GeoJSON download +- run summary view +- useful empty states + +Acceptance: +- user can download detection/QA output + +## Pass 15 — Documentation and handoff +Update: +- docs/TODO.md +- docs/CODEX_EXECUTION_LOG.md +- README quickstart +- known limitations + +Run: +- backend tests +- frontend typecheck/build if available + +Final response must include: +- what was built +- tests run +- what remains open +- exact next pass recommendation + +## Absolute restrictions +- Do not remove documentation. +- Do not implement unrelated features. +- Do not add authentication. +- Do not add payment/sharing/multi-user. +- Do not hide broken endpoints behind UI-only mock data. +- Do not calculate metric geometry on EPSG:4326. +- Do not export misleading geospatial outputs from non-georeferenced rasters. diff --git a/docs/CODEX_MASTER_PROMPT.md b/docs/CODEX_MASTER_PROMPT.md new file mode 100644 index 00000000..9969bce0 --- /dev/null +++ b/docs/CODEX_MASTER_PROMPT.md @@ -0,0 +1,54 @@ +# Codex Master Prompt — GeoIntel Kempen + +You are working inside the GeoIntel Kempen repository. + +GeoIntel Kempen is a GeoAI Workbench for the Belgian Kempen. It is not a generic GIS viewer and not a policy dashboard. The core value is turning raster data, vector data and AI outputs into geospatially correct layers, metrics, QA findings and exports. + +## Mandatory reading before coding +Read these documents first: +- README.md +- docs/SPECIFICATION_FREEZE_M0.md +- docs/V1_SCOPE_FREEZE.md +- docs/DATASET_STRATEGY.md +- docs/SERVICE_ARCHITECTURE.md +- docs/REPOSITORY_CONVENTIONS.md +- docs/CODEX_EXECUTION_PLAN.md +- docs/ACCEPTANCE_CRITERIA.md +- docs/FIXTURE_STRATEGY.md + +## Main build target +Build toward Demo 1: +Building detection on imagery, conversion to geospatial detection outputs, QA/QC against GRB or a local reference building fixture, and GeoJSON export. + +## Development rules +- Backend-first. +- Service boundaries must be respected. +- No placeholder-only pages. +- No fake success responses without persisted data. +- No unrelated features. +- No auth/multi-user/payment. +- No heavy LiDAR/MLOps/training studio in V1. +- Never calculate metric area/length directly in EPSG:4326. +- Always preserve CRS and source metadata. +- Mock/development providers are allowed only if isolated and clearly named. + +## Implementation order +Follow docs/CODEX_EXECUTION_PLAN.md pass by pass. Do not skip ahead to UI polish. + +## Handoff requirements after each pass +Update docs/CODEX_EXECUTION_LOG.md with: +- completed items; +- files changed; +- tests run; +- known limitations; +- next recommended pass. + +Also update docs/TODO.md where relevant. + +## Final response format after a build pass +Return: +1. Summary of completed work. +2. Tests/build checks run. +3. Known limitations. +4. Next recommended pass. +5. Whether the build meets the acceptance criteria for the current milestone. diff --git a/docs/CODEX_PASS_0_REPO_AUDIT.md b/docs/CODEX_PASS_0_REPO_AUDIT.md new file mode 100644 index 00000000..3c6e1c07 --- /dev/null +++ b/docs/CODEX_PASS_0_REPO_AUDIT.md @@ -0,0 +1,14 @@ +# Codex Pass — Repo Audit and Baseline + +## Doel +Controleer mappenstructuur, docs, env, docker-compose en maak geen grote codewijzigingen. + +## Regels +- Volg `docs/BUILD_GOVERNANCE.md`. +- Geen mockdata als echte functionaliteit. +- Werk changelog, TODO en execution log bij. + +## Acceptatiecriteria +- Passdoel is testbaar. +- Blocking issues zijn expliciet gedocumenteerd. +- Geen regressies in eerdere passcontracten. diff --git a/docs/CODEX_PASS_1_BACKEND_FOUNDATION.md b/docs/CODEX_PASS_1_BACKEND_FOUNDATION.md new file mode 100644 index 00000000..204e67c5 --- /dev/null +++ b/docs/CODEX_PASS_1_BACKEND_FOUNDATION.md @@ -0,0 +1,14 @@ +# Codex Pass — Backend Foundation + +## Doel +Maak FastAPI app, config loader, /health endpoint, error model, database skeleton en health tests. + +## Regels +- Volg `docs/BUILD_GOVERNANCE.md`. +- Geen mockdata als echte functionaliteit. +- Werk changelog, TODO en execution log bij. + +## Acceptatiecriteria +- Passdoel is testbaar. +- Blocking issues zijn expliciet gedocumenteerd. +- Geen regressies in eerdere passcontracten. diff --git a/docs/CODEX_PASS_2_DATABASE_AND_MODELS.md b/docs/CODEX_PASS_2_DATABASE_AND_MODELS.md new file mode 100644 index 00000000..1d539f20 --- /dev/null +++ b/docs/CODEX_PASS_2_DATABASE_AND_MODELS.md @@ -0,0 +1,14 @@ +# Codex Pass — Database and Models + +## Doel +Implementeer SQLAlchemy, Alembic, PostGIS extension en eerste tabellen/projects/areas/datasets. + +## Regels +- Volg `docs/BUILD_GOVERNANCE.md`. +- Geen mockdata als echte functionaliteit. +- Werk changelog, TODO en execution log bij. + +## Acceptatiecriteria +- Passdoel is testbaar. +- Blocking issues zijn expliciet gedocumenteerd. +- Geen regressies in eerdere passcontracten. diff --git a/docs/CODEX_PASS_3_DATASET_MANAGER.md b/docs/CODEX_PASS_3_DATASET_MANAGER.md new file mode 100644 index 00000000..c05527b8 --- /dev/null +++ b/docs/CODEX_PASS_3_DATASET_MANAGER.md @@ -0,0 +1,14 @@ +# Codex Pass — Dataset Manager + +## Doel +Maak upload endpoint, veilige storage policy, dataset metadata en status lifecycle. + +## Regels +- Volg `docs/BUILD_GOVERNANCE.md`. +- Geen mockdata als echte functionaliteit. +- Werk changelog, TODO en execution log bij. + +## Acceptatiecriteria +- Passdoel is testbaar. +- Blocking issues zijn expliciet gedocumenteerd. +- Geen regressies in eerdere passcontracten. diff --git a/docs/CODEX_PASS_4_RASTER_VECTOR_CORE.md b/docs/CODEX_PASS_4_RASTER_VECTOR_CORE.md new file mode 100644 index 00000000..740d28fc --- /dev/null +++ b/docs/CODEX_PASS_4_RASTER_VECTOR_CORE.md @@ -0,0 +1,14 @@ +# Codex Pass — Raster and Vector Core + +## Doel +Maak Rasterio metadata extractor, GeoPandas vector loader, CRS/bounds/geometry validation en clip skeleton. + +## Regels +- Volg `docs/BUILD_GOVERNANCE.md`. +- Geen mockdata als echte functionaliteit. +- Werk changelog, TODO en execution log bij. + +## Acceptatiecriteria +- Passdoel is testbaar. +- Blocking issues zijn expliciet gedocumenteerd. +- Geen regressies in eerdere passcontracten. diff --git a/docs/CODEX_PASS_5_FRONTEND_WORKBENCH_SHELL.md b/docs/CODEX_PASS_5_FRONTEND_WORKBENCH_SHELL.md new file mode 100644 index 00000000..deeaa36c --- /dev/null +++ b/docs/CODEX_PASS_5_FRONTEND_WORKBENCH_SHELL.md @@ -0,0 +1,14 @@ +# Codex Pass — Frontend Workbench Shell + +## Doel +Maak React routes, layout, API client, health panel, project list/create en dataset list skeleton. + +## Regels +- Volg `docs/BUILD_GOVERNANCE.md`. +- Geen mockdata als echte functionaliteit. +- Werk changelog, TODO en execution log bij. + +## Acceptatiecriteria +- Passdoel is testbaar. +- Blocking issues zijn expliciet gedocumenteerd. +- Geen regressies in eerdere passcontracten. diff --git a/docs/CODEX_PASS_6_DETECTION_QA_SKELETON.md b/docs/CODEX_PASS_6_DETECTION_QA_SKELETON.md new file mode 100644 index 00000000..9af6ee7f --- /dev/null +++ b/docs/CODEX_PASS_6_DETECTION_QA_SKELETON.md @@ -0,0 +1,14 @@ +# Codex Pass — Detection and QA Skeleton + +## Doel +Maak model registry records, detection analysis_run endpoint, testadapter, detection schema en QA metrics op fixtures. + +## Regels +- Volg `docs/BUILD_GOVERNANCE.md`. +- Geen mockdata als echte functionaliteit. +- Werk changelog, TODO en execution log bij. + +## Acceptatiecriteria +- Passdoel is testbaar. +- Blocking issues zijn expliciet gedocumenteerd. +- Geen regressies in eerdere passcontracten. diff --git a/docs/CODEX_PASS_MATRIX_M3.md b/docs/CODEX_PASS_MATRIX_M3.md new file mode 100644 index 00000000..96ba8fd6 --- /dev/null +++ b/docs/CODEX_PASS_MATRIX_M3.md @@ -0,0 +1,38 @@ +# Codex Pass Matrix M3 + +This matrix is the recommended order for the first real build. + +| Pass | Name | Primary output | Must be green before next pass | +|---:|---|---|---| +| 01 | Runtime Foundation | Backend, frontend, DB boot | health checks | +| 02 | Core Database | migrations and models | migration tests | +| 03 | Project/Area APIs | project and area CRUD | API tests | +| 04 | Frontend Shell | routes and layout | app boots, nav works | +| 05 | Dataset Manager | upload + metadata records | upload smoke test | +| 06 | Raster/Vector Metadata | Rasterio/GeoPandas metadata | fixture tests | +| 07 | Map Workbench | draw/select area | area creation from UI | +| 08 | GRB Fetch Skeleton | reference layer service | graceful failure or fixture success | +| 09 | Detection Interface | analysis run + detections | fixture detection output | +| 10 | QA/QC Engine | metrics and error classes | deterministic QA test | +| 11 | Exports | GeoJSON/summary exports | file created and downloadable | +| 12 | Stabilization | usability and docs | all smoke tests green | + +## Stop conditions + +Codex must stop and report instead of continuing when: + +- database cannot migrate +- backend cannot import +- frontend cannot compile +- an API contract cannot be satisfied without changing frozen scope +- a required dependency cannot be installed + +## Reporting after each pass + +Codex must produce: + +- files changed +- tests run +- pass/fail status +- open issues +- next recommended pass diff --git a/docs/CODEX_PHASE_1_PROMPT.md b/docs/CODEX_PHASE_1_PROMPT.md new file mode 100644 index 00000000..4b6f229e --- /dev/null +++ b/docs/CODEX_PHASE_1_PROMPT.md @@ -0,0 +1,47 @@ +# Codex Phase 1 Prompt — Backend Foundation + +You are working in the GeoIntel Kempen repository. + +Read these files first: + +1. `docs/PRODUCT_BLUEPRINT.md` +2. `docs/DEVELOPMENT_RULES.md` +3. `docs/DATABASE_SCHEMA.md` +4. `docs/API_SPECIFICATION.md` +5. `docs/STORAGE_ARCHITECTURE.md` +6. `docs/IMPLEMENTATION_BACKLOG.md` + +## Mission + +Build the backend foundation for GeoIntel Kempen without implementing fake features. + +## Required output + +- FastAPI backend skeleton. +- Config/settings module. +- Database connection to PostgreSQL/PostGIS. +- Initial models for projects, areas, datasets, layers, analysis runs, metrics, exports and jobs. +- Alembic migrations if applicable. +- Health endpoint. +- Basic project/area/dataset CRUD endpoints where safe. +- File storage service skeleton with real path handling. +- Tests for config, health and core models where feasible. + +## Non-negotiable rules + +- Do not fake GIS processing. +- Do not create endpoints that pretend analysis was completed. +- Every geometry field must be designed for PostGIS. +- Every dataset must support metadata. +- Keep code modular. +- Update TODO/backlog after changes. + +## Delivery format + +At the end, report: + +- Files changed. +- What works. +- How to run. +- Tests run. +- Remaining tasks. diff --git a/docs/CODEX_PHASE_2_PROMPT.md b/docs/CODEX_PHASE_2_PROMPT.md new file mode 100644 index 00000000..e8b14d0c --- /dev/null +++ b/docs/CODEX_PHASE_2_PROMPT.md @@ -0,0 +1,32 @@ +# Codex Phase 2 Prompt — Dataset Manager + Raster/Vector Metadata + +Read first: + +- `docs/DATA_CATALOG.md` +- `docs/RASTER_OPERATIONS_SPEC.md` +- `docs/VECTOR_OPERATIONS_SPEC.md` +- `docs/API_SPECIFICATION.md` +- `docs/STORAGE_ARCHITECTURE.md` + +## Mission + +Implement dataset upload and metadata extraction for raster and vector data. + +## Required features + +- Upload endpoint for datasets. +- Save original file unchanged. +- Compute checksum. +- Detect raster vs vector. +- Extract raster metadata with Rasterio. +- Extract vector metadata with GeoPandas/Fiona. +- Store metadata in dataset record. +- Generate basic preview for raster if feasible. +- Add tests with small fixtures. + +## Rules + +- Do not require external production datasets for tests. +- Use small test fixtures. +- If GDAL/Rasterio dependency issues exist, document clearly and keep code structured. +- Do not fake metadata. diff --git a/docs/CODEX_PHASE_3_PROMPT.md b/docs/CODEX_PHASE_3_PROMPT.md new file mode 100644 index 00000000..55a0e301 --- /dev/null +++ b/docs/CODEX_PHASE_3_PROMPT.md @@ -0,0 +1,30 @@ +# Codex Phase 3 Prompt — Detection + QA/QC Skeleton + +Read first: + +- `docs/DETECTION_PIPELINE_SPEC.md` +- `docs/QA_QC_SPECIFICATION.md` +- `docs/AI_PIPELINES.md` +- `docs/DATABASE_SCHEMA.md` + +## Mission + +Build the detection pipeline contracts and QA/QC engine foundation. + +## Required features + +- Detection analysis run model/API. +- Detection storage model/API. +- Model registry skeleton. +- Tile metadata structures. +- Pixel bbox to geospatial polygon utility. +- IoU calculation utility. +- One-to-one polygon matching utility. +- Precision/recall/F1 calculation. +- GeoJSON export for detections and QA outputs. + +## Rules + +- Do not fake YOLO inference if no model is available. +- It is acceptable to implement the model adapter interface and readiness checks first. +- QA/QC utilities must be testable with fixture polygons. diff --git a/docs/CODEX_PHASE_4_PROMPT.md b/docs/CODEX_PHASE_4_PROMPT.md new file mode 100644 index 00000000..7adcdbd9 --- /dev/null +++ b/docs/CODEX_PHASE_4_PROMPT.md @@ -0,0 +1,26 @@ +# Codex Phase 4 Prompt — Raster and Vector Core + +Read all M0/M1 docs first. Implement the first real geospatial processing layer. + +## Goals + +- Raster metadata extraction service. +- Vector metadata extraction service. +- Vector GeoJSON import into PostGIS. +- Area clipping for vector fixtures. +- Building stats analyzer using vector polygons. + +## Requirements + +- Use Rasterio for raster metadata when available. +- Use GeoPandas/Shapely for vector processing. +- Preserve CRS metadata. +- Return `unsupported_dataset_type` for unsupported files. +- Add tests with small fixtures. + +## Done when + +- Backend tests pass. +- Project/area/dataset flows still work. +- A vector fixture can be registered/imported and analyzed. +- Docs/log updated. diff --git a/docs/CODEX_PHASE_5_PROMPT.md b/docs/CODEX_PHASE_5_PROMPT.md new file mode 100644 index 00000000..88722310 --- /dev/null +++ b/docs/CODEX_PHASE_5_PROMPT.md @@ -0,0 +1,26 @@ +# Codex Phase 5 Prompt — Detection Pipeline Interface + +Implement the object detection pipeline interface without pretending that AI works when models are absent. + +## Goals + +- Detection service abstraction. +- YOLO provider interface. +- `not_configured` behavior when YOLO is disabled or model missing. +- Test fixture provider for deterministic detection polygons in tests only. +- Detection result persistence. +- GeoJSON export of detections. + +## Requirements + +- Do not hardcode fake production detections. +- Store confidence, class, geometry and model metadata. +- Add API endpoint according to `docs/API_CONTRACTS.md`. +- Add frontend state that shows configured/not configured. + +## Done when + +- Detection endpoint exists. +- It returns `not_configured` by default unless explicitly enabled. +- Tests can use fixture provider. +- Persisted detections can be exported. diff --git a/docs/CODEX_PHASE_6_PROMPT.md b/docs/CODEX_PHASE_6_PROMPT.md new file mode 100644 index 00000000..18d9ca31 --- /dev/null +++ b/docs/CODEX_PHASE_6_PROMPT.md @@ -0,0 +1,24 @@ +# Codex Phase 6 Prompt — QA/QC Engine + +Implement detection/reference QA. + +## Goals + +- Compare detection polygons against reference polygons. +- Compute IoU-based matches. +- Calculate precision, recall, F1. +- Persist quality check result. +- Return false positives and false negatives. + +## Rules + +- Default IoU threshold: 0.5. +- One detection can match at most one reference. +- Use greedy matching by highest IoU for V1. +- Document limitations. + +## Done when + +- Deterministic fixture test passes. +- QA endpoint returns metrics and findings. +- Frontend QA panel can display the result. diff --git a/docs/CODEX_PHASE_7_PROMPT.md b/docs/CODEX_PHASE_7_PROMPT.md new file mode 100644 index 00000000..8df3d19d --- /dev/null +++ b/docs/CODEX_PHASE_7_PROMPT.md @@ -0,0 +1,25 @@ +# Codex Phase 7 Prompt — Frontend Workbench Integration + +Build the first end-to-end UI around implemented APIs. + +## Goals + +- Home page with project list/create. +- Project workspace. +- Map workbench with area creation flow. +- Dataset manager panel. +- Capabilities panel. +- Analysis/QA placeholders connected to real backend statuses. + +## Rules + +- No fake success states. +- Every panel has loading, empty, error and success states. +- Keep UI clean and technical: GeoAI Workbench, not generic admin dashboard. + +## Done when + +- User can create a project from UI. +- User can create/register an area. +- User can see datasets and capabilities. +- Build passes. diff --git a/docs/CODEX_PHASE_8_PROMPT.md b/docs/CODEX_PHASE_8_PROMPT.md new file mode 100644 index 00000000..c730eb51 --- /dev/null +++ b/docs/CODEX_PHASE_8_PROMPT.md @@ -0,0 +1,19 @@ +# Codex Phase 8 Prompt — Demo Scenario Hardening + +Turn the foundation into a recruiter-friendly demo path. + +## Target demo + +Gebouwdetectie + QA tegen referentielaag. + +## Goals + +- Seed or import lightweight demo fixtures. +- Add one-click demo project creation. +- Provide clear stepper: area → reference data → detection → QA → export. +- Export a GeoJSON result. +- Add README demo instructions. + +## Done when + +A fresh clone can run the demo with documented commands and no manual architecture choices. diff --git a/docs/CODEX_PROMPT_M5_LONG_AUTONOMOUS_BUILD.md b/docs/CODEX_PROMPT_M5_LONG_AUTONOMOUS_BUILD.md new file mode 100644 index 00000000..b3e15eb5 --- /dev/null +++ b/docs/CODEX_PROMPT_M5_LONG_AUTONOMOUS_BUILD.md @@ -0,0 +1,24 @@ +# Codex Prompt — Long Autonomous Build + +```text +Je werkt aan GeoIntel Kempen, een GeoAI Workbench voor raster/vectoranalyse, objectdetectie, segmentatie, QA/QC tegen GRB en geospatiale export. + +Lees eerst AGENTS.md, README.md, docs/M5_OPERATIONAL_READINESS.md, docs/BUILD_GOVERNANCE.md, docs/V1_SCOPE_FREEZE.md, docs/REPOSITORY_CONVENTIONS.md, docs/SERVICE_ARCHITECTURE.md, docs/API_CONTRACTS.md, docs/DATABASE_SCHEMA.md, docs/HEALTHCHECK_CONTRACTS.md, docs/DEFINITION_OF_READY.md en docs/DEFINITION_OF_DONE.md. + +Werk volgens deze volgorde: +1. Pass 0 Repo Audit. +2. Pass 1 Backend Foundation. +3. Pass 2 Database and Models indien Pass 1 groen is. +4. Pass 3 Dataset Manager indien Pass 2 groen is. +5. Stop bij blocking issue en documenteer exact. + +Regels: +- Geen mockdata als echte functionaliteit presenteren. +- Geen nieuwe architectuurkeuzes zonder ADR/RFC. +- Geen regressies accepteren. +- Elke endpoint gebruikt consistent errorcontract. +- Elke lange taak wordt als job/analysis_run gemodelleerd. +- Werk CHANGELOG.md, docs/TODO.md en docs/CODEX_EXECUTION_LOG.md bij. + +Eindig met Completed, Partially completed, Not completed, Tests run, Known issues en Recommended next pass. +``` diff --git a/docs/COMPONENT_BREAKDOWN.md b/docs/COMPONENT_BREAKDOWN.md new file mode 100644 index 00000000..d7810b69 --- /dev/null +++ b/docs/COMPONENT_BREAKDOWN.md @@ -0,0 +1,253 @@ +# Frontend Component Breakdown + +## Global layout + +### AppShell +Responsibilities: +- main navigation +- project context indicator +- active job indicator +- responsive layout + +### LeftNavigation +Items: +- Home +- Projects +- Map Workbench +- Dataset Manager +- Raster Lab +- Vector Lab +- Detection Lab +- Segmentation Lab +- QA/QC Lab +- Change Lab +- Exports +- Settings + +### TopStatusBar +Shows: +- selected project +- selected area +- backend health +- queued/running jobs + +## Pages + +### HomePage +Purpose: +- start a new project +- open recent projects +- open demo scenario + +Components: +- HeroPanel +- NewProjectCard +- RecentProjectsList +- DemoScenarioLauncher + +API: +- GET /projects +- POST /projects + +### ProjectWorkspacePage +Purpose: +- overview for one project + +Components: +- ProjectHeader +- AreaSummaryCard +- DatasetSummaryCard +- AnalysisRunList +- KeyOutputCards +- NextActionPanel + +API: +- GET /projects/{project_id} +- GET /projects/{project_id}/areas +- GET /projects/{project_id}/datasets +- GET /projects/{project_id}/analysis-runs + +### MapWorkbenchPage +Purpose: +- central map inspection and area/layer management + +Components: +- MapCanvas +- LayerManager +- AreaDrawingTools +- FeatureInspector +- MapToolbar +- OpacitySlider + +API: +- GET /projects/{project_id}/areas +- GET /projects/{project_id}/layers +- GET /layers/{layer_id}/features + +### DatasetManagerPage +Purpose: +- upload/import datasets and inspect metadata + +Components: +- DatasetUploadDropzone +- DatasetTable +- DatasetMetadataPanel +- DatasetStatusBadge +- DatasetActionsMenu + +API: +- POST /projects/{project_id}/datasets/upload +- GET /projects/{project_id}/datasets +- GET /datasets/{dataset_id}/metadata + +### RasterLabPage +Purpose: +- raster inspection and preprocessing + +Components: +- RasterDatasetSelector +- RasterMetadataCard +- BandSelector +- RasterPreviewPanel +- RasterOperationPanel +- RasterArtifactList + +Actions: +- clip by area +- generate tiles +- calculate histogram +- calculate index later + +API: +- GET /datasets/{dataset_id}/metadata +- POST /rasters/{dataset_id}/clip +- POST /rasters/{dataset_id}/tile +- POST /rasters/{dataset_id}/histogram + +### VectorLabPage +Purpose: +- vector inspection and spatial operations + +Components: +- VectorDatasetSelector +- GeometrySummaryCard +- VectorOperationPanel +- AttributePreviewTable +- VectorLayerPreview + +Actions: +- validate geometry +- reproject +- clip +- buffer +- intersect + +API: +- POST /vectors/{dataset_id}/validate +- POST /vectors/{dataset_id}/clip +- POST /vectors/{dataset_id}/buffer +- POST /vectors/{dataset_id}/intersect + +### DetectionLabPage +Purpose: +- object detection workflow + +Components: +- DetectionInputSelector +- ModelSelector +- DetectionParameterPanel +- DetectionRunButton +- DetectionResultsMap +- DetectionMetricsCards +- DetectionExportPanel + +Parameters: +- confidence threshold +- tile size +- overlap +- target classes + +API: +- POST /analysis/object-detection +- GET /analysis-runs/{run_id} +- GET /analysis-runs/{run_id}/detections + +### SegmentationLabPage +Purpose: +- segmentation workflow + +Components: +- SegmentationInputSelector +- SegmentationModelSelector +- SegmentationParameterPanel +- MaskPreview +- PolygonizedOutputPanel +- SegmentationExportPanel + +API: +- POST /analysis/segmentation +- GET /analysis-runs/{run_id}/segmentations + +### QaqcLabPage +Purpose: +- compare AI output against reference data + +Components: +- PredictionLayerSelector +- ReferenceLayerSelector +- MatchThresholdSelector +- QaqcRunButton +- QaqcMetricsPanel +- ErrorLayerTabs +- FindingTable + +API: +- POST /analysis/{run_id}/quality-check +- GET /quality-checks/{check_id} + +### ChangeLabPage +Purpose: +- compare two vector or raster-derived layers + +Components: +- BaselineLayerSelector +- ComparisonLayerSelector +- ChangeMethodSelector +- ChangeResultsMap +- ChangeStatsPanel + +API: +- POST /analysis/change-detection +- GET /analysis-runs/{run_id}/changes + +### ExportsPage +Purpose: +- generate/download artifacts + +Components: +- ExportTypeSelector +- ExportTargetSelector +- ExportHistoryTable +- ExportDownloadButton + +API: +- POST /exports/geojson +- POST /exports/package +- GET /exports + +## Shared components +- LoadingState +- EmptyState +- ErrorState +- StatusBadge +- MetricCard +- JsonInspector +- GeometryBadge +- CrsBadge +- LayerLegend + +## UI principles +- Processing state must be visible. +- Empty states must suggest next action. +- No hidden GIS assumptions: CRS, bounds and source must be visible. +- Exports must clearly say which run/source they belong to. diff --git a/docs/DATABASE_IMPLEMENTATION_PLAN.md b/docs/DATABASE_IMPLEMENTATION_PLAN.md new file mode 100644 index 00000000..3df1c1fd --- /dev/null +++ b/docs/DATABASE_IMPLEMENTATION_PLAN.md @@ -0,0 +1,231 @@ +# Database Implementation Plan + +Database: PostgreSQL + PostGIS. + +## Rules + +- Store geometries in PostGIS with explicit SRID. +- Preserve original CRS metadata even when normalized geometry is stored as EPSG:4326 or a local projected CRS. +- Prefer UUID primary keys. +- Store large raster/mask/model files in filesystem or object storage; store metadata and paths in PostgreSQL. +- Keep analysis outputs reproducible by storing parameters JSON. + +## Required extensions + +```sql +CREATE EXTENSION IF NOT EXISTS postgis; +CREATE EXTENSION IF NOT EXISTS postgis_topology; +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +``` + +## Core tables + +### projects + +- `id uuid primary key` +- `name text not null` +- `description text` +- `region text default 'Kempen'` +- `status text default 'active'` +- `created_at timestamptz` +- `updated_at timestamptz` + +### areas + +- `id uuid primary key` +- `project_id uuid references projects(id)` +- `name text not null` +- `geometry geometry(MultiPolygon, 4326) not null` +- `original_crs text` +- `area_m2 double precision` +- `bbox geometry(Polygon, 4326)` +- `created_at timestamptz` + +Spatial index required on `geometry`. + +### datasets + +- `id uuid primary key` +- `project_id uuid references projects(id)` +- `area_id uuid nullable references areas(id)` +- `name text not null` +- `dataset_type text not null` +- `source text not null` +- `storage_path text` +- `derived_from_dataset_id uuid nullable references datasets(id)` +- `crs text` +- `bounds_json jsonb` +- `resolution_json jsonb` +- `bands_json jsonb` +- `metadata_json jsonb` +- `status text default 'created'` +- `created_at timestamptz` + +### vector_features + +Used for imported vector datasets and derived vector outputs when feature-level storage is needed. Original files remain source artifacts; this table is the queryable PostGIS state for vector features. + +- `id uuid primary key` +- `dataset_id uuid references datasets(id) on delete cascade` +- `feature_class text` +- `source_feature_id text` +- `properties_json jsonb` +- `geometry geometry(Geometry, 4326) not null` +- `created_at timestamptz` + +Required indexes: + +- `dataset_id` +- GiST index on `geometry` + +### analysis_runs + +- `id uuid primary key` +- `project_id uuid references projects(id)` +- `area_id uuid references areas(id)` +- `dataset_id uuid nullable references datasets(id)` +- `job_id uuid nullable references jobs(id)` +- `analysis_type text not null` +- `status text not null` +- `model_name text nullable` +- `model_version text nullable` +- `parameters_json jsonb not null` +- `result_json jsonb nullable` +- `created_at timestamptz` +- `started_at timestamptz` +- `finished_at timestamptz` +- `error_message text` + +Analysis runs are domain lifecycle records. Jobs track execution state; analysis runs track reproducibility, model metadata, parameters and result summaries. + +### detections + +- `id uuid primary key` +- `project_id uuid references projects(id)` +- `dataset_id uuid nullable references datasets(id)` +- `analysis_run_id uuid nullable references analysis_runs(id)` +- `job_id uuid nullable references jobs(id)` +- `model_name text not null` +- `model_version text nullable` +- `class_name text not null` +- `confidence double precision not null` +- `geometry geometry(Geometry, 4326)` +- `bbox_json jsonb` +- `source_tile_path text nullable` +- `properties_json jsonb` +- `created_at timestamptz` + +Required indexes: + +- `project_id` +- `dataset_id` +- `analysis_run_id` +- `class_name` +- GiST index on `geometry` + +Sprint 8 persists detections as first-class PostGIS records. Detections are never stored only in `jobs.result_json`. + +### segmentations + +- `id uuid primary key` +- `project_id uuid references projects(id)` +- `dataset_id uuid nullable references datasets(id)` +- `job_id uuid nullable references jobs(id)` +- `analysis_run_id uuid nullable references analysis_runs(id)` +- `model_name text not null` +- `model_version text nullable` +- `class_name text not null` +- `confidence double precision nullable` +- `geometry geometry(MultiPolygon, 4326) not null` +- `bbox_json jsonb` +- `area_m2 double precision` +- `mask_path text` +- `source_tile_path text` +- `tile_index integer` +- `properties_json jsonb` +- `provenance_json jsonb` +- `created_at timestamptz` + +Required indexes: + +- `project_id` +- `dataset_id` +- `analysis_run_id` +- `job_id` +- `class_name` +- GiST index on `geometry` + +Sprint 9 persists segmentation outputs as first-class PostGIS records. Mask paths are artifact/provenance references only; map display, QA and GeoJSON output use `segmentations.geometry`. + +### metrics + +- `id uuid primary key` +- `quality_check_id uuid nullable references quality_checks(id)` +- `analysis_run_id uuid nullable references analysis_runs(id)` +- `metric_key text not null` +- `metric_value double precision` +- `metric_unit text` +- `label text` +- `metadata_json jsonb` +- `created_at timestamptz` + +Metrics may belong to a quality check, an analysis run, or both. Sprint 7A persists QA/QC metrics through `quality_check_id`. + +### quality_checks + +- `id uuid primary key` +- `project_id uuid references projects(id)` +- `job_id uuid nullable references jobs(id)` +- `analysis_run_id uuid nullable references analysis_runs(id)` +- `candidate_dataset_id uuid nullable references datasets(id)` +- `reference_dataset_id uuid references datasets(id)` +- `check_type text not null` +- `status text not null` +- `score double precision` +- `parameters_json jsonb` +- `findings_json jsonb` +- `created_at timestamptz` +- `completed_at timestamptz nullable` + +Quality checks are domain records. Jobs track execution state; quality checks track the persisted QA/QC result; metrics track individual measurements. + +### exports + +- `id uuid primary key` +- `project_id uuid references projects(id)` +- `analysis_run_id uuid nullable references analysis_runs(id)` +- `export_type text not null` +- `storage_path text not null` +- `metadata_json jsonb` +- `created_at timestamptz` + +## Migration strategy + +- Use Alembic. +- First migration creates extensions and core tables. +- Second migration adds spatial indexes. +- Seed script may create a sample project and sample area only if explicitly run. + +## Sprint 7B provider-to-dataset mapping + +Provider integration is a contract layer only in Sprint 7B. Providers do not write directly to `vector_features`; future provider output must flow through `DatasetService` and `VectorFeatureService` so dataset provenance, storage metadata and feature persistence remain consistent. + +- `grb`: maps to `dataset_role='reference'`, `source_name='grb'`. +- `osm`: maps to `dataset_role='source'` by default, or `dataset_role='reference'` only when explicitly requested; `source_name='osm'`. +- `manual`: maps to `dataset_role='reference'`, `source_name='manual'`. +- `fixture`: maps to `dataset_role='reference'`, `source_name='fixture'`. + +GRB and OSM live imports are intentionally `not_configured` in Sprint 7B. Manual and fixture reference datasets use existing upload and fixture flows. + +## Geometry normalization + +- User-drawn polygons arrive as EPSG:4326. +- Uploaded vector data may arrive in another CRS; preserve original CRS and reproject to EPSG:4326 for storage. +- Area calculations should use a projected CRS suitable for Belgium, preferably EPSG:31370 or another documented Belgian projection. + +## Out of scope for V1 + +- Raster-in-database storage. +- Multi-tenant row-level security. +- User accounts. +- Full model registry tables. diff --git a/docs/DATABASE_SCHEMA.md b/docs/DATABASE_SCHEMA.md new file mode 100644 index 00000000..423dba8b --- /dev/null +++ b/docs/DATABASE_SCHEMA.md @@ -0,0 +1,233 @@ +# Database Schema v1.0 + +Doel: PostGIS-first schema dat projecten, gebieden, datasets, analyses, AI-resultaten, QA/QC en exports ondersteunt. + +## 1. projects + +```sql +id UUID PRIMARY KEY +name TEXT NOT NULL +description TEXT +region TEXT DEFAULT 'Kempen' +created_at TIMESTAMP NOT NULL +updated_at TIMESTAMP NOT NULL +``` + +## 2. areas + +```sql +id UUID PRIMARY KEY +project_id UUID REFERENCES projects(id) +name TEXT NOT NULL +geometry GEOMETRY(POLYGON, 4326) NOT NULL +area_m2 DOUBLE PRECISION +crs TEXT DEFAULT 'EPSG:4326' +created_at TIMESTAMP NOT NULL +``` + +## 3. datasets + +```sql +id UUID PRIMARY KEY +project_id UUID REFERENCES projects(id) +name TEXT NOT NULL +dataset_type TEXT NOT NULL -- raster, vector, lidar, reference, model_output +source TEXT -- upload, GRB, OSM, Sentinel, DHMV +original_path TEXT +processed_path TEXT +crs TEXT +srid INTEGER +bounds GEOMETRY(POLYGON, 4326) +metadata_json JSONB +status TEXT NOT NULL +created_at TIMESTAMP NOT NULL +updated_at TIMESTAMP NOT NULL +``` + +## 4. dataset_versions + +```sql +id UUID PRIMARY KEY +dataset_id UUID REFERENCES datasets(id) +version_label TEXT +path TEXT +metadata_json JSONB +created_at TIMESTAMP NOT NULL +``` + +## 5. vector_features + +```sql +id UUID PRIMARY KEY +dataset_id UUID REFERENCES datasets(id) +source_feature_id TEXT +feature_type TEXT +properties JSONB +geometry GEOMETRY(GEOMETRY, 4326) +created_at TIMESTAMP NOT NULL +``` + +Spatial index required on geometry. + +## 6. raster_assets + +```sql +id UUID PRIMARY KEY +dataset_id UUID REFERENCES datasets(id) +asset_type TEXT -- original, clipped, tile, preview, index, mask +path TEXT NOT NULL +bounds GEOMETRY(POLYGON, 4326) +metadata_json JSONB +created_at TIMESTAMP NOT NULL +``` + +## 7. analysis_runs + +```sql +id UUID PRIMARY KEY +project_id UUID REFERENCES projects(id) +area_id UUID REFERENCES areas(id) +analysis_type TEXT NOT NULL -- detection, segmentation, qaqc, change, raster_index +status TEXT NOT NULL +parameters_json JSONB +started_at TIMESTAMP +finished_at TIMESTAMP +error_message TEXT +created_at TIMESTAMP NOT NULL +``` + +## 8. detections + +```sql +id UUID PRIMARY KEY +analysis_run_id UUID REFERENCES analysis_runs(id) +class_name TEXT NOT NULL +confidence DOUBLE PRECISION +bbox_json JSONB +source_tile_path TEXT +geometry GEOMETRY(POLYGON, 4326) +metadata_json JSONB +created_at TIMESTAMP NOT NULL +``` + +Spatial index required on geometry. + +## 9. segmentations + +```sql +id UUID PRIMARY KEY +analysis_run_id UUID REFERENCES analysis_runs(id) +class_name TEXT NOT NULL +confidence DOUBLE PRECISION +area_m2 DOUBLE PRECISION +raster_mask_path TEXT +geometry GEOMETRY(GEOMETRY, 4326) +metadata_json JSONB +created_at TIMESTAMP NOT NULL +``` + +## 10. metrics + +```sql +id UUID PRIMARY KEY +analysis_run_id UUID REFERENCES analysis_runs(id) +metric_key TEXT NOT NULL +metric_value DOUBLE PRECISION +unit TEXT +metadata_json JSONB +created_at TIMESTAMP NOT NULL +``` + +## 11. quality_checks + +```sql +id UUID PRIMARY KEY +analysis_run_id UUID REFERENCES analysis_runs(id) +reference_dataset_id UUID REFERENCES datasets(id) +check_type TEXT NOT NULL +score DOUBLE PRECISION +findings_json JSONB +created_at TIMESTAMP NOT NULL +``` + +## 12. quality_findings + +```sql +id UUID PRIMARY KEY +quality_check_id UUID REFERENCES quality_checks(id) +finding_type TEXT NOT NULL -- false_positive, false_negative, low_iou, matched +class_name TEXT +score DOUBLE PRECISION +geometry GEOMETRY(GEOMETRY, 4326) +metadata_json JSONB +created_at TIMESTAMP NOT NULL +``` + +## 13. annotations + +```sql +id UUID PRIMARY KEY +project_id UUID REFERENCES projects(id) +dataset_id UUID REFERENCES datasets(id) +class_name TEXT NOT NULL +annotation_type TEXT -- bbox, polygon, mask +geometry GEOMETRY(GEOMETRY, 4326) +metadata_json JSONB +created_at TIMESTAMP NOT NULL +``` + +## 14. model_registry + +```sql +id UUID PRIMARY KEY +name TEXT NOT NULL +model_type TEXT -- yolo, sam, unet, custom_pytorch +version TEXT +artifact_path TEXT +classes_json JSONB +metrics_json JSONB +created_at TIMESTAMP NOT NULL +``` + +## 15. exports + +```sql +id UUID PRIMARY KEY +project_id UUID REFERENCES projects(id) +analysis_run_id UUID REFERENCES analysis_runs(id) +export_type TEXT NOT NULL -- geojson, yolo, coco, mask, report +path TEXT NOT NULL +metadata_json JSONB +created_at TIMESTAMP NOT NULL +``` + +## 16. jobs + +```sql +id UUID PRIMARY KEY +job_type TEXT NOT NULL +status TEXT NOT NULL +payload_json JSONB +result_json JSONB +error_message TEXT +created_at TIMESTAMP NOT NULL +started_at TIMESTAMP +finished_at TIMESTAMP +``` + +## Indexen + +- areas.geometry GIST +- datasets.bounds GIST +- vector_features.geometry GIST +- detections.geometry GIST +- segmentations.geometry GIST +- quality_findings.geometry GIST +- annotations.geometry GIST + +## Belangrijke regels + +- Grote bestanden nooit in database opslaan. +- Geometrieën altijd met SRID opslaan. +- Iedere modeloutput moet gekoppeld zijn aan een analysis_run. +- Iedere analysis_run moet parameters opslaan voor reproduceerbaarheid. diff --git a/docs/DATASET_STRATEGY.md b/docs/DATASET_STRATEGY.md new file mode 100644 index 00000000..9b7d2386 --- /dev/null +++ b/docs/DATASET_STRATEGY.md @@ -0,0 +1,140 @@ +# Dataset Strategy + +## Goal +Define exactly how GeoIntel obtains, stores, caches and uses source data. This prevents implementation drift during autonomous Codex passes. + +## Source priority +| Priority | Source | Role | V1 status | +|---:|---|---|---| +| 1 | GRB / Basiskaart Vlaanderen | Professional reference vector data for Flemish features | Required foundation | +| 2 | OSM | Fallback and supplementary vector context | Required foundation | +| 3 | User-uploaded raster/vector data | Portfolio demo data and controlled test input | Required foundation | +| 4 | Sentinel-2 L2A | Remote-sensing indices | V2 | +| 5 | DHMV / DEM / DSM | Height analysis | V3 | +| 6 | LAS / LAZ LiDAR | Point-cloud workbench | V4 | + +## GRB strategy +GRB is treated as the most important V1 external reference dataset. + +### Why +- It is the Flemish large-scale reference map. +- It provides official topographic reference geometry for professional GIS workflows. +- It is ideal for QA/QC of AI-generated building detections and other extracted features. + +### Access strategy +Preferred V1 strategy: +1. Implement `ReferenceLayerService` with a provider abstraction. +2. First provider: WFS/remote fetch if service endpoint configuration is available. +3. Second provider: local GeoPackage/GeoJSON/shapefile import for offline demos. +4. Cache fetched features in PostGIS with a dataset version record. + +This dual approach avoids blocking the project on one external API while still aligning with the real GRB workflow. + +### V1 GRB feature classes +Minimum required: +- buildings / building ground geometry +- roads or road-related reference geometry if easily available +- water features if easily available + +V1 building QA only needs the building reference layer to be fully functional. + +### Cache policy +- Cache by project area, layer name, provider and fetch date. +- Store original provider metadata. +- Store normalized geometry in PostGIS. +- Do not mutate cached reference features during analysis. + +## OSM strategy +OSM is used for fast, broad, fallback context. + +### Access options +- Overpass API for small selected areas. +- Local test fixtures for repeatable tests. +- Later: Geofabrik extracts if performance becomes an issue. + +### V1 OSM layers +- buildings +- roads +- water +- green/landuse where available + +### OSM caveat +OSM is community-maintained and may be incomplete. UI and reports must describe it as contextual/fallback data, not official ground truth. + +## User-uploaded raster strategy +V1 must support controlled local datasets because public raster access and model compatibility can be difficult. + +### Required upload types +- GeoTIFF / TIFF where raster georeferencing is available. +- JPG/PNG for non-georeferenced demo inference, with clear warning that outputs cannot be geospatially exported unless georeferencing is supplied. + +### Raster metadata required +- CRS if available +- bounds if available +- transform if available +- resolution if available +- band count +- nodata value if available +- width/height +- dtype + +## User-uploaded vector strategy +Required formats: +- GeoJSON +- zipped shapefile +- GeoPackage later if easy + +Required normalization: +- validate geometry +- determine CRS +- reproject to canonical project CRS when necessary +- store original metadata + +## Sentinel-2 strategy +Not required in first build, but architecture must prepare for it. + +### Preferred discovery +Use STAC-style catalogue access for Sentinel-2 L2A scenes once implemented. + +### V2 indices +- NDVI = `(NIR - Red) / (NIR + Red)` +- NDWI = `(Green - NIR) / (Green + NIR)` +- NDBI = `(SWIR - NIR) / (SWIR + NIR)` + +### Cloud handling +V2 should include cloud percentage filtering and warn when cloud contamination may affect outputs. + +## DHMV / height strategy +Not V1. Store architecture placeholders only. + +Future uses: +- elevation statistics +- slope +- low-point detection +- water sensitivity proxy +- building height if DSM and building footprints are available + +## Canonical CRS strategy +- Store project areas and vector outputs in PostGIS with SRID metadata. +- Use EPSG:4326 for API interchange when practical. +- Use a metric projected CRS for area/length calculations. For Flanders, prefer Belgian Lambert 72 / EPSG:31370 or another documented metric CRS. +- Never calculate area or distance on raw EPSG:4326 geometries. + +## Dataset versioning +Every dataset must have a version record: +- source name +- provider +- fetch/import date +- original path or endpoint +- CRS +- bounds +- checksum where applicable +- processing status + +## V1 acceptance +V1 dataset strategy is complete when: +- a user can upload one raster and one vector dataset; +- metadata is extracted and persisted; +- an area can request/cache a reference building layer; +- detection outputs can be compared with that reference layer; +- exports include source metadata. diff --git a/docs/DATA_CATALOG.md b/docs/DATA_CATALOG.md new file mode 100644 index 00000000..b3a64acb --- /dev/null +++ b/docs/DATA_CATALOG.md @@ -0,0 +1,227 @@ +# GeoIntel Kempen — Data Catalog v1.0 + +This catalog is the implementation reference for every dataset that GeoIntel may ingest, cache, analyse, validate against, or export. Codex must treat this file as source-of-truth when implementing data ingestion and analysis modules. + +## Data strategy + +GeoIntel is built around a professional reference-data-first strategy: + +1. Use official Flemish geodata where available. +2. Use OpenStreetMap as a supplemental and fallback source. +3. Use raster/satellite/aerial imagery for AI and remote-sensing pipelines. +4. Cache all expensive or external requests in PostGIS and/or filesystem storage. +5. Store provenance metadata for every derived result. + +## Priority sources + +| Priority | Source | Role | Data type | V1 status | +|---|---|---|---|---| +| P0 | GRB / Basiskaart Vlaanderen | official reference geometry | vector | required as primary reference target | +| P0 | User uploaded GeoTIFF / imagery | model input | raster | required | +| P0 | User uploaded GeoJSON/Shapefile/GPKG | vector input/reference | vector | required | +| P1 | OpenStreetMap | fallback and additional context | vector | required in V1 if GRB connector is not ready | +| P1 | Gebouwenregister | building identifiers and metadata | vector/API | prepare architecture | +| P2 | Sentinel-2 | NDVI/NDWI/NDBI and temporal analysis | raster | roadmap | +| P2 | DHMV / height products | DEM/DSM/slope/height analysis | raster/point cloud | roadmap | +| P3 | LAS/LAZ point clouds | LiDAR workbench | point cloud | roadmap | + +## GRB — Basiskaart Vlaanderen + +### Purpose +GRB is the main official reference layer for GeoIntel Kempen. It is used for QA/QC, validation, feature comparison, and map context. It contains accurately measured reference objects such as buildings, parcels, roads and road inrichting, watercourses, railway beds and road networks. It is a cost-free authentic source managed by Digitaal Vlaanderen. + +### Implementation role +GRB is not just a background map. It must become a validation and reference backbone: + +- Compare AI building detections with official building footprints. +- Identify false positives and false negatives. +- Create QA/QC dashboards. +- Provide high-quality vector context for map overlays. +- Support change-detection workflows where official snapshots are available. + +### Required layer groups +The exact service layer names must be discovered during implementation through the GRB WFS/WMS capabilities endpoint or downloaded package metadata. The application-level canonical layer groups are: + +| Canonical group | Expected geometry | Usage | +|---|---|---| +| `grb_buildings` | polygon | building reference, QA/QC, footprint analysis | +| `grb_roads` | line/polygon | road/infrastructure context, pressure metrics | +| `grb_water` | line/polygon | water context, water proximity, hydro overlays | +| `grb_railways` | line/polygon | infrastructure barrier/context | +| `grb_parcels` | polygon | optional parcel context, not required for V1 | + +### Required normalized fields +For every imported GRB feature, normalize at least: + +```yaml +id: internal UUID +source: "GRB" +source_layer: original layer name +source_feature_id: original feature identifier when available +canonical_group: grb_buildings | grb_roads | grb_water | grb_railways | grb_parcels +geometry: PostGIS geometry +geometry_type: Polygon | MultiPolygon | LineString | MultiLineString +crs_original: original CRS +crs_storage: EPSG:31370 or EPSG:4326 depending DB policy +attributes_json: raw attributes +fetched_at: timestamp +source_updated_at: timestamp when available +bbox: calculated bounds +area_m2: calculated for polygon features +length_m: calculated for linear features +``` + +### Access strategy +V1 may implement either: + +1. WFS area-of-interest fetch, preferred for targeted analysis. +2. Downloaded package import for cached offline analysis. +3. OSM fallback when GRB access is not implemented yet. + +Codex may build an abstraction `ReferenceDataProvider` so GRB and OSM can both satisfy the same downstream analysis contracts. + +## User uploaded raster imagery + +### Supported formats + +- GeoTIFF `.tif`, `.tiff` +- Cloud Optimized GeoTIFF if possible +- JPEG/PNG only if accompanied by georeferencing metadata or used as non-geospatial demo input + +### Required metadata + +```yaml +dataset_id +file_path +file_size_bytes +raster_driver +width +height +band_count +crs +transform +bounds +resolution_x +resolution_y +nodata_values +dtype_per_band +statistics_per_band +color_interpretation +is_georeferenced +created_at +``` + +### Required uses + +- Map overlay preview. +- Raster metadata inspection. +- Clipping to area of interest. +- Tiling for object detection and segmentation. +- Remote sensing index calculations when bands support it. + +## User uploaded vector data + +### Supported formats + +- GeoJSON +- Shapefile ZIP +- GeoPackage `.gpkg` +- KML/KMZ later + +### Required metadata + +```yaml +dataset_id +source_filename +format +layer_names +feature_count +geometry_types +crs +bounds +attributes_schema +created_at +``` + +### Required uses + +- Reference data. +- Manual annotations. +- User-supplied areas of interest. +- Comparison layers for QA/QC. + +## OpenStreetMap + +### Purpose +OSM is used for fast open-data enrichment and fallback when official sources are not yet implemented. + +### Canonical groups + +- `osm_buildings` +- `osm_roads` +- `osm_water` +- `osm_green` +- `osm_landuse` +- `osm_poi` + +### Required fields + +```yaml +osm_id +tags_json +canonical_group +geometry +area_m2 +length_m +fetched_at +``` + +## Sentinel-2 + +### Purpose +Sentinel-2 supports remote sensing indices and temporal change analysis. + +### Required indices later + +- NDVI = `(NIR - Red) / (NIR + Red)` +- NDWI = `(Green - NIR) / (Green + NIR)` +- NDBI = `(SWIR - NIR) / (SWIR + NIR)` + +### V1 rule +Prepare architecture, but do not block V1 on a full Sentinel downloader. + +## DHMV / DEM / DSM + +### Purpose +Height intelligence: slope, low points, terrain context, DSM minus DEM for object height approximations. + +### Required outputs later + +- `min_elevation_m` +- `max_elevation_m` +- `mean_elevation_m` +- `slope_mean_deg` +- `low_point_polygons` +- `height_profile_samples` + +## Data provenance requirements + +Every derived dataset must keep: + +```yaml +source_dataset_ids +processing_pipeline +parameters_json +software_versions +created_at +created_by_job_id +crs +quality_flags +``` + +## V1 non-negotiables + +- Do not hardcode temporary demo assumptions into data models. +- Every dataset must have metadata. +- Every derived output must reference its source dataset and analysis run. +- AI outputs must be stored as geospatial outputs, not only image overlays. diff --git a/docs/DATA_PRIVACY_AND_LICENSING.md b/docs/DATA_PRIVACY_AND_LICENSING.md new file mode 100644 index 00000000..7228a7ad --- /dev/null +++ b/docs/DATA_PRIVACY_AND_LICENSING.md @@ -0,0 +1,7 @@ +# Data Privacy and Licensing + +Elke dataset registreert source_name, source_url, license, attribution, retrieved_at, source_version en processing_steps. + +GRB, OSM en Sentinel krijgen correcte bronvermelding. Demo fixtures mogen synthetisch zijn maar moeten zo gemarkeerd worden. + +V1 verwerkt geen persoonsgegevens. Latere user accounts of adresflows vereisen privacy-analyse. diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md new file mode 100644 index 00000000..5a260207 --- /dev/null +++ b/docs/DATA_SOURCES.md @@ -0,0 +1,67 @@ +# Data Sources + +Dit document verzamelt concrete databronnen voor GeoIntel Kempen. + +## GRB — Basiskaart Vlaanderen + +- Naam: Basiskaart Vlaanderen / GRB +- Beheerder: Digitaal Vlaanderen +- Type: vector +- Gebruik: referentiegebouwen, wegen, terreinobjecten, QA/QC +- Toegang: WFS of download +- Cache: PostGIS per analysegebied +- Prioriteit: V1 +- Opmerking: belangrijkste officiële referentiebron voor QA/QC. + +## OSM + +- Naam: OpenStreetMap +- Type: vector +- Gebruik: fallback referentie, wegen, POI's, landuse +- Toegang: Overpass API of lokale extracten +- Cache: PostGIS per analysegebied +- Prioriteit: V1 fallback + +## Sentinel-2 + +- Naam: Copernicus Sentinel-2 +- Type: multispectraal raster +- Gebruik: NDVI, NDWI, NDBI, remote sensing change detection +- Toegang: Copernicus/STAC later te bepalen +- Cache: raster storage + metadata in PostGIS +- Prioriteit: V2 + +## DHMV + +- Naam: Digitaal Hoogtemodel Vlaanderen +- Type: raster/hoogte +- Gebruik: DEM, DSM, helling, laagste punten +- Toegang: Vlaamse open data, download/WCS nader te bepalen +- Cache: raster storage +- Prioriteit: V3 + +## Gebouwenregister + +- Naam: Gebouwenregister Vlaanderen +- Type: vector/API/metadata +- Gebruik: gebouwmetadata en statusinformatie +- Toegang: nader te bepalen +- Cache: PostGIS/metadata tabellen +- Prioriteit: V2 + +## Lokale demo datasets + +De repo moet ruimte voorzien voor kleine samplebestanden in `tests/fixtures/` of `datasets/raw/demo/`. Grote datasets mogen niet standaard in git. + +## Sprint 7B provider registry + +Sprint 7B exposes provider metadata only. It does not perform GRB WFS calls, OSM Overpass calls, downloads, or fake data generation. + +| provider | authority_level | configured | layers | dataset mapping | +| --- | --- | --- | --- | --- | +| `grb` | authoritative | false | buildings, roads, parcels | `dataset_role=reference`, `source_name=grb` | +| `osm` | contextual | false | buildings, roads, water, landuse | default `dataset_role=source`, explicit reference `dataset_role=reference`, `source_name=osm` | +| `manual` | manual | true | uploaded user layers | `dataset_role=reference`, `source_name=manual` | +| `fixture` | fixture | true | demo/test fixture layers | `dataset_role=reference`, `source_name=fixture` | + +Future provider output must flow through `DatasetService` and `VectorFeatureService`; providers must not write directly to `vector_features`. diff --git a/docs/DATA_SPECIFICATION.md b/docs/DATA_SPECIFICATION.md new file mode 100644 index 00000000..11fa8485 --- /dev/null +++ b/docs/DATA_SPECIFICATION.md @@ -0,0 +1,301 @@ +# Data Specification v1.0 + +## 1. Principes + +GeoIntel werkt documentatiegedreven met expliciete data-contracten. + +Elke dataset krijgt: + +- bron +- type +- formaat +- CRS +- bounds +- resolutie of schaal +- licentie +- updatefrequentie +- gebruiksdoel +- cache-strategie +- kwaliteitsstatus + +## 2. Kernbronnen + +## GRB — Basiskaart Vlaanderen + +### Rol + +GRB is een primaire referentiebron voor Vlaanderen. Voor GeoIntel is GRB belangrijker dan OSM wanneer officiële referentiegeometrieën nodig zijn. + +### Gebruik + +- referentiegebouwen +- wegen en terreinobjecten +- QA/QC tegenover AI-detecties +- validatie van gebouwdetectie +- vergelijking met eigen detecties of segmentaties + +### Type + +Vector. + +### Mogelijke toegang + +- WFS per gebied +- downloadpakketten +- periodieke PostGIS-cache + +### Cache-strategie + +Voor V1 is een gebiedsgerichte cache het beste: + +1. gebruiker selecteert gebied +2. backend vraagt relevante GRB-lagen op +3. geometrieën worden opgeslagen in PostGIS +4. analysis runs verwijzen naar de gecachete versie + +### Belangrijk voor QA/QC + +GRB-gebouwpolygonen worden gebruikt als ground-truth proxy. Niet absoluut perfect, maar zeer bruikbaar als officiële referentie. + +## Gebouwenregister + +### Rol + +Aanvullende gebouwinformatie bij geometrieën. + +### Gebruik + +- gebouwmetadata +- identificatie +- status +- koppeling met GRB-gebouwpolygonen indien mogelijk + +### Type + +Vector/API/metadata. + +### Prioriteit + +V2 of V1.5. + +## OpenStreetMap + +### Rol + +Snelle open referentiedata, nuttig als fallback en voor POI's. + +### Gebruik + +- gebouwen indien GRB nog niet beschikbaar is +- wegen +- water +- landuse +- POI's +- demo- en fallbackdata + +### Type + +Vector. + +### Toegang + +- Overpass API +- pyrosm/osmnx +- lokale extracten + +### Cache-strategie + +Opslaan per project/area in PostGIS. + +## Sentinel-2 + +### Rol + +Remote sensing bron voor vegetatie, water en bebouwingsindices. + +### Gebruik + +- NDVI +- NDWI +- NDBI +- seizoensvergelijking +- trendanalyse +- change detection + +### Type + +Raster multispectral. + +### Prioriteit + +V2. + +## DHMV / Hoogtemodel Vlaanderen + +### Rol + +Hoogtedata voor terrein- en watergevoeligheidsanalyse. + +### Gebruik + +- DEM +- DSM +- helling +- laagste punten +- hoogteprofiel +- gebouwhoogte-inschatting indien DSM + gebouwpolygonen beschikbaar zijn + +### Type + +Raster / afgeleid van LiDAR. + +### Prioriteit + +V3. + +## LAS/LAZ LiDAR + +### Rol + +Advanced module voor puntenwolkanalyse. + +### Gebruik + +- puntenwolk metadata +- clipping +- classificatie +- DEM/DSM generatie +- hoogteprofielen + +### Tools + +- PDAL +- laspy + +### Prioriteit + +V4/V5. + +## 3. Dataset Types + +### Raster + +Ondersteund: + +- GeoTIFF +- TIFF +- JPEG2000 later +- PNG/JPG met worldfile later + +Metadata: + +- CRS +- transform +- bounds +- width/height +- band count +- dtype +- nodata +- resolution + +### Vector + +Ondersteund: + +- GeoJSON +- Shapefile +- GPKG +- WFS-resultaten + +Metadata: + +- CRS +- bounds +- geometry type +- feature count +- attributes +- validity summary + +### Model outputs + +Ondersteund: + +- bounding boxes +- polygons +- masks +- confidence scores +- class labels +- source tile reference + +## 4. Data Quality Requirements + +Elke dataset moet bij intake minimaal controleren: + +- bestand leesbaar +- CRS aanwezig of expliciet onbekend +- geometrieën geldig of herstelbaar +- bounds binnen verwacht gebied +- raster heeft bruikbare resolutie +- geen lege lagen +- opslagpad bestaat + +## 5. Data Lineage + +Elke output moet kunnen terugwijzen naar: + +- bronbestand +- processing step +- analysis run +- modelversie +- parameters +- tijdstip +- eventuele referentielaag + +## Sprint 7B Provider Contracts + +Sprint 7B makes provider architecture operationally visible without performing live external fetches. + +### GRB + +- provider_name: `grb` +- authority_level: `authoritative` +- configured: `false` +- status: `not_configured` +- supported_layers: `buildings`, `roads`, `parcels` +- supported_geometry_types: `Polygon`, `MultiPolygon`, `LineString`, `MultiLineString` +- supported_query_modes: `area` +- dataset mapping: `dataset_role=reference`, `source_name=grb` +- write path: future provider output must flow through `DatasetService` and `VectorFeatureService` +- limitation: no live WFS call, no download, no fake GRB data in Sprint 7B + +### OSM + +- provider_name: `osm` +- authority_level: `contextual` +- configured: `false` +- status: `not_configured` +- supported_layers: `buildings`, `roads`, `water`, `landuse` +- supported_geometry_types: `Polygon`, `MultiPolygon`, `LineString`, `MultiLineString` +- supported_query_modes: `area` +- default dataset mapping: `dataset_role=source`, `source_name=osm` +- explicit reference mapping: `dataset_role=reference`, `source_name=osm` +- write path: future provider output must flow through `DatasetService` and `VectorFeatureService` +- limitation: no live Overpass call, no download, no fake OSM data in Sprint 7B + +### Manual + +- provider_name: `manual` +- authority_level: `manual` +- configured: `true` +- status: `configured` +- dataset mapping: `dataset_role=reference`, `source_name=manual` +- write path: existing dataset upload/reference flow + +### Fixture + +- provider_name: `fixture` +- authority_level: `fixture` +- configured: `true` +- status: `configured` +- dataset mapping: `dataset_role=reference`, `source_name=fixture` +- write path: checked-in demo/test fixture flow diff --git a/docs/DEFINITION_OF_DONE.md b/docs/DEFINITION_OF_DONE.md new file mode 100644 index 00000000..86889e88 --- /dev/null +++ b/docs/DEFINITION_OF_DONE.md @@ -0,0 +1,47 @@ +# Definition of Done + +A feature is done only when all applicable criteria are met. + +## Backend feature + +- Endpoint or service is implemented. +- Input schema exists. +- Output schema exists. +- Error behavior is defined. +- Test exists or limitation is documented. +- No unhandled tracebacks for expected user errors. +- Docs are updated when behavior differs from spec. + +## Frontend feature + +- Page/component is connected to real API or explicit fixture mode. +- Loading state exists. +- Empty state exists. +- Error state exists. +- Main action is discoverable. +- API failures are visible to the user. +- No hardcoded fake success for real workflows. + +## Geo-processing feature + +- CRS handling is explicit. +- Units are documented. +- Input geometry validity is checked. +- Output geometry is valid or repair attempt is documented. +- Large-file limitations are surfaced. + +## AI feature + +- Model dependency/configuration is explicit. +- If model is unavailable, user sees `not_configured`. +- Input preprocessing is documented. +- Output georeferencing is documented. +- Confidence threshold is configurable. + +## Repo pass + +Every Codex build pass must update: + +- `docs/CODEX_EXECUTION_LOG.md` +- `docs/TODO.md` if tasks are completed or added +- relevant API/schema docs if changed diff --git a/docs/DEFINITION_OF_READY.md b/docs/DEFINITION_OF_READY.md new file mode 100644 index 00000000..6e2b404b --- /dev/null +++ b/docs/DEFINITION_OF_READY.md @@ -0,0 +1,13 @@ +# Definition of Ready + +A task is ready for Codex implementation only when: + +- The module is described in docs. +- The expected input/output is known. +- The API contract exists or the task explicitly creates it. +- The data model is defined or the task explicitly creates it. +- Acceptance criteria exist. +- Error behavior is defined. +- The task does not require hidden external credentials. + +If these conditions are not met, Codex must update the specification before implementing code. diff --git a/docs/DEMO_FIXTURE_MANIFEST.md b/docs/DEMO_FIXTURE_MANIFEST.md new file mode 100644 index 00000000..0bab20d4 --- /dev/null +++ b/docs/DEMO_FIXTURE_MANIFEST.md @@ -0,0 +1,38 @@ +# Demo Fixture Manifest + +Demo fixtures are used for UI, API and QA/QC development before live data connectors are complete. + +## Geel Building Detection Demo +Path: `demo/geel/` +Purpose: building detection and GRB-like reference QA. + +Required fixture files: +- `area_geel_center.geojson`: analysis polygon. +- `reference_buildings.geojson`: GRB-like building reference layer. +- `demo_detections.geojson`: deterministic AI detection output. +- `expected_qaqc_metrics.json`: expected precision/recall/F1. + +## Mol Vegetation Demo +Path: `demo/mol/` +Purpose: segmentation and vegetation/water area metrics. + +Required fixture files: +- `area_mol_lakes.geojson` +- `demo_segmentation_vegetation.geojson` +- `expected_segmentation_metrics.json` + +## Turnhout Change Demo +Path: `demo/turnhout/` +Purpose: vector change detection workflow. + +Required fixture files: +- `area_turnhout_growth.geojson` +- `buildings_t0.geojson` +- `buildings_t1.geojson` +- `expected_change_metrics.json` + +## Fixture Rules +- Fixtures must be small enough for git. +- Fixtures must not contain private data. +- Fixtures must be deterministic. +- Expected metrics must be versioned with the fixture. diff --git a/docs/DEMO_SCENARIOS.md b/docs/DEMO_SCENARIOS.md new file mode 100644 index 00000000..57928c56 --- /dev/null +++ b/docs/DEMO_SCENARIOS.md @@ -0,0 +1,87 @@ +# GeoIntel Kempen — Demo Scenarios v1.0 + +Demo scenarios guide development toward portfolio-ready workflows. + +## Demo 1 — Building Detection + GRB QA/QC + +### Goal +Show the complete GeoAI loop: raster imagery → AI detections → geospatial output → official reference validation. + +### Workflow + +1. Create project: `Geel Building Detection QA`. +2. Select or draw area in Geel. +3. Upload a georeferenced aerial/orthophoto raster. +4. Fetch GRB buildings for the area or use OSM fallback. +5. Run object detection for buildings. +6. Convert detections to geospatial polygons. +7. Run QA/QC against GRB buildings. +8. Show precision, recall, F1, mean IoU. +9. Display TP/FP/FN on map. +10. Export detections and QA findings as GeoJSON/CSV. + +### Recruiter value +Demonstrates Rasterio, tiling, YOLO/PyTorch, GeoPandas/Shapely, PostGIS, QA metrics and official geodata validation. + +## Demo 2 — Vegetation Segmentation + +### Goal +Show segmentation/mask-to-vector workflow. + +### Workflow + +1. Select area around Mol or Kasterlee. +2. Upload raster or use sample imagery. +3. Run segmentation or NDVI-based threshold. +4. Polygonize vegetation masks. +5. Calculate green area and fragmentation. +6. Export vegetation polygons. + +## Demo 3 — Vector Change Detection + +### Goal +Show change analysis before full raster AI change detection is implemented. + +### Workflow + +1. Select two building/reference layers for the same area. +2. Run vector change detection. +3. Classify added, removed and modified features. +4. Show change statistics and map overlays. + +## Demo 4 — Raster Workbench + +### Goal +Show professional raster handling. + +### Workflow + +1. Upload GeoTIFF. +2. Inspect metadata. +3. Clip to area. +4. Generate preview. +5. Generate tiles. +6. Export clipped raster. + +## Demo 5 — Vector Workbench + +### Goal +Show core GIS operations. + +### Workflow + +1. Upload GeoJSON/GPKG. +2. Inspect metadata. +3. Clip to area. +4. Buffer roads. +5. Intersect buffers with green/water/building layers. +6. Export result. + +## Demo readiness checklist + +- [ ] Demo datasets present or documented. +- [ ] No fake results. +- [ ] Every result is exportable. +- [ ] Every map layer is inspectable. +- [ ] Every long operation has job status. +- [ ] QA output can be explained in one minute. diff --git a/docs/DEMO_USE_CASES.md b/docs/DEMO_USE_CASES.md new file mode 100644 index 00000000..8dbace45 --- /dev/null +++ b/docs/DEMO_USE_CASES.md @@ -0,0 +1,117 @@ +# Demo Use Cases + +These are the workflows GeoIntel must be built around. Codex should prioritize these over generic feature expansion. + +## Demo 1 — Building Detection + GRB QA + +### Purpose +Show the exact portfolio fit for GeoAI engineering: raster input, computer vision, georeferenced detections, reference-data comparison, QA metrics and GIS export. + +### Area +Default: Geel or another Kempen area with mixed buildings and roads. + +### Input +- Georeferenced aerial image or GeoTIFF. +- Reference building layer from GRB or local fixture. + +### Pipeline +1. Create project. +2. Define area. +3. Import raster. +4. Fetch/cache GRB building reference layer. +5. Tile raster. +6. Run YOLO/object-detection provider. +7. Convert detections back to geospatial polygons or boxes. +8. Store detections in PostGIS. +9. Run QA/QC against GRB building footprints. +10. Show results on map. +11. Export detections and QA report. + +### Required outputs +- Detection layer. +- GRB reference layer. +- Matched features layer. +- False positives layer. +- False negatives layer. +- Precision/recall/F1/IoU dashboard. +- GeoJSON export. + +### Portfolio message +“I built a GeoAI pipeline that detects buildings on imagery and validates the results against official Flemish reference geometry.” + +## Demo 2 — Vegetation Segmentation + +### Purpose +Show segmentation and raster/vector conversion. + +### Input +- Raster image or Sentinel-derived raster later. +- Optional area polygon. + +### Pipeline +1. Import raster. +2. Run segmentation provider or thresholded index workflow. +3. Produce mask. +4. Polygonize mask. +5. Calculate area and coverage percentage. +6. Visualize mask and polygons. +7. Export GeoJSON/mask. + +### Required outputs +- Segmentation mask. +- Polygonized vegetation layer. +- Area statistics. +- Export package. + +## Demo 3 — Vector Change Detection + +### Purpose +Show geospatial comparison before full remote-sensing change detection is implemented. + +### Input +- Two reference/detection vector layers for different dates or runs. + +### Pipeline +1. Select baseline layer. +2. Select comparison layer. +3. Normalize CRS. +4. Match overlapping objects. +5. Identify added/removed/changed geometries. +6. Calculate area deltas. +7. Show change layer. + +### Required outputs +- Added features. +- Removed features. +- Changed features. +- Change statistics. +- GeoJSON export. + +## Demo 4 — Raster Indices Preview + +### Purpose +Prepare the Sentinel/remote-sensing part. + +### Input +- Multiband raster with suitable bands. + +### Pipeline +1. Select bands. +2. Calculate NDVI/NDWI/NDBI where bands exist. +3. Render preview. +4. Calculate min/max/mean/histogram. +5. Extract thresholded areas later. + +### Required outputs +- Index raster artifact. +- Histogram/statistics. +- Preview overlay. + +## Demo priority +Build in this order: +1. Demo 1 first. +2. Demo 3 second. +3. Demo 2 third. +4. Demo 4 fourth. + +Demo 1 is the main recruiter-facing workflow. diff --git a/docs/DEPENDENCY_LOCK_PLAN.md b/docs/DEPENDENCY_LOCK_PLAN.md new file mode 100644 index 00000000..86b961cb --- /dev/null +++ b/docs/DEPENDENCY_LOCK_PLAN.md @@ -0,0 +1,14 @@ +# Dependency Lock Plan + +## Python +Gebruik `pyproject.toml`, optioneel `requirements.in`, en locked `requirements.txt`. + +V1 direct: fastapi, uvicorn, sqlalchemy, alembic, psycopg, pydantic, redis, rq, geopandas, shapely, pyproj, rasterio, numpy, pillow. + +AI-profiel: torch, ultralytics, opencv-python, SAM-package. + +## Node +Gebruik één package manager en lockfile. Niet mengen. + +## Policy +Nieuwe dependency vereist doel, alternatief-overweging en groene quick smoke. diff --git a/docs/DEPENDENCY_POLICY.md b/docs/DEPENDENCY_POLICY.md new file mode 100644 index 00000000..3bb15a8f --- /dev/null +++ b/docs/DEPENDENCY_POLICY.md @@ -0,0 +1,77 @@ +# Dependency Policy + +GeoIntel deliberately uses technologies matching the GeoAI Engineer profile. Dependencies must support the core goal: geospatial processing, remote sensing, AI inference, QA/QC, and professional frontend visualization. + +## Approved backend core + +- FastAPI +- Uvicorn +- Pydantic +- SQLAlchemy +- Alembic +- GeoAlchemy2 +- psycopg +- python-multipart + +## Approved GIS/remote-sensing + +- GeoPandas +- Shapely +- PyProj +- Rasterio +- GDAL where available +- Fiona or pyogrio where needed +- NumPy +- OpenCV when needed for image processing + +## Docker GIS runtime + +The backend Docker image may install the approved `gis` optional dependency +group so the deployed workbench has real raster/vector runtime capability: + +- rasterio +- numpy +- pillow +- geopandas +- pyogrio + +The Docker image may also install GDAL, GEOS and PROJ system packages required +by those GIS libraries. This does not enable new product behavior by itself; it +only allows existing raster/vector endpoints to run when requested. + +AI dependencies remain separate in the `ai` optional dependency group and must +not be installed by the default Docker backend image unless an explicit AI image +or profile is introduced later. + +## Approved AI + +- PyTorch +- Ultralytics +- Segment Anything only after detection foundation works + +## Approved frontend + +- React +- TypeScript +- Vite +- MapLibre GL +- Deck.gl +- TanStack Query +- Zustand or React context for local UI state +- Recharts for charts + +## Add-dependency rule + +Before adding a dependency: + +1. Explain why existing dependencies are insufficient. +2. Add it to this document. +3. Add setup notes if it has native/system requirements. +4. Ensure Docker build still works. + +## Avoid in V1 + +- Heavy MLOps platforms. +- User auth frameworks. +- Full workflow orchestration stacks beyond simple queueing. +- Unnecessary UI component mega-libraries. diff --git a/docs/DESIGN_SYSTEM.md b/docs/DESIGN_SYSTEM.md new file mode 100644 index 00000000..46644095 --- /dev/null +++ b/docs/DESIGN_SYSTEM.md @@ -0,0 +1,47 @@ +# Design System + +## Core components + +### AppShell +Global layout with left navigation, top project context bar, and main workspace area. + +### MapPanel +MapLibre-powered viewport with layer controls and inspection tools. + +### InspectorPanel +Right-side contextual panel showing metadata, selected features, analysis settings, and results. + +### DatasetTable +List of datasets with type, source, status, CRS, bounds, created date, and actions. + +### AnalysisRunCard +Displays an analysis run status, progress, parameters, and outputs. + +### LayerTree +Allows toggling, opacity changes, and layer ordering. + +### MetricCard +Shows one calculated metric with unit and provenance. + +### QualityMetricCard +Shows precision, recall, F1, IoU, false positives, and false negatives. + +### EmptyState +Every page must have a useful empty state with a next action. + +## Required states + +Every interactive component should handle: + +- idle, +- loading, +- success, +- empty, +- error. + +## Forbidden patterns + +- Hidden mock-only states. +- Buttons without implemented actions. +- Tables without empty state. +- Map overlays without legend. diff --git a/docs/DETECTION_CLASSES_CATALOG.md b/docs/DETECTION_CLASSES_CATALOG.md new file mode 100644 index 00000000..e591530d --- /dev/null +++ b/docs/DETECTION_CLASSES_CATALOG.md @@ -0,0 +1,37 @@ +# Detection Classes Catalog + +## V1 required class + +### building + +Reason: +Buildings are the best first detection target because GRB can be used as a reference layer for QA/QC. + +Outputs: + +- bounding box polygon +- confidence +- source tile +- model id + +## V1 optional classes + +### solar_panel + +High portfolio value but dataset/model availability may be harder. + +### vehicle + +Useful for generic object detection demo but less central to GIS validation. + +### tree + +Potentially useful but segmentation may be more appropriate than bounding boxes. + +## Not required for V1 + +- roads as object detection class +- water as object detection class +- agriculture as object detection class + +These are better handled through segmentation or raster indices. diff --git a/docs/DETECTION_PIPELINE_SPEC.md b/docs/DETECTION_PIPELINE_SPEC.md new file mode 100644 index 00000000..70afee6d --- /dev/null +++ b/docs/DETECTION_PIPELINE_SPEC.md @@ -0,0 +1,165 @@ +# GeoIntel Kempen — Detection Pipeline Specification v1.0 + +The detection pipeline converts geospatial imagery into georeferenced object detections stored as GIS features. + +## Goal + +Run object detection on aerial or satellite imagery and export results as map layers and GeoJSON. + +## Supported model family + +V1 target: + +- Ultralytics YOLO detection model. + +Later: + +- custom PyTorch models +- OpenMMLab detectors + +## Pipeline + +```text +Raster dataset +↓ +Raster validation +↓ +Tile generation +↓ +Model inference per tile +↓ +Detection postprocessing +↓ +Pixel coordinates → geospatial coordinates +↓ +Merge / de-duplicate overlapping detections +↓ +Store detections in PostGIS +↓ +Render as map overlay +↓ +Export as GeoJSON/CSV +``` + +## Step 1 — Validation + +Check: + +- raster exists +- raster is readable +- raster has CRS and transform +- selected bands are valid +- tile size is valid +- model file exists or selected model is available + +## Step 2 — Tile generation + +Parameters: + +```yaml +tile_size_px: 640 +overlap_px: 96 +bands: [1, 2, 3] +area_id: optional +``` + +Every tile must store: + +- tile path +- source pixel window +- affine transform +- bounds +- source dataset id + +## Step 3 — Inference + +For each tile: + +- convert to model input image +- run YOLO +- collect boxes, classes, confidence + +Raw detection format: + +```json +{ + "tile_id": "uuid", + "class_name": "building", + "confidence": 0.91, + "bbox_px": [x1, y1, x2, y2] +} +``` + +## Step 4 — Georeferencing + +Convert pixel bbox corners using the tile affine transform. + +Output geometry: + +- polygon footprint of bbox for detection models +- centroid point optional +- original pixel bbox kept in metadata + +## Step 5 — Merge and de-duplicate + +Because tiles overlap, duplicated detections must be removed. + +Default method: + +- group by class +- calculate IoU between overlapping geospatial bboxes +- apply non-maximum suppression by confidence +- default IoU merge threshold: 0.5 + +## Step 6 — Storage + +Store each detection: + +```yaml +analysis_run_id +class_name +confidence +geometry +bbox_json +source_tile_id +model_name +model_version +metadata_json +``` + +## Step 7 — Metrics + +Calculate: + +- detection count by class +- mean confidence +- low confidence count +- confidence histogram +- detected area by class when polygon geometry is valid + +## API + +```http +POST /analysis/object-detection +GET /analysis/{id} +GET /analysis/{id}/detections +POST /analysis/{id}/exports/geojson +``` + +## UI + +Detection Lab must allow: + +- raster selection +- model selection +- class selection +- confidence threshold +- tile size +- overlap +- run detection +- map overlay visibility +- QA/QC handoff button + +## V1 acceptable model strategy + +If no locally trained model exists yet, the system may support a configurable YOLO model path and clearly mark model classes as model-dependent. The code must not fake detections. diff --git a/docs/DEVELOPMENT_RULES.md b/docs/DEVELOPMENT_RULES.md new file mode 100644 index 00000000..2a771371 --- /dev/null +++ b/docs/DEVELOPMENT_RULES.md @@ -0,0 +1,69 @@ +# GeoIntel Kempen — Development Rules v1.0 + +These rules apply to Codex and every AI/developer working in this repository. + +## Product rules + +1. GeoIntel is a GeoAI Workbench, not a generic dashboard. +2. Analysis and geospatial outputs are the product. Maps support the analysis. +3. The main technical story is: data → processing → AI/model → GIS output → QA/QC → export. +4. GRB/official reference data and QA/QC are core differentiators. + +## Implementation rules + +1. Backend-first for data models, processing contracts and APIs. +2. UI must be API-driven; do not hardcode fake results. +3. Mock data is allowed only for explicit UI skeletons and must be clearly marked. +4. No placeholder endpoints that return success without doing useful work. +5. Every long-running operation must be modeled as a job/analysis run. +6. Every dataset must have metadata. +7. Every derived output must reference source datasets and analysis run. +8. Every export must be traceable. +9. Every geometry operation must handle CRS and invalid geometries. +10. Every metric must include unit and calculation method. + +## Code quality rules + +1. Type-safe code where possible. +2. Separate API routes, services, database models, schemas and processing utilities. +3. Avoid giant files. +4. Use explicit names: `RasterMetadataService`, `DetectionPipeline`, `QaqcService`. +5. Add tests for pure processing functions. +6. Do not silently swallow errors. +7. Return actionable validation messages. + +## GIS rules + +1. Never calculate area/length in geographic degrees. +2. Reproject to a metric CRS for calculations. +3. Store original CRS metadata. +4. Repair invalid geometries before analysis, but record that repair happened. +5. Keep raw attributes in `attributes_json` when normalizing external features. + +## AI rules + +1. AI detections must be stored as geospatial features. +2. Model outputs must record model name, version, classes and confidence threshold. +3. Do not fake model results. +4. If a model is unavailable, show a clear system readiness message. +5. The AI copilot is optional and must only interpret real metrics. + +## Frontend rules + +1. Every page needs useful empty states. +2. Every analysis result needs an export action. +3. Every map layer needs visibility, opacity and inspect controls. +4. Every job needs visible status. +5. Avoid clutter; use workbench panels. + +## Delivery rules + +Every build response should include: + +- What changed. +- What files changed. +- How to run/test. +- What remains open. +- Any risks or assumptions. + +Do not hide unfinished core functionality behind a TODO list unless explicitly requested. diff --git a/docs/DOMAIN_MODEL.md b/docs/DOMAIN_MODEL.md new file mode 100644 index 00000000..ccf44a0c --- /dev/null +++ b/docs/DOMAIN_MODEL.md @@ -0,0 +1,100 @@ +# Domain Model + +## Project + +A project represents a GeoAI investigation. It is not just a folder. It defines the purpose, spatial areas, datasets, analysis runs, and outputs. + +Fields: + +- id +- name +- description +- region +- status +- created_at +- updated_at + +Relationships: + +- has many areas +- has many datasets +- has many analysis_runs +- has many exports + +## Area + +An area is a selected geometry, usually a polygon. It may represent a municipality, manually drawn polygon, bounding box, or uploaded GeoJSON geometry. + +Rules: + +- Must have valid geometry. +- Must have a calculated area in square meters. +- Must keep original input geometry if uploaded. +- Must be convertible to GeoJSON. + +## Dataset + +A dataset is any input or derived geospatial artifact. + +Dataset types: + +- raster +- vector +- reference +- detection_output +- segmentation_output +- mask +- tile_set +- export + +Dataset statuses: + +- registered +- uploaded +- metadata_extracted +- processed +- failed + +## AnalysisRun + +Tracks execution of a pipeline. + +Analysis types: + +- raster_metadata +- raster_clip +- raster_tile +- vector_import +- object_detection +- segmentation +- qaqc +- change_detection +- export + +## Detection + +A model-detected object. V1 supports bounding boxes converted into geospatial polygons. + +Important fields: + +- class_name +- confidence +- geometry +- bbox_pixel +- bbox_geo +- source_tile +- model_id + +## QualityCheck + +A measured comparison between predicted features and reference features. + +Core metrics: + +- precision +- recall +- f1 +- mean_iou +- matched_count +- false_positive_count +- false_negative_count diff --git a/docs/ENVIRONMENT_SPEC.md b/docs/ENVIRONMENT_SPEC.md new file mode 100644 index 00000000..e96694ed --- /dev/null +++ b/docs/ENVIRONMENT_SPEC.md @@ -0,0 +1,110 @@ +# Environment Specification + +## Required runtime services + +- Backend: FastAPI. +- Frontend: Vite React. +- Database: PostgreSQL with PostGIS. +- Optional queue: Redis/RQ after core APIs work. + +## Environment variables + +### Backend + +```env +GEOINTEL_ENV=development +GEOINTEL_API_PREFIX=/api/v1 +DATABASE_URL=postgresql+psycopg://geointel:geointel@db:5432/geointel +STORAGE_ROOT=/app/storage +MAX_UPLOAD_MB=500 +YOLO_ENABLED=false +YOLO_MODEL_PATH= +YOLO_MODEL_VERSION= +YOLO_MAX_TILES=100 +ENABLE_GRB_WFS=false +GRB_WFS_URL= +OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter +``` + +### Frontend + +```env +VITE_API_BASE_URL= +VITE_API_PROXY_TARGET=http://localhost:8000 +VITE_MAP_STYLE_URL=https://demotiles.maplibre.org/style.json +``` + +`VITE_API_BASE_URL` is intentionally empty by default so the browser calls the +same origin as the frontend. In Docker Compose, nginx serves the built frontend +and reverse proxies `/api` and `/health` to the backend container. For local +Vite development, `VITE_API_PROXY_TARGET` can point to the local backend. + +## Docker compose services + +- `db`: PostGIS image. +- `backend`: FastAPI app. +- `frontend`: nginx-served Vite build with `/api` and `/health` reverse proxy. +- `redis`: optional, introduced when async jobs are implemented. + +## Docker GIS capabilities + +The default backend Docker image installs the approved GIS runtime extra and +system libraries needed for existing raster/vector processing: + +- rasterio +- numpy +- pillow +- geopandas +- pyogrio +- GDAL/GEOS/PROJ runtime packages + +After a Docker rebuild, the browser-facing runtime should report: + +```json +{ + "postgis": true, + "rasterio": true, + "geopandas": true +} +``` + +Verify this through the frontend proxy: + +```bash +bash scripts/verify_gis_runtime.sh http://localhost:1202 +``` + +For a LAN deployment, replace the URL with the published host address, for +example `http://192.168.10.150:1202`. + +## Local development commands + +Backend: + +```bash +cd backend +uvicorn app.main:app --reload +pytest +``` + +Frontend: + +```bash +cd frontend +npm install +npm run dev +npm run build +``` + +## Storage mounts + +Development storage: + +```text +storage/originals +storage/derived +storage/tiles +storage/masks +storage/exports +storage/models +``` diff --git a/docs/ERROR_HANDLING_AND_STATUSES.md b/docs/ERROR_HANDLING_AND_STATUSES.md new file mode 100644 index 00000000..add24dee --- /dev/null +++ b/docs/ERROR_HANDLING_AND_STATUSES.md @@ -0,0 +1,55 @@ +# Error Handling and Statuses + +## API error contract + +All expected errors return `ApiError`. + +Examples: + +- `validation_error` +- `not_found` +- `unsupported_dataset_type` +- `invalid_geometry` +- `dependency_not_configured` +- `processing_failed` +- `file_too_large` + +## Job/analysis statuses + +Use these statuses consistently: + +- `queued` +- `running` +- `completed` +- `failed` +- `cancelled` +- `not_configured` + +## Dataset statuses + +- `created` +- `metadata_pending` +- `metadata_ready` +- `metadata_failed` +- `processing` +- `ready` +- `failed` + +## Frontend display rules + +- Never show raw stack traces. +- Show the user what failed and what they can do next. +- For `dependency_not_configured`, show which feature flag or environment variable is missing. +- For geospatial errors, include CRS/geometry hints when available. + +## Logging + +Backend should log: + +- request id if available +- project id when relevant +- dataset id when relevant +- analysis run id when relevant +- exception summary + +Do not log secrets or large file contents. diff --git a/docs/EXTERNAL_SERVICES_ADAPTERS.md b/docs/EXTERNAL_SERVICES_ADAPTERS.md new file mode 100644 index 00000000..65dc0d0e --- /dev/null +++ b/docs/EXTERNAL_SERVICES_ADAPTERS.md @@ -0,0 +1,9 @@ +# External Services Adapters + +Externe bronnen worden uitsluitend via adapterservices aangeroepen. + +Elke adapter heeft: name, capabilities, fetch(area, parameters), normalize(raw_data), to_dataset_record(), error mapping. + +V1 adapters: GRB en OSM. V2: Sentinel. V3: DHMV. + +Elke adapter krijgt fixture-backed testmodus zodat CI niet afhankelijk is van internet. diff --git a/docs/FIXTURE_STRATEGY.md b/docs/FIXTURE_STRATEGY.md new file mode 100644 index 00000000..414d9d52 --- /dev/null +++ b/docs/FIXTURE_STRATEGY.md @@ -0,0 +1,59 @@ +# Fixture Strategy + +## Purpose +GeoIntel needs small, repeatable fixtures so Codex can build and test without relying on huge external downloads. + +## Fixture principles +- Fixtures must be tiny. +- Fixtures must be deterministic. +- Fixtures must be documented. +- Fixtures must not pretend to be official data. +- Real-source providers can be added later without breaking tests. + +## Required V1 fixtures + +### reference_buildings.geojson +Purpose: +- reference building polygons for QA/QC tests. + +Content: +- 3 to 10 simple polygons. +- one polygon matching a prediction exactly; +- one partially overlapping; +- one unmatched reference object. + +### predicted_buildings.geojson +Purpose: +- deterministic detection output fixture. + +Content: +- one true positive exact/near match; +- one true positive partial match; +- one false positive; +- one low-confidence detection. + +### project_area.geojson +Purpose: +- test project area. + +Content: +- polygon enclosing all reference/predicted fixtures. + +### tiny_raster.tif +Purpose: +- raster metadata and clipping tests. + +Content: +- very small georeferenced raster if feasible. +- if not feasible initially, document missing fixture and provide tests for raster metadata with generated temporary raster. + +## Development providers + +### DevelopmentDetectionProvider +Returns predictions from `predicted_buildings.geojson` for a selected demo run. + +### LocalFixtureReferenceProvider +Returns `reference_buildings.geojson` as a cached reference layer. + +## Acceptance +The fixture strategy is complete when QA/QC tests can run fully offline. diff --git a/docs/FRONTEND_ROUTE_MAP.md b/docs/FRONTEND_ROUTE_MAP.md new file mode 100644 index 00000000..6f71476a --- /dev/null +++ b/docs/FRONTEND_ROUTE_MAP.md @@ -0,0 +1,32 @@ +# Frontend Route Map + +Target frontend routes: + +| Route | Page | Purpose | +|---|---|---| +| `/` | HomePage | Open recent projects or create demo project | +| `/projects` | ProjectsPage | List project workspaces | +| `/projects/:projectId` | ProjectWorkspacePage | Workspace overview | +| `/projects/:projectId/map` | MapWorkbenchPage | Draw/select areas and inspect layers | +| `/projects/:projectId/datasets` | DatasetManagerPage | Upload and manage datasets | +| `/projects/:projectId/raster` | RasterLabPage | Inspect raster datasets | +| `/projects/:projectId/vector` | VectorLabPage | Inspect vector datasets | +| `/projects/:projectId/detection` | DetectionLabPage | Start object detection runs | +| `/projects/:projectId/segmentation` | SegmentationLabPage | Start segmentation runs | +| `/projects/:projectId/qaqc` | QaQcPage | Validate AI outputs against reference layers | +| `/projects/:projectId/exports` | ExportsPage | Download GeoJSON/JSON outputs | +| `/settings` | SettingsPage | Source and runtime configuration | + +## Layout + +- Left navigation: persistent product navigation. +- Center content: page-specific workspace. +- Map pages: map canvas plus right inspection panel. +- Analysis pages: configuration panel, run status, result summary, map preview. + +## Frontend contracts + +- All server communication must go through `src/services/apiClient.ts`. +- Use typed DTOs in `src/types/api.ts`. +- Loading, empty and error states are mandatory. +- No page may depend on fake global state when an API exists. diff --git a/docs/FRONTEND_STATE_AND_API_CLIENT.md b/docs/FRONTEND_STATE_AND_API_CLIENT.md new file mode 100644 index 00000000..0ae21abb --- /dev/null +++ b/docs/FRONTEND_STATE_AND_API_CLIENT.md @@ -0,0 +1,80 @@ +# Frontend State and API Client Specification + +## API client + +Create a typed API client under: + +```text +frontend/src/services/api/ +``` + +Suggested files: + +- `client.ts`: fetch wrapper with base URL and error handling. +- `projects.ts` +- `areas.ts` +- `datasets.ts` +- `analysis.ts` +- `qa.ts` +- `exports.ts` + +## Error handling + +All API errors should be normalized to: + +```ts +type ApiError = { + error: string; + message: string; + details?: unknown; + request_id?: string; +}; +``` + +## State model + +Use TanStack Query for server state: + +- projects list +- project detail +- datasets list +- analysis runs +- QA results + +Use local state for UI only: + +- selected map layer +- layer opacity +- active tool +- drawn geometry draft +- side panel state + +## Required page states + +Every data-driven page must show: + +- Loading state. +- Empty state. +- Error state. +- Success state. + +## Map state + +Map state should not be embedded in every page component. Create reusable hooks/components: + +- `MapCanvas` +- `LayerPanel` +- `DrawAreaTool` +- `GeoJsonLayer` +- `RasterLayerPlaceholder` until real tile serving exists. + +## Frontend first-pass priority + +Do not build a beautiful but disconnected UI. The first UI must connect to real foundation endpoints: + +1. Health check. +2. Project list/create. +3. Project detail. +4. Area creation placeholder/drawn GeoJSON form. +5. Dataset list. +6. Capabilities panel. diff --git a/docs/FRONTEND_STATE_CONTRACTS.md b/docs/FRONTEND_STATE_CONTRACTS.md new file mode 100644 index 00000000..dab0f946 --- /dev/null +++ b/docs/FRONTEND_STATE_CONTRACTS.md @@ -0,0 +1,60 @@ +# Frontend State Contracts + +## Global API State +Every API-backed component must support: +- idle +- loading +- success +- empty +- error +- refreshing + +## Route State Pattern +Each route must render a usable state for: +- no project selected +- data loading +- data loaded +- API error +- permission/configuration warning + +## Map State +Map state includes: +- active project +- active area +- active base layer +- visible layers +- selected feature +- drawing mode +- measurement mode + +## Analysis State +Analysis state includes: +- selected dataset +- selected area +- selected model +- parameters +- current run +- run history +- output layers + +## Layer State +Layer object: +```ts +interface MapLayerState { + id: string; + name: string; + type: 'raster' | 'vector' | 'detections' | 'segmentations' | 'qaqc'; + visible: boolean; + opacity: number; + styleId?: string; + sourceId: string; +} +``` + +## Empty State Copy +Use precise empty states: +- Projects: "Create your first GeoIntel project to start an analysis." +- Areas: "Draw or import an area before running geospatial processing." +- Datasets: "Upload a raster/vector dataset or connect a data source." +- Detections: "Run object detection to create geospatial AI outputs." +- QA/QC: "Select an AI run and reference layer to calculate quality metrics." diff --git a/docs/GEOINTEL_STYLE_GUIDE.md b/docs/GEOINTEL_STYLE_GUIDE.md new file mode 100644 index 00000000..92ae77e6 --- /dev/null +++ b/docs/GEOINTEL_STYLE_GUIDE.md @@ -0,0 +1,47 @@ +# GeoIntel Style Guide + +## Visual direction + +GeoIntel should feel like a modern engineering workbench, not old desktop GIS software. + +## Map layer colors + +- Buildings: amber/orange. +- Roads: neutral gray. +- Water: blue. +- Vegetation: green. +- Detection boxes: yellow/orange. +- False positives: red. +- False negatives: purple. +- Matched detections: green. + +## UI principles + +- Analysis-first, map-supported. +- Always show status and provenance. +- No decorative clutter. +- Technical but readable. +- Every output should be exportable or inspectable. + +## Component style + +- Dense but not cramped. +- Clear empty states. +- Explicit loading states. +- Confidence/status badges. +- Sidebar navigation with workbench sections. + +## Terminology + +Use consistent module names: + +- Project Workspace +- Map Workbench +- Dataset Manager +- Raster Lab +- Vector Lab +- Detection Lab +- Segmentation Lab +- QA/QC Lab +- Change Lab +- Exports diff --git a/docs/GEOSPATIAL_VALIDATION_RULES.md b/docs/GEOSPATIAL_VALIDATION_RULES.md new file mode 100644 index 00000000..b5a10afd --- /dev/null +++ b/docs/GEOSPATIAL_VALIDATION_RULES.md @@ -0,0 +1,13 @@ +# Geospatial Validation Rules + +## Vector +Geometry bestaat, is geldig of herstelbaar, niet leeg, type klopt, CRS bekend, bounds overlappen projectarea indien verwacht. + +## CRS +Frontend GeoJSON wordt WGS84. Metrische berekeningen gebeuren niet in WGS84 maar in geschikte projectie. + +## Raster +CRS, transform, bounds, band count, nodata, resolutie en leesbaarheid door Rasterio zijn verplicht. + +## Analysis startvoorwaarden +Input datasets bestaan, status ready, area geldig, CRS-transformaties mogelijk, output path beschikbaar. diff --git a/docs/HEALTHCHECK_CONTRACTS.md b/docs/HEALTHCHECK_CONTRACTS.md new file mode 100644 index 00000000..a9baa5d6 --- /dev/null +++ b/docs/HEALTHCHECK_CONTRACTS.md @@ -0,0 +1,57 @@ +# Healthcheck Contracts + +## Doel +Elke runtime-component moet een eenvoudige en machineleesbare healthcheck hebben. + +## Backend + +### Endpoint +`GET /health` + +### Response 200 +```json +{ + "status": "ok", + "service": "geointel-backend", + "version": "0.1.0", + "database": "ok", + "redis": "ok", + "storage": "ok" +} +``` + +### Response 503 +```json +{ + "status": "degraded", + "service": "geointel-backend", + "database": "unavailable", + "redis": "ok", + "storage": "ok", + "errors": ["database connection failed"] +} +``` + +## Worker + +### Endpoint of command +Codex mag kiezen tussen een internal endpoint of CLI-command, maar het contract moet dit opleveren: + +```json +{ + "status": "ok", + "queue": "default", + "pending_jobs": 0, + "failed_jobs": 0, + "last_heartbeat": "ISO-8601" +} +``` + +## Frontend +Frontend moet een `/status` of settings/statuspanel hebben dat toont: backend bereikbaar, API version, auth status, feature flags en laatste healthchecktijd. + +## Storage +Storage healthcheck controleert of upload, originals, derived en exports writable zijn. + +## Database +Database healthcheck controleert connectie, PostGIS extension en migrationstatus. diff --git a/docs/IMPLEMENTATION_BACKLOG.md b/docs/IMPLEMENTATION_BACKLOG.md new file mode 100644 index 00000000..be757834 --- /dev/null +++ b/docs/IMPLEMENTATION_BACKLOG.md @@ -0,0 +1,128 @@ +# GeoIntel Kempen — Implementation Backlog v1.0 + +## Milestone 0 — Repo readiness + +- [ ] Ensure docs are complete and consistent. +- [ ] Add README quickstart. +- [ ] Add docker-compose with PostGIS, Redis, backend, frontend placeholders. +- [ ] Add `.env.example`. +- [ ] Add development rules. + +## Milestone 1 — Backend foundation + +- [ ] FastAPI app skeleton. +- [ ] Settings/config module. +- [ ] SQLAlchemy or SQLModel setup. +- [ ] Alembic migrations. +- [ ] PostGIS extension migration. +- [ ] Health endpoint. +- [ ] Job/analysis status model. +- [ ] File storage service. + +## Milestone 2 — Core data model + +- [ ] Projects table/model/API. +- [ ] Areas table/model/API with geometry. +- [ ] Datasets table/model/API. +- [ ] Layers table/model/API. +- [ ] Analysis runs table/model/API. +- [ ] Metrics table/model/API. +- [ ] Exports table/model/API. + +## Milestone 3 — Dataset Manager + +- [ ] Raster upload. +- [ ] Vector upload. +- [ ] Shapefile ZIP support. +- [ ] Dataset metadata extraction. +- [ ] Dataset list API. +- [ ] Dataset detail API. +- [ ] Basic preview generation. + +## Milestone 4 — Raster Lab + +- [ ] Raster metadata endpoint. +- [ ] Band statistics. +- [ ] Raster clipping. +- [ ] Tile generation. +- [ ] Preview rendering. +- [ ] Raster operation tests. + +## Milestone 5 — Vector Lab + +- [ ] Vector metadata endpoint. +- [ ] Reprojection utility. +- [ ] Clip operation. +- [ ] Buffer operation. +- [ ] Intersect operation. +- [ ] Difference operation. +- [ ] Spatial join operation. +- [ ] Vector operation tests. + +## Milestone 6 — Reference data providers + +- [ ] ReferenceDataProvider interface. +- [ ] OSM provider fallback. +- [ ] GRB provider architecture. +- [ ] GRB WFS capability discovery task. +- [ ] Cache imported reference features in PostGIS. + +## Milestone 7 — Detection Pipeline + +- [ ] Model registry table. +- [ ] YOLO model adapter. +- [ ] Tile inference job. +- [ ] Pixel bbox to geospatial polygon conversion. +- [ ] Duplicate removal/NMS. +- [ ] Detection storage. +- [ ] Detection GeoJSON export. + +## Milestone 8 — QA/QC Engine + +- [ ] IoU utility. +- [ ] One-to-one matching. +- [ ] TP/FP/FN classification. +- [ ] Precision/recall/F1 metrics. +- [ ] QA layer outputs. +- [ ] QA GeoJSON/CSV export. + +## Milestone 9 — Frontend foundation + +- [ ] React/Vite/TypeScript app. +- [ ] API client. +- [ ] Layout/navigation. +- [ ] Project Workspace. +- [ ] Dataset Manager UI. +- [ ] Map Workbench with MapLibre. + +## Milestone 10 — Frontend labs + +- [ ] Raster Lab UI. +- [ ] Vector Lab UI. +- [ ] Detection Lab UI. +- [ ] QA/QC Lab UI. +- [ ] Exports UI. +- [ ] Job status UI. + +## Milestone 11 — Segmentation + +- [ ] Segmentation data model. +- [ ] Mask storage. +- [ ] Polygonization utility. +- [ ] Segmentation Lab UI shell. +- [ ] Optional SAM/YOLO-seg integration. + +## Milestone 12 — Change Detection + +- [ ] Vector change detection. +- [ ] Added/removed/modified layers. +- [ ] Change metrics. +- [ ] Change Lab UI. + +## Milestone 13 — Portfolio polish + +- [ ] Demo scenario setup. +- [ ] Seed demo project. +- [ ] README screenshots placeholders. +- [ ] Architecture diagram. +- [ ] Portfolio explanation page. diff --git a/docs/IMPLEMENTATION_EPICS.md b/docs/IMPLEMENTATION_EPICS.md new file mode 100644 index 00000000..7f6e67c9 --- /dev/null +++ b/docs/IMPLEMENTATION_EPICS.md @@ -0,0 +1,130 @@ +# Implementation Epics + +## EPIC 01 — Repository and Runtime Foundation + +Goal: create a runnable full-stack foundation with backend, frontend, database, worker and shared conventions. + +Must deliver: + +- FastAPI app booting +- health endpoints +- SQLAlchemy/Alembic setup +- PostGIS-enabled PostgreSQL connection +- React app booting +- Docker Compose local stack +- basic CI-friendly test commands + +## EPIC 02 — Project and Area Management + +Goal: allow users to create project workspaces and define Kempen analysis areas. + +Must deliver: + +- project CRUD +- area CRUD +- polygon validation +- area surface calculation +- geometry storage in PostGIS +- frontend project dashboard +- map drawing workflow + +## EPIC 03 — Dataset Manager + +Goal: upload, register and inspect geospatial datasets. + +Must deliver: + +- dataset upload endpoint +- metadata extraction +- storage path registration +- dataset table +- dataset detail page +- processing status model + +## EPIC 04 — Raster Foundation + +Goal: support GeoTIFF metadata, preview, clipping and tiling readiness. + +Must deliver: + +- Rasterio metadata extraction +- bounds, CRS, band count, resolution +- clip request contract +- tile generation contract +- raster preview placeholder backed by real metadata + +## EPIC 05 — Vector Foundation + +Goal: support GeoJSON/Shapefile/GPKG import and core vector operations. + +Must deliver: + +- GeoPandas import +- CRS transform +- clip to area +- area/perimeter calculation +- vector layer display/export + +## EPIC 06 — GRB Reference Integration + +Goal: retrieve GRB features for selected areas and cache them in PostGIS. + +Must deliver: + +- GRB source config +- WFS request service +- cache table or dataset registration +- building reference layer +- failure handling and source metadata + +## EPIC 07 — Detection Pipeline Skeleton + +Goal: run object detection over prepared raster tiles and store georeferenced outputs. + +Must deliver: + +- model registry stub +- detection job request +- tile inference interface +- detections table +- GeoJSON export +- map overlay + +The first implementation may use a deterministic fixture model only if the full interface is identical to a future YOLO implementation. + +## EPIC 08 — QA/QC Engine + +Goal: compare detections against GRB reference features. + +Must deliver: + +- spatial matching +- IoU calculation +- precision/recall/F1 +- false positives/false negatives +- QA result layer +- metrics panel + +## EPIC 09 — Export Layer + +Goal: export useful outputs. + +Must deliver: + +- GeoJSON export +- analysis metadata export +- QA summary JSON +- future-compatible report registry + +## EPIC 10 — Stabilization and Polish + +Goal: make the MVP coherent, testable and demo-ready. + +Must deliver: + +- error states +- empty states +- loading states +- validation messages +- test coverage on critical services +- changelog and release notes diff --git a/docs/IMPLEMENTATION_GAP_REPORT.md b/docs/IMPLEMENTATION_GAP_REPORT.md new file mode 100644 index 00000000..a845bb5e --- /dev/null +++ b/docs/IMPLEMENTATION_GAP_REPORT.md @@ -0,0 +1,38 @@ +# Implementation Gap Report + +This file is intended for Codex Pass 0 and subsequent audits. + +## Current Implementation State + +To be filled by Codex after repository audit. + +## Missing P0 Items + +- [ ] Backend foundation implementation. +- [ ] Database models and migrations. +- [ ] API envelope implementation. +- [ ] Project/Area endpoints. +- [ ] Dataset upload skeleton. +- [ ] Frontend foundation. + +## Missing P1 Items + +- [ ] Map workbench shell. +- [ ] Raster/vector metadata service skeletons. +- [ ] Demo fixture loader. +- [ ] QA/QC geometry utilities. + +## Missing P2 Items + +- [ ] Expanded tests. +- [ ] CI hardening. +- [ ] UI polish. +- [ ] Advanced validation copy. + +## Blockers + +To be filled by Codex. + +## Next Pass + +To be filled by Codex. diff --git a/docs/JOB_LIFECYCLE.md b/docs/JOB_LIFECYCLE.md new file mode 100644 index 00000000..36e957da --- /dev/null +++ b/docs/JOB_LIFECYCLE.md @@ -0,0 +1,67 @@ +# Job Lifecycle + +Long-running work must run through jobs. + +## Job statuses + +```text +queued +running +succeeded +failed +cancelled +``` + +## Job table fields + +- id +- job_type +- status +- project_id +- dataset_id nullable +- analysis_run_id nullable +- progress_percent nullable +- message nullable +- error_code nullable +- error_detail nullable +- created_at +- started_at +- finished_at + +## Job types + +- dataset_metadata_extract +- raster_clip +- raster_tile +- vector_import +- vector_clip +- grb_fetch +- object_detection +- segmentation +- qaqc_compare +- export_generate + +## Job events + +Each status transition should create a domain event. + +## API behavior + +Starting a long-running operation returns immediately with: + +```json +{ + "data": { + "job_id": "...", + "status": "queued" + }, + "error": null, + "meta": {} +} +``` + +The frontend polls a job endpoint until success or failure. + +## Failure handling + +Jobs must fail loudly and clearly. Do not silently skip processing. diff --git a/docs/JOB_LIFECYCLE_CONTRACT.md b/docs/JOB_LIFECYCLE_CONTRACT.md new file mode 100644 index 00000000..947dde6b --- /dev/null +++ b/docs/JOB_LIFECYCLE_CONTRACT.md @@ -0,0 +1,64 @@ +# Job Lifecycle Contract + +GeoIntel uses background jobs for long-running geospatial and AI tasks. + +## Status Values +- `queued`: job record exists but worker has not started. +- `running`: worker is processing. +- `completed`: output persisted and metrics available. +- `failed`: processing stopped with a recoverable or unrecoverable error. +- `cancelled`: user requested cancellation before completion. + +## Common Job Fields +- id +- project_id +- job_type +- status +- progress_percent +- current_step +- parameters_json +- result_json +- error_json +- created_at +- started_at +- finished_at + +## Job Types +- raster_metadata +- raster_clip +- raster_tile +- vector_metadata +- vector_clip +- object_detection +- segmentation +- qaqc +- export_geojson +- export_report + +## Progress Steps +Object detection should use these steps: +1. validate_input +2. prepare_tiles +3. load_model +4. run_inference +5. georeference_outputs +6. persist_results +7. calculate_metrics +8. completed + +## Error Contract +A failed job must store: +```json +{ + "code": "RASTER_CRS_MISSING", + "message": "Raster CRS could not be determined.", + "recoverable": true, + "step": "validate_input" +} +``` + +## Frontend Behavior +- queued: show pending state. +- running: show current step and progress. +- completed: show outputs and actions. +- failed: show error, details and retry where safe. diff --git a/docs/KNOWN_LIMITATIONS_M3.md b/docs/KNOWN_LIMITATIONS_M3.md new file mode 100644 index 00000000..a77f2f11 --- /dev/null +++ b/docs/KNOWN_LIMITATIONS_M3.md @@ -0,0 +1,27 @@ +# Known Limitations at M3 + +These limitations are intentional and must not be treated as bugs during the first build. + +## Real-time global data ingestion is out of scope + +V1 focuses on selected Kempen areas. + +## LiDAR is out of scope for V1 + +The architecture reserves space for PDAL/laspy, but V1 should not implement LAS/LAZ processing. + +## Full model training is out of scope for V1 + +V1 may include model registry and inference interfaces. Training Studio is a later phase. + +## Sentinel is not V1-critical + +Remote sensing indices are valuable, but the first recruiter-focused demo should prioritize building detection, GRB validation and QA/QC. + +## Fixture detector is allowed only as a bridge + +A deterministic fixture detector is acceptable to validate the pipeline before YOLO is wired in. It must not be marketed as AI output. + +## Reports are secondary + +Exports are required. Rich PDF reporting is a later enhancement. diff --git a/docs/LOCAL_DEVELOPMENT_RUNBOOK.md b/docs/LOCAL_DEVELOPMENT_RUNBOOK.md new file mode 100644 index 00000000..b3ecb367 --- /dev/null +++ b/docs/LOCAL_DEVELOPMENT_RUNBOOK.md @@ -0,0 +1,89 @@ +# Local Development Runbook + +## Required tools + +- Docker Desktop or compatible Docker runtime +- Node.js LTS +- Python 3.11+ +- Git + +## Start infrastructure + +```bash +docker compose up -d db +``` + +## Backend setup + +```bash +cd backend +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +alembic upgrade head +uvicorn app.main:app --reload +``` + +On Windows PowerShell: + +```powershell +cd backend +py -m venv .venv +.\.venv\Scripts\Activate.ps1 +pip install -r requirements.txt +alembic upgrade head +uvicorn app.main:app --reload +``` + +## Frontend setup + +```bash +cd frontend +npm install +npm run dev +``` + +## Worker setup + +```bash +cd backend +rq worker geointel +``` + +## Smoke checks + +- Backend health: `GET http://localhost:8000/health` +- Frontend with Docker Compose: `http://localhost:1202` +- Frontend with local Vite dev server: `http://localhost:5173` +- Database: PostGIS runs on the Docker network as `db:5432`; it is not published on host port `5432` by default. +- Redis: worker connects + +## Common failure modes + +### PostGIS extension missing + +Ensure the Docker image is `postgis/postgis`, not plain PostgreSQL. + +### Rasterio install fails on Windows + +Prefer Docker backend runtime or conda/mamba environment. Do not remove Rasterio from the architecture. + +### Frontend cannot reach backend + +For Docker Compose, rebuild/restart the frontend and verify the Vite proxy: + +```bash +docker compose build --no-cache frontend backend +docker compose up -d +bash scripts/verify_browser_runtime.sh http://localhost:1202 http://localhost:8000/health +``` + +`GET /api/v1/projects` on the frontend origin must return the backend JSON +envelope. If it returns ``, the frontend container is stale or +the nginx `/api` proxy config is not active. For local non-Docker development, +`VITE_API_BASE_URL` may point directly at `http://localhost:8000`, but the +default is same-origin plus a local Vite proxy. + +### Worker jobs stay pending + +Check Redis URL and queue name. diff --git a/docs/M0_HANDOFF_SUMMARY.md b/docs/M0_HANDOFF_SUMMARY.md new file mode 100644 index 00000000..bb93cb01 --- /dev/null +++ b/docs/M0_HANDOFF_SUMMARY.md @@ -0,0 +1,24 @@ +# M0 Handoff Summary + +## What this repo is prepared for +This repository is prepared for a backend-first Codex build of GeoIntel Kempen: a GeoAI Workbench focused on raster/vector processing, object detection, segmentation, GRB/reference QA and GIS exports. + +## Read first +Codex should read these documents before coding: +1. README.md +2. docs/SPECIFICATION_FREEZE_M0.md +3. docs/V1_SCOPE_FREEZE.md +4. docs/DATASET_STRATEGY.md +5. docs/SERVICE_ARCHITECTURE.md +6. docs/REPOSITORY_CONVENTIONS.md +7. docs/CODEX_EXECUTION_PLAN.md +8. docs/ACCEPTANCE_CRITERIA.md + +## Build target +The first meaningful target is Demo 1: Building Detection + GRB/reference QA. + +## First implementation pass +Start with Pass 1 in `docs/CODEX_EXECUTION_PLAN.md`. + +## Important mindset +This is not a dashboard-first app. It is an engineering workbench. A plain UI with correct GIS pipelines is better than a polished UI with fake outputs. diff --git a/docs/M1_HANDOFF_SUMMARY.md b/docs/M1_HANDOFF_SUMMARY.md new file mode 100644 index 00000000..96f65262 --- /dev/null +++ b/docs/M1_HANDOFF_SUMMARY.md @@ -0,0 +1,25 @@ +# M1 Handoff Summary — Codex-Ready Extension + +This package extends M0 with implementation contracts and guardrails. + +## Added focus + +- API contracts. +- Database implementation details. +- Test strategy. +- Definition of Done. +- Dependency policy. +- Environment specification. +- Frontend state/API client rules. +- Error/status contract. +- Security/data boundaries. +- Performance budgets. +- Additional Codex phase prompts. + +## Intended next action + +Start Codex with `docs/CODEX_BOOTSTRAP_PROMPT.md`, then proceed through `docs/CODEX_EXECUTION_PLAN.md` and the phase prompts. + +## Key instruction + +Do not expand scope. Build the foundation and the first detection + QA workflow before adding LiDAR, model training, advanced reports, or QGIS integration. diff --git a/docs/M2_ENGINEERING_PACKAGE.md b/docs/M2_ENGINEERING_PACKAGE.md new file mode 100644 index 00000000..62323529 --- /dev/null +++ b/docs/M2_ENGINEERING_PACKAGE.md @@ -0,0 +1,46 @@ +# GeoIntel Kempen — M2 Engineering Package + +This milestone turns the M1 Codex-ready blueprint into an engineering-grade repository specification. M2 adds decision records, explicit domain boundaries, event contracts, model registry rules, API response contracts, test matrices, and release conventions. + +## Purpose + +Codex should not invent architecture. It should implement the architecture defined in this repository. + +M2 makes the following decisions explicit: + +- FastAPI is the backend application framework. +- PostgreSQL + PostGIS is the source of truth for relational and spatial data. +- Redis + RQ is the V1 background job mechanism. +- Local filesystem storage is the V1 object storage layer, with a clear path to MinIO/S3 later. +- GRB is the primary authoritative reference source for Flemish building and base-map validation. +- OSM is useful as supplemental and fallback context, not as the primary QA reference when GRB is available. +- YOLO/Ultralytics is the V1 object detection runtime. +- SAM or YOLO-seg is the segmentation direction, but segmentation can be introduced after the detection foundation is stable. +- Reports, chat/copilot, LiDAR, MLOps, and QGIS plugins remain secondary to the GeoAI pipeline. + +## Milestone status + +M0 froze the product direction. +M1 made the repo Codex-ready. +M2 makes the repo engineering-ready. + +## What Codex should build first + +1. Backend foundation. +2. Database models and migrations. +3. Dataset intake and metadata extraction. +4. Area/project management. +5. Raster/vector processing contracts. +6. Object detection job skeleton. +7. QA/QC calculation skeleton. +8. Frontend workspace wired to real API contracts. + +## What Codex must not build yet + +- Full LiDAR point cloud processing. +- Custom model training. +- Multi-user authentication. +- QGIS plugin. +- Production cloud deployment. +- Complex report designer. +- Large demo datasets committed into git. diff --git a/docs/M3_HANDOFF_SUMMARY.md b/docs/M3_HANDOFF_SUMMARY.md new file mode 100644 index 00000000..a8f85f68 --- /dev/null +++ b/docs/M3_HANDOFF_SUMMARY.md @@ -0,0 +1,42 @@ +# M3 Handoff Summary + +M3 prepares the repository for the first implementation sprint. + +## Added in M3 + +- implementation epics +- ticket-level build plan +- migration plan +- seed data plan +- local development runbook +- backend package map +- frontend route map +- module contracts +- job lifecycle +- Codex pass matrix +- Codex prompts for passes 05 through 12 +- known limitations + +## Recommended next action + +Start Codex with: + +1. `prompts/codex/M2_MASTER_BUILD_PROMPT.md` +2. `docs/CODEX_PASS_MATRIX_M3.md` +3. `prompts/codex/PASS_01_BACKEND_FOUNDATION.md` + +Then continue pass-by-pass. + +## Most important build target + +A complete V0.1 should prove: + +- project creation +- area creation +- dataset upload +- raster/vector metadata +- fixture detection +- QA/QC against reference buildings +- GeoJSON export + +This is enough to demonstrate the core GeoAI workflow before heavy AI integrations are added. diff --git a/docs/M3_IMPLEMENTATION_READINESS.md b/docs/M3_IMPLEMENTATION_READINESS.md new file mode 100644 index 00000000..511e7c4a --- /dev/null +++ b/docs/M3_IMPLEMENTATION_READINESS.md @@ -0,0 +1,54 @@ +# M3 Implementation Readiness Package + +M3 turns the M2 engineering blueprint into an implementation-ready repository handoff for Codex. + +## Purpose + +Codex must be able to start building without inventing architecture, naming, workflow, or scope. + +M3 adds: + +- implementation epics +- build tickets +- migration plan +- seed data plan +- test-first checklist +- local development runbook +- module contracts +- frontend route map +- backend package map +- worker/job lifecycle +- Codex pass prompts for the first real build rounds + +## Non-negotiable rule + +When a document in M3 conflicts with an older document, M3 wins unless explicitly marked as exploratory. + +## Build philosophy + +GeoIntel must be built backend-first and contract-first: + +1. database schema +2. backend service contracts +3. API responses +4. tests +5. frontend integration +6. GIS/AI processing implementations + +Do not build UI-only mock features that are not backed by API contracts. + +## Implementation readiness status + +| Area | Status | Notes | +|---|---:|---| +| Product direction | Frozen | GeoAI Workbench for Kempen | +| V1 scope | Frozen | Dataset, raster/vector, detection, QA/QC | +| Datastore | Frozen | PostgreSQL + PostGIS | +| Backend | Frozen | FastAPI | +| Frontend | Frozen | React + TypeScript + MapLibre | +| Jobs | Frozen | Redis + RQ | +| GRB strategy | Frozen for V1 | WFS-first with PostGIS cache | +| OSM strategy | Frozen for V1 | Overpass/OSMnx optional fallback | +| Sentinel | Deferred | V2/RFC | +| LiDAR | Deferred | V4/RFC | +| Training Studio | Deferred | V3/RFC | diff --git a/docs/M4_AUTONOMOUS_BUILD_READINESS.md b/docs/M4_AUTONOMOUS_BUILD_READINESS.md new file mode 100644 index 00000000..dd2c942f --- /dev/null +++ b/docs/M4_AUTONOMOUS_BUILD_READINESS.md @@ -0,0 +1,43 @@ +# M4 Autonomous Build Readiness + +This milestone converts GeoIntel from an implementation-ready blueprint into an autonomous-build package for Codex. + +## Goal +Codex must be able to start from the repository root, read the contracts, build the foundation, and keep moving through backend, frontend, GIS processing, AI pipeline stubs, QA/QC, tests and documentation without making new product or architecture decisions. + +## M4 Adds +- Sprint board with sequential implementation lanes. +- Module-level build contracts. +- Acceptance-test catalog per module. +- Frontend state and page contracts. +- Backend service IO contracts. +- Example API responses and error shapes. +- Job lifecycle contract. +- Model registry seed contract. +- Demo fixture manifest. +- Codex prompts per autonomous build pass. +- Final readiness checklist. + +## Golden Rule +When code conflicts with documentation, documentation wins unless an explicit ADR is added. + +## Build Priority +1. Running local stack. +2. Database models and migrations. +3. API contracts. +4. Dataset manager. +5. Map workspace. +6. Raster/vector metadata flows. +7. Analysis run lifecycle. +8. Detection/segmentation job shells with deterministic demo mode. +9. QA/QC engine. +10. Exports. +11. Polish, tests and documentation. + +## Do Not Build Yet +- Full LiDAR workbench. +- Full training studio. +- QGIS plugin. +- Cloud deployment. +- Real production-grade model training. +- Complex user auth. diff --git a/docs/M5_OPERATIONAL_READINESS.md b/docs/M5_OPERATIONAL_READINESS.md new file mode 100644 index 00000000..0d065d7a --- /dev/null +++ b/docs/M5_OPERATIONAL_READINESS.md @@ -0,0 +1,37 @@ +# M5 Operational Readiness Package + +## Doel +M5 maakt de GeoIntel-repo klaar voor een lange autonome Codex-bouwronde. De focus verschuift van specificatie naar uitvoerbaarheid, controleerbaarheid en herstelbaarheid. + +## Wat M5 toevoegt +- Runbooks voor lokale ontwikkeling, smoke tests, troubleshooting en releasecontrole. +- CI/CD-specificaties zodat Codex weet welke kwaliteitschecks verplicht zijn. +- Healthcheck- en observability-contracten voor backend, worker, database en frontend. +- Incident- en rollback-procedures voor kapotte builds. +- Fixture- en demo-validatie zodat modules zonder echte externe datasets testbaar zijn. +- Build governance: hoe Codex per pass werkt, rapporteert, test en afrondt. + +## M5-principe +Elke bouwronde moet eindigen met drie dingen: + +1. Een werkende applicatiestatus of een eerlijk gedocumenteerde blokkade. +2. Een testbaar bewijs: commando-output, screenshots, API-responses of logs. +3. Een bijgewerkte statuslijst met voltooid, gedeeltelijk voltooid en openstaand werk. + +## Niet-toegestaan +- Nieuwe grote features toevoegen zonder bestaande M0-M4-contracten te volgen. +- Externe afhankelijkheden toevoegen zonder `docs/DEPENDENCY_POLICY.md` en `docs/DEPENDENCY_LOCK_PLAN.md` bij te werken. +- Mock-resultaten presenteren als echte geospatiale verwerking. +- API-contracten breken zonder migration-notitie. +- Frontend-only schijnfunctionaliteit bouwen wanneer een backendcontract bestaat. + +## M5 status +Na M5 moet Codex kunnen starten met: + +- Backend foundation. +- PostGIS schema. +- Dataset Manager. +- Raster metadata en upload. +- Vector import en clipping. +- Eerste QA/QC skeleton. +- Frontend workbench shell. diff --git a/docs/M6_ARTIFACT_MANIFEST.md b/docs/M6_ARTIFACT_MANIFEST.md new file mode 100644 index 00000000..1aab727b --- /dev/null +++ b/docs/M6_ARTIFACT_MANIFEST.md @@ -0,0 +1,281 @@ +# M6 Artifact Manifest + +- `.env.example` +- `.github/ISSUE_TEMPLATE/bug_report.md` +- `.github/ISSUE_TEMPLATE/feature_request.md` +- `.github/pull_request_template.md` +- `.github/workflows/ci.yml` +- `.gitignore` +- `.gitkeep` +- `AGENTS.md` +- `CHANGELOG.md` +- `M5_UPDATE_MANIFEST.txt` +- `README.md` +- `RELEASE_NOTES/v0.0-M2.md` +- `RELEASE_NOTES/v0.0-M3.md` +- `RELEASE_NOTES/v0.4-m4-autonomous-build-readiness.md` +- `RELEASE_NOTES/v0.5-m5-operational-readiness.md` +- `adr/ADR-001-technology-stack.md` +- `adr/ADR-002-postgis-choice.md` +- `adr/ADR-003-grb-strategy.md` +- `adr/ADR-004-storage-strategy.md` +- `adr/ADR-005-ai-model-strategy.md` +- `adr/ADR-006-job-processing.md` +- `adr/ADR-007-api-design.md` +- `backend/.gitkeep` +- `backend/README.md` +- `backend/app/.gitkeep` +- `backend/app/ai/.gitkeep` +- `backend/app/analysis/.gitkeep` +- `backend/app/api/.gitkeep` +- `backend/app/api/routes/.gitkeep` +- `backend/app/core/.gitkeep` +- `backend/app/db/.gitkeep` +- `backend/app/geo/.gitkeep` +- `backend/app/models/.gitkeep` +- `backend/app/providers/.gitkeep` +- `backend/app/repositories/.gitkeep` +- `backend/app/schemas/.gitkeep` +- `backend/app/services/.gitkeep` +- `backend/app/storage/.gitkeep` +- `backend/app/utils/.gitkeep` +- `backend/app/workers/.gitkeep` +- `backend/tests/.gitkeep` +- `contracts/api/response-envelope.md` +- `contracts/database/domain-model.md` +- `contracts/events/event-contracts.md` +- `datasets/.gitkeep` +- `datasets/cache/.gitkeep` +- `datasets/processed/.gitkeep` +- `datasets/raw/.gitkeep` +- `demo/geel/README.md` +- `demo/geel/area_geel_center.geojson` +- `demo/geel/demo_detections.geojson` +- `demo/geel/expected_qaqc_metrics.json` +- `demo/geel/reference_buildings.geojson` +- `demo/mol/README.md` +- `demo/turnhout/README.md` +- `docker-compose.yml` +- `docs/.gitkeep` +- `docs/ACCEPTANCE_CRITERIA.md` +- `docs/ACCEPTANCE_MATRIX.md` +- `docs/ACCEPTANCE_TEST_CATALOG.md` +- `docs/AI_PIPELINES.md` +- `docs/ANALYSIS_ENGINE.md` +- `docs/ANALYSIS_SPECIFICATIONS.md` +- `docs/API_CONTRACTS.md` +- `docs/API_CONTRACT_FREEZE_M2.md` +- `docs/API_EXAMPLE_RESPONSES.md` +- `docs/API_SPECIFICATION.md` +- `docs/ARCHITECTURE.md` +- `docs/BACKEND_PACKAGE_MAP.md` +- `docs/BUILD_GOVERNANCE.md` +- `docs/BUILD_TICKETS_M3.md` +- `docs/CHANGELOG_M4.md` +- `docs/CHANGE_DETECTION_SPEC.md` +- `docs/CI_CD_SPECIFICATION.md` +- `docs/CODEX_AUTONOMOUS_RUNBOOK_M4.md` +- `docs/CODEX_BOOTSTRAP_PROMPT.md` +- `docs/CODEX_BUILD_PLAN.md` +- `docs/CODEX_EXECUTION_LOG.md` +- `docs/CODEX_EXECUTION_PLAN.md` +- `docs/CODEX_MASTER_PROMPT.md` +- `docs/CODEX_PASS_0_REPO_AUDIT.md` +- `docs/CODEX_PASS_1_BACKEND_FOUNDATION.md` +- `docs/CODEX_PASS_2_DATABASE_AND_MODELS.md` +- `docs/CODEX_PASS_3_DATASET_MANAGER.md` +- `docs/CODEX_PASS_4_RASTER_VECTOR_CORE.md` +- `docs/CODEX_PASS_5_FRONTEND_WORKBENCH_SHELL.md` +- `docs/CODEX_PASS_6_DETECTION_QA_SKELETON.md` +- `docs/CODEX_PASS_MATRIX_M3.md` +- `docs/CODEX_PHASE_1_PROMPT.md` +- `docs/CODEX_PHASE_2_PROMPT.md` +- `docs/CODEX_PHASE_3_PROMPT.md` +- `docs/CODEX_PHASE_4_PROMPT.md` +- `docs/CODEX_PHASE_5_PROMPT.md` +- `docs/CODEX_PHASE_6_PROMPT.md` +- `docs/CODEX_PHASE_7_PROMPT.md` +- `docs/CODEX_PHASE_8_PROMPT.md` +- `docs/CODEX_PROMPT_M5_LONG_AUTONOMOUS_BUILD.md` +- `docs/COMPONENT_BREAKDOWN.md` +- `docs/DATABASE_IMPLEMENTATION_PLAN.md` +- `docs/DATABASE_SCHEMA.md` +- `docs/DATASET_STRATEGY.md` +- `docs/DATA_CATALOG.md` +- `docs/DATA_PRIVACY_AND_LICENSING.md` +- `docs/DATA_SOURCES.md` +- `docs/DATA_SPECIFICATION.md` +- `docs/DEFINITION_OF_DONE.md` +- `docs/DEFINITION_OF_READY.md` +- `docs/DEMO_FIXTURE_MANIFEST.md` +- `docs/DEMO_SCENARIOS.md` +- `docs/DEMO_USE_CASES.md` +- `docs/DEPENDENCY_LOCK_PLAN.md` +- `docs/DEPENDENCY_POLICY.md` +- `docs/DESIGN_SYSTEM.md` +- `docs/DETECTION_CLASSES_CATALOG.md` +- `docs/DETECTION_PIPELINE_SPEC.md` +- `docs/DEVELOPMENT_RULES.md` +- `docs/DOMAIN_MODEL.md` +- `docs/ENVIRONMENT_SPEC.md` +- `docs/ERROR_HANDLING_AND_STATUSES.md` +- `docs/EXTERNAL_SERVICES_ADAPTERS.md` +- `docs/FIXTURE_STRATEGY.md` +- `docs/FRONTEND_ROUTE_MAP.md` +- `docs/FRONTEND_STATE_AND_API_CLIENT.md` +- `docs/FRONTEND_STATE_CONTRACTS.md` +- `docs/GEOINTEL_STYLE_GUIDE.md` +- `docs/GEOSPATIAL_VALIDATION_RULES.md` +- `docs/HEALTHCHECK_CONTRACTS.md` +- `docs/IMPLEMENTATION_BACKLOG.md` +- `docs/IMPLEMENTATION_EPICS.md` +- `docs/JOB_LIFECYCLE.md` +- `docs/JOB_LIFECYCLE_CONTRACT.md` +- `docs/KNOWN_LIMITATIONS_M3.md` +- `docs/LOCAL_DEVELOPMENT_RUNBOOK.md` +- `docs/M0_HANDOFF_SUMMARY.md` +- `docs/M1_HANDOFF_SUMMARY.md` +- `docs/M2_ENGINEERING_PACKAGE.md` +- `docs/M3_HANDOFF_SUMMARY.md` +- `docs/M3_IMPLEMENTATION_READINESS.md` +- `docs/M4_AUTONOMOUS_BUILD_READINESS.md` +- `docs/M5_OPERATIONAL_READINESS.md` +- `docs/M6_AUTONOMY_BOUNDARIES.md` +- `docs/M6_CODEX_AUTONOMY_PACK.md` +- `docs/M6_FAILURE_RECOVERY_PLAYBOOK.md` +- `docs/M6_FINAL_HANDOFF_TEMPLATE.md` +- `docs/M6_GAP_REGISTRY.md` +- `docs/M6_HANDOFF_SUMMARY.md` +- `docs/M6_NEXT_DAY_EXECUTION_CHECKLIST.md` +- `docs/M6_QUALITY_GATES.md` +- `docs/M6_SELF_REVIEW_CHECKLIST.md` +- `docs/MIGRATION_PLAN.md` +- `docs/MODEL_REGISTRY_SEED.md` +- `docs/MODEL_REGISTRY_SPEC.md` +- `docs/MODULES.md` +- `docs/MODULE_BUILD_CONTRACTS.md` +- `docs/MODULE_CONTRACTS.md` +- `docs/OBSERVABILITY_PLAN.md` +- `docs/PERFORMANCE_BUDGETS.md` +- `docs/PRODUCT_BLUEPRINT.md` +- `docs/PRODUCT_VISION.md` +- `docs/QA_QC_ENGINE.md` +- `docs/QA_QC_SPECIFICATION.md` +- `docs/QUEUE_ARCHITECTURE.md` +- `docs/RASTER_OPERATIONS_SPEC.md` +- `docs/README.md` +- `docs/RELEASE_PROCESS.md` +- `docs/REPOSITORY_CONVENTIONS.md` +- `docs/ROADMAP.md` +- `docs/ROLLBACK_AND_RECOVERY.md` +- `docs/SECURITY_AND_DATA_BOUNDARIES.md` +- `docs/SECURITY_CHECKLIST.md` +- `docs/SEED_DATA_PLAN.md` +- `docs/SEGMENTATION_CLASSES_CATALOG.md` +- `docs/SEGMENTATION_PIPELINE_SPEC.md` +- `docs/SERVICE_ARCHITECTURE.md` +- `docs/SERVICE_IO_CONTRACTS.md` +- `docs/SPECIFICATION_FREEZE_M0.md` +- `docs/SPRINT_BOARD_M4.md` +- `docs/STORAGE_ARCHITECTURE.md` +- `docs/TEST_CATALOG.md` +- `docs/TEST_STRATEGY.md` +- `docs/TODO.md` +- `docs/TROUBLESHOOTING_RUNBOOK.md` +- `docs/UI_PAGE_SPECIFICATIONS.md` +- `docs/UI_ROUTE_CONTRACTS.md` +- `docs/UI_UX_SPEC.md` +- `docs/V1_SCOPE_FREEZE.md` +- `docs/VECTOR_OPERATIONS_SPEC.md` +- `exports/.gitkeep` +- `fixtures/geojson/predicted_buildings_fixture.geojson` +- `fixtures/geojson/reference_buildings_fixture.geojson` +- `frontend/.gitkeep` +- `frontend/README.md` +- `frontend/src/.gitkeep` +- `frontend/src/app/.gitkeep` +- `frontend/src/components/.gitkeep` +- `frontend/src/features/.gitkeep` +- `frontend/src/lib/.gitkeep` +- `frontend/src/pages/.gitkeep` +- `frontend/src/services/api/.gitkeep` +- `frontend/src/stores/.gitkeep` +- `frontend/src/styles/.gitkeep` +- `frontend/src/types/.gitkeep` +- `knowledge/dhmv/README.md` +- `knowledge/grb/README.md` +- `knowledge/postgis/README.md` +- `knowledge/sam/README.md` +- `knowledge/sentinel/README.md` +- `knowledge/yolo/README.md` +- `models/.gitkeep` +- `prompts/codex/M2_MASTER_BUILD_PROMPT.md` +- `prompts/codex/M4_PASS_01_BACKEND_FOUNDATION.md` +- `prompts/codex/M4_PASS_02_DATABASE_DOMAIN.md` +- `prompts/codex/M4_PASS_03_DATASET_MANAGER.md` +- `prompts/codex/M4_PASS_04_MAP_AREA_WORKSPACE.md` +- `prompts/codex/M4_PASS_05_AI_DEMO_PIPELINES.md` +- `prompts/codex/M4_PASS_06_QAQC_EXPORTS.md` +- `prompts/codex/PASS_00_REPO_AUDIT.md` +- `prompts/codex/PASS_01_BACKEND_FOUNDATION.md` +- `prompts/codex/PASS_02_DATABASE_MODELS.md` +- `prompts/codex/PASS_02_PROJECT_AREA_DATASET.md` +- `prompts/codex/PASS_03_PROJECT_AREA_API.md` +- `prompts/codex/PASS_03_RASTER_VECTOR_FOUNDATION.md` +- `prompts/codex/PASS_04_DATASET_MANAGER.md` +- `prompts/codex/PASS_04_DETECTION_QA_SKELETON.md` +- `prompts/codex/PASS_05_DATASET_MANAGER.md` +- `prompts/codex/PASS_05_RASTER_VECTOR_METADATA.md` +- `prompts/codex/PASS_06_FRONTEND_SHELL.md` +- `prompts/codex/PASS_06_RASTER_VECTOR_METADATA.md` +- `prompts/codex/PASS_07_MAP_WORKBENCH.md` +- `prompts/codex/PASS_08_GRB_REFERENCE.md` +- `prompts/codex/PASS_08_TEST_AND_FIXTURE_HARDENING.md` +- `prompts/codex/PASS_09_DETECTION_INTERFACE.md` +- `prompts/codex/PASS_09_DETECTION_SERVICE_SCAFFOLD.md` +- `prompts/codex/PASS_10_EXPORT_PIPELINE.md` +- `prompts/codex/PASS_10_QAQC_ENGINE.md` +- `prompts/codex/PASS_11_EXPORTS.md` +- `prompts/codex/PASS_11_QA_QC_FOUNDATION.md` +- `prompts/codex/PASS_12_STABILIZATION.md` +- `prompts/codex/PASS_12_V1_VERTICAL_SLICE_REVIEW.md` +- `prompts/codex/README.md` +- `rfc/RFC-001-sentinel-integration.md` +- `rfc/RFC-002-lidar-workbench.md` +- `rfc/RFC-003-training-studio.md` +- `rfc/RFC-004-qgis-plugin.md` +- `rfc/RFC-005-mlops-model-registry.md` +- `scripts/.gitkeep` +- `scripts/README.md` +- `scripts/check_repo_structure.sh` +- `scripts/codex_pass_end_check.sh` +- `scripts/codex_preflight.sh` +- `scripts/smoke_backend_import.sh` +- `scripts/smoke_contracts.py` +- `scripts/smoke_docs.py` +- `scripts/validate_fixtures.py` +- `storage/.gitkeep` +- `storage/derived/README.md` +- `storage/exports/README.md` +- `storage/masks/.gitkeep` +- `storage/masks/README.md` +- `storage/models/README.md` +- `storage/originals/README.md` +- `storage/reports/.gitkeep` +- `storage/tiles/.gitkeep` +- `storage/tiles/README.md` +- `storage/uploads/.gitkeep` +- `tests/.gitkeep` +- `tests/backend/.gitkeep` +- `tests/fixtures/.gitkeep` +- `tests/fixtures/README.md` +- `tests/fixtures/geojson/README.md` +- `tests/fixtures/geojson/aoi_geel_demo.geojson` +- `tests/fixtures/geojson/detected_buildings.geojson` +- `tests/fixtures/geojson/reference_buildings.geojson` +- `tests/fixtures/geospatial/.gitkeep` +- `tests/fixtures/rasters/.gitkeep` +- `tests/fixtures/rasters/README.md` +- `tests/fixtures/vectors/.gitkeep` +- `tests/fixtures/vectors/README.md` +- `tests/frontend/.gitkeep` diff --git a/docs/M6_AUTONOMY_BOUNDARIES.md b/docs/M6_AUTONOMY_BOUNDARIES.md new file mode 100644 index 00000000..676c062c --- /dev/null +++ b/docs/M6_AUTONOMY_BOUNDARIES.md @@ -0,0 +1,90 @@ +# M6 Autonomy Boundaries + +## Codex may change without asking + +Codex may freely improve: + +- internal service decomposition when public contracts remain compatible; +- helper functions and utility modules; +- validation functions; +- error messages; +- unit tests; +- smoke tests; +- docs that clarify existing decisions; +- small UI layout improvements that follow the design system; +- fixtures that support documented demo cases; +- logging details that follow observability guidance; +- type names if the change is applied consistently; +- generated API clients if contracts remain stable. + +## Codex may change only with explicit rationale in the handoff + +Codex may adjust: + +- exact internal table indexes; +- pagination defaults; +- threshold defaults for demo inference; +- frontend component split; +- worker retry counts; +- cache invalidation strategy; +- validation strictness; +- local development scripts. + +Every such change must include: + +- what changed; +- why it improves the project; +- which contract remains unchanged; +- whether docs were updated. + +## Codex must not change without approval + +Codex must not change: + +- core stack; +- V1 module list; +- product positioning; +- GRB reference role; +- geospatial CRS strategy; +- API route prefixes once implemented; +- database primary key strategy; +- storage root layout; +- security assumptions; +- license assumptions; +- model class catalog unless documented as proposal; +- public export formats; +- Docker service names; +- environment variable names once introduced. + +## Codex may add proposal documents + +When Codex sees a meaningful improvement outside scope, it may create a proposal in: + +`rfc/RFC-next-.md` + +The proposal must contain: + +- problem; +- proposed change; +- affected modules; +- benefits; +- risks; +- migration effort; +- recommendation. + +It must not implement that proposal in the same pass unless it is inside the allowed autonomy boundaries. + +## Interpretation rule + +When the docs disagree, Codex must apply this priority order: + +1. `V1_SCOPE_FREEZE.md` +2. ADRs +3. `M6_AUTONOMY_BOUNDARIES.md` +4. API contracts +5. database contracts +6. module contracts +7. UI specs +8. roadmap / backlog + +When still ambiguous, Codex must choose the smallest implementation that preserves future extensibility and document the assumption. diff --git a/docs/M6_CODEX_AUTONOMY_PACK.md b/docs/M6_CODEX_AUTONOMY_PACK.md new file mode 100644 index 00000000..ca5a247b --- /dev/null +++ b/docs/M6_CODEX_AUTONOMY_PACK.md @@ -0,0 +1,107 @@ +# M6 — Codex Autonomy Pack + +## Purpose + +M6 converts the GeoIntel repository from an implementation-ready specification into a self-driving build package for Codex. The goal is not to remove engineering judgment. The goal is to remove preventable manual decisions while leaving Codex enough bounded freedom to improve implementation details. + +Codex must be able to: + +1. audit the repository, +2. derive the next build pass, +3. implement one coherent vertical slice, +4. run the relevant tests, +5. update docs and changelog, +6. produce a handoff summary, +7. stop safely when blocked instead of guessing. + +## Autonomy principle + +Codex may improve the implementation when it improves maintainability, correctness, testability, performance, or UX while staying inside the frozen product direction. + +Codex must not change: + +- the product positioning: GeoAI Workbench for the Kempen; +- the backend-first architecture; +- FastAPI, React, TypeScript, PostGIS as core stack; +- GRB as official reference layer for QA/QC; +- data -> processing -> model -> geospatial output -> QA -> export as the core workflow; +- V1 scope boundaries; +- safety rules around external services, licensing, and generated outputs. + +## Build autonomy levels + +### Level 0 — No autonomy + +Used for scope, product direction, data licensing, security boundaries, and architecture decisions already frozen in ADRs. + +### Level 1 — Controlled autonomy + +Used for internal code organization, helper functions, small UX improvements, tests, fixtures, naming consistency, and documentation improvements. + +### Level 2 — Engineering autonomy + +Used for improving implementation quality where the docs define goals but not exact code. Examples: better validation, better error messages, safer geometry handling, cleaner service boundaries. + +### Level 3 — Proposal only + +Used for new modules, new external dependencies, new providers, model changes, new data sources, or changed workflows. Codex may document a proposal but must not implement without explicit approval. + +## Required Codex behavior + +Each pass must start by reading: + +1. `AGENTS.md` +2. `docs/CODEX_MASTER_PROMPT.md` +3. `docs/V1_SCOPE_FREEZE.md` +4. `docs/REPOSITORY_CONVENTIONS.md` +5. the pass-specific prompt in `prompts/codex/` +6. `docs/M6_AUTONOMY_BOUNDARIES.md` +7. `docs/M6_QUALITY_GATES.md` + +Each pass must end with: + +1. tests run or a precise explanation why not; +2. changed files list; +3. completion status against acceptance criteria; +4. known limitations; +5. next recommended pass; +6. no hidden TODOs in code unless documented in `docs/IMPLEMENTATION_BACKLOG.md`. + +## Output discipline + +Codex must keep implementation output small enough to review. A pass should build one coherent vertical slice, not ten half-finished modules. + +Preferred pass size: + +- 5 to 25 files changed; +- one backend service plus related schemas/tests; or +- one frontend route plus related API client/store/tests; or +- one full vertical slice when small enough. + +## Non-negotiable stop conditions + +Codex must stop and report instead of guessing when: + +- a data license is unclear; +- a required external URL/API is unknown; +- a migration could destroy data; +- a dependency introduces GPU/CUDA assumptions without CPU fallback; +- a security boundary would be weakened; +- a design conflicts with frozen docs; +- test failures cannot be isolated. + +## M6 deliverables + +This pack adds: + +- autonomy boundaries; +- strict-but-flexible improvement rules; +- pass prompts from audit to V1 completion; +- self-review checklist; +- failure recovery playbook; +- gap registry; +- final handoff template; +- issue templates; +- CI expectations; +- Codex build command reference; +- next-day execution checklist. diff --git a/docs/M6_FAILURE_RECOVERY_PLAYBOOK.md b/docs/M6_FAILURE_RECOVERY_PLAYBOOK.md new file mode 100644 index 00000000..9e077b40 --- /dev/null +++ b/docs/M6_FAILURE_RECOVERY_PLAYBOOK.md @@ -0,0 +1,84 @@ +# M6 Failure Recovery Playbook + +## Purpose + +Codex must recover from failures systematically instead of patching randomly. + +## Failure classes + +### Class A — Syntax/import failure + +Action: + +1. identify failing module; +2. fix import path or missing dependency; +3. add a smoke import test; +4. rerun minimal test; +5. document root cause. + +### Class B — Database/migration failure + +Action: + +1. stop destructive changes; +2. inspect model/migration mismatch; +3. create additive migration fix; +4. verify PostGIS extension assumptions; +5. add migration note. + +### Class C — API contract mismatch + +Action: + +1. compare endpoint to API contract; +2. fix route/schema/status code; +3. update typed client if required; +4. add contract test. + +### Class D — Geospatial processing failure + +Action: + +1. inspect CRS; +2. inspect geometry validity; +3. inspect empty geometry cases; +4. verify projected/unit assumptions; +5. add a small fixture reproducing the issue. + +### Class E — AI inference failure + +Action: + +1. check model availability; +2. check CPU fallback; +3. check image tiling shape; +4. check georeference transform; +5. return a structured job failure instead of crashing. + +### Class F — Frontend state failure + +Action: + +1. reproduce route state; +2. check API response type; +3. add loading/error/empty state; +4. prevent stale local-only data; +5. add component-level test where possible. + +## Repair limits + +Codex may do targeted fixes in the same pass. If repair requires redesigning a module, Codex must stop and document a recovery plan. + +## Required recovery note + +Every recovered failure must be logged as: + +```md +### Failure recovered + +- Symptom: +- Root cause: +- Fix: +- Test added: +- Risk remaining: +``` diff --git a/docs/M6_FINAL_HANDOFF_TEMPLATE.md b/docs/M6_FINAL_HANDOFF_TEMPLATE.md new file mode 100644 index 00000000..2d039fcc --- /dev/null +++ b/docs/M6_FINAL_HANDOFF_TEMPLATE.md @@ -0,0 +1,61 @@ +# M6 Final Handoff Template + +Codex must use this structure at the end of each build pass. + +## Pass name + +`PASS XX — ` + +## Summary + +Short summary of what was implemented. + +## Changed files + +- `path/to/file`: reason +- `path/to/file`: reason + +## Acceptance criteria status + +| Criterion | Status | Notes | +|---|---|---| +| Criterion 1 | Done/Partial/Blocked | Notes | + +## Tests run + +```bash +command +``` + +Result: + +```txt +pass/fail output summary +``` + +## Manual verification + +- Step 1: +- Step 2: +- Expected result: + +## Known limitations + +- Limitation 1 +- Limitation 2 + +## Risks + +- Risk 1 +- Risk 2 + +## Next recommended pass + +`PASS XX — ` + +## Do not forget + +- [ ] docs updated +- [ ] TODO/backlog updated +- [ ] no hidden placeholder completion +- [ ] no architectural drift diff --git a/docs/M6_GAP_REGISTRY.md b/docs/M6_GAP_REGISTRY.md new file mode 100644 index 00000000..7956771b --- /dev/null +++ b/docs/M6_GAP_REGISTRY.md @@ -0,0 +1,39 @@ +# M6 Gap Registry + +This file tracks known preparation gaps that Codex may close during implementation if they are in scope. + +## Gaps Codex may close directly + +| Gap | Allowed action | +|---|---| +| Missing small fixture | Create deterministic fixture. | +| Missing test for implemented endpoint | Add test. | +| Ambiguous UI empty state | Implement using UI state contracts. | +| Minor missing error code | Add according to error contract. | +| Missing helper docs after implementation | Update docs. | +| Missing local script for repeated command | Add script under `scripts/`. | + +## Gaps requiring proposal only + +| Gap | Required action | +|---|---| +| New external data source | Create RFC. | +| New model family | Create RFC. | +| GPU-only dependency | Create RFC. | +| New database engine | Reject unless approved. | +| New authentication system | Create RFC. | +| Multi-user permissions | Create RFC. | +| QGIS plugin | Create RFC. | +| Full MLOps platform | Create RFC. | + +## Current strategic gaps intentionally deferred + +- Real Sentinel provider credentials and download flow. +- Production GRB WFS adapter details. +- Full LiDAR processing. +- Training/fine-tuning studio. +- Hosted deployment target. +- Model registry UI. +- QGIS plugin. + +These are not blockers for V1 foundation. diff --git a/docs/M6_HANDOFF_SUMMARY.md b/docs/M6_HANDOFF_SUMMARY.md new file mode 100644 index 00000000..b4346e64 --- /dev/null +++ b/docs/M6_HANDOFF_SUMMARY.md @@ -0,0 +1,45 @@ +# M6 Handoff Summary — Codex Autonomy Pack + +## What M6 adds + +M6 adds the documents and prompts needed to let Codex execute the first build passes with minimal manual steering. + +## New capabilities in the repo preparation + +- Codex autonomy boundaries. +- Quality gates per layer. +- Self-review checklist. +- Failure recovery playbook. +- Gap registry. +- Next-day execution checklist. +- Final handoff template. +- Sequential Codex prompt pack. +- GitHub issue and PR templates. +- CI smoke workflow. +- Codex preflight and pass-end scripts. + +## How to use tomorrow + +1. Extract the full M6 zip. +2. Start a git repo if not already initialized. +3. Run `bash scripts/codex_preflight.sh`. +4. Give Codex `prompts/codex/PASS_00_REPO_AUDIT.md`. +5. Only after the audit, continue with `PASS_01_BACKEND_FOUNDATION.md`. +6. After every pass, require the handoff format in `docs/M6_FINAL_HANDOFF_TEMPLATE.md`. +7. Commit only after smoke checks pass. + +## Recommended first objective + +Do not try to finish GeoIntel tomorrow. The ideal first Codex day should aim for: + +- backend foundation; +- database foundation; +- project/area API; +- dataset upload foundation; +- first frontend shell if backend is stable. + +That already creates a serious base without architecture drift. + +## Current maturity + +Preparation maturity after M6: high for autonomous foundation building, medium for AI-heavy implementation, intentionally deferred for live external data providers and full training/LiDAR modules. diff --git a/docs/M6_NEXT_DAY_EXECUTION_CHECKLIST.md b/docs/M6_NEXT_DAY_EXECUTION_CHECKLIST.md new file mode 100644 index 00000000..2e141e49 --- /dev/null +++ b/docs/M6_NEXT_DAY_EXECUTION_CHECKLIST.md @@ -0,0 +1,66 @@ +# M6 Next-day Execution Checklist + +Use this tomorrow before starting Codex. + +## Local prep + +- [ ] Extract the latest full zip into the working folder. +- [ ] Open the repo root in Codex. +- [ ] Confirm `AGENTS.md` is visible. +- [ ] Confirm docs folder is intact. +- [ ] Confirm `.env.example` is present. +- [ ] Confirm Docker is available if using containerized PostGIS. +- [ ] Create a new git branch: `build/foundation-v0-1`. + +## First Codex instruction + +Start with: + +`prompts/codex/PASS_00_REPO_AUDIT.md` + +Do not ask Codex to build yet. First ask it to audit, summarize, and identify blockers. + +## Build order + +1. PASS 00 — Repo audit +2. PASS 01 — Backend foundation +3. PASS 02 — Database + models +4. PASS 03 — Project + area API +5. PASS 04 — Dataset manager +6. PASS 05 — Raster/vector metadata +7. PASS 06 — Frontend shell +8. PASS 07 — Map workbench +9. PASS 08 — QA fixtures and smoke tests +10. PASS 09 — Detection service scaffold +11. PASS 10 — Export pipeline + +## After every pass + +- [ ] Ask Codex for changed files. +- [ ] Ask Codex for tests run. +- [ ] Ask Codex for remaining blockers. +- [ ] Commit after passing smoke tests. +- [ ] Do not merge broken states. + +## Minimal commit message format + +```txt +feat(scope): short description + +- changed thing +- changed thing +- tests: command/result +``` + +## When to interrupt Codex + +Interrupt if Codex: + +- starts implementing LiDAR in V1; +- changes the stack; +- uses mock-only data for core flows; +- skips backend persistence; +- adds authentication; +- hardcodes data paths outside the storage spec; +- removes PostGIS assumptions; +- marks placeholder pages as done. diff --git a/docs/M6_QUALITY_GATES.md b/docs/M6_QUALITY_GATES.md new file mode 100644 index 00000000..81c7bbf5 --- /dev/null +++ b/docs/M6_QUALITY_GATES.md @@ -0,0 +1,110 @@ +# M6 Quality Gates + +## Gate 0 — Repository health + +Before implementing, Codex must verify: + +- expected root folders exist; +- docs are present; +- `.env.example` exists; +- Docker compose exists; +- scripts are executable or documented; +- no obvious merge conflict markers exist; +- no generated secrets are committed. + +## Gate 1 — Contract compliance + +Every implemented endpoint must have: + +- request schema; +- response schema; +- status codes; +- validation behavior; +- error shape following the error contract; +- at least one unit or smoke test. + +## Gate 2 — Database safety + +Every migration must: + +- be additive unless explicitly planned; +- include indexes for geometry and foreign keys where relevant; +- avoid destructive operations; +- preserve future PostGIS use; +- be documented in migration notes. + +## Gate 3 — Geospatial correctness + +Every geometry operation must: + +- validate geometry; +- preserve CRS metadata; +- use projected CRS for area/length where required; +- return units explicitly; +- handle empty geometries; +- handle invalid polygons safely; +- never silently mix coordinate systems. + +## Gate 4 — Job safety + +Long-running tasks must: + +- run through the job layer; +- record status transitions; +- record parameters; +- record errors; +- be restart-safe where feasible; +- not block the API process. + +## Gate 5 — Frontend state correctness + +Every frontend module must: + +- handle loading, empty, success, error, and processing states; +- not assume demo data exists; +- use typed API clients; +- show job progress if job-backed; +- avoid hidden state that cannot be refreshed. + +## Gate 6 — AI pipeline safety + +Every AI pipeline must: + +- support CPU fallback; +- write deterministic metadata; +- record model id/version; +- record thresholds; +- preserve georeferencing for outputs; +- expose confidence values; +- never overwrite raw input data. + +## Gate 7 — Export integrity + +Every export must include: + +- source dataset reference; +- analysis run id; +- generated timestamp; +- CRS; +- units; +- limitations where relevant. + +## Gate 8 — Handoff completeness + +Every Codex pass must update: + +- `docs/CODEX_EXECUTION_LOG.md` or a pass log; +- relevant module docs if behavior changed; +- tests or test notes; +- TODO/backlog if work remains. + +## Gate 9 — No fake completion + +A feature may not be marked done when it only has: + +- placeholder UI; +- stub endpoint without real service behavior; +- mock data not clearly marked as fixture/demo; +- no persistence despite requiring persistence; +- no error handling; +- no acceptance criteria coverage. diff --git a/docs/M6_SELF_REVIEW_CHECKLIST.md b/docs/M6_SELF_REVIEW_CHECKLIST.md new file mode 100644 index 00000000..79d79cb1 --- /dev/null +++ b/docs/M6_SELF_REVIEW_CHECKLIST.md @@ -0,0 +1,56 @@ +# M6 Self-review Checklist for Codex + +Codex must run this checklist before ending any pass. + +## Scope check + +- [ ] I implemented only the requested pass or a clearly justified adjacent dependency. +- [ ] I did not add out-of-scope features. +- [ ] I did not change frozen architecture decisions. +- [ ] I documented any assumption I made. + +## Backend check + +- [ ] API schemas are typed. +- [ ] Services contain business logic; routes stay thin. +- [ ] Errors use the standard error contract. +- [ ] Jobs use the job lifecycle contract when needed. +- [ ] Database operations are safe and migration-compatible. + +## Frontend check + +- [ ] The UI has loading, empty, error, success states. +- [ ] API calls go through the shared client. +- [ ] Components follow the design system. +- [ ] No important action is hidden behind unclear UI. +- [ ] State can be refreshed from backend data. + +## Geospatial check + +- [ ] CRS is captured and shown where relevant. +- [ ] Geometry validity is checked. +- [ ] Area/length units are explicit. +- [ ] GeoJSON export is valid. +- [ ] No coordinate conversion is silently assumed. + +## AI check + +- [ ] Model metadata is recorded. +- [ ] Thresholds are explicit. +- [ ] Inference output is georeferenced when input is georeferenced. +- [ ] CPU fallback exists or limitation is documented. +- [ ] Raw inputs are immutable. + +## Test check + +- [ ] Unit tests were added or updated. +- [ ] Smoke tests still pass or failure is documented. +- [ ] Fixtures are deterministic. +- [ ] No test relies on live external services unless explicitly marked integration. + +## Documentation check + +- [ ] Docs reflect changed behavior. +- [ ] Changelog/pass log updated. +- [ ] Remaining TODOs are in docs, not hidden in code. +- [ ] Handoff summary includes changed files, tests, limits, next pass. diff --git a/docs/MIGRATION_PLAN.md b/docs/MIGRATION_PLAN.md new file mode 100644 index 00000000..40f0ff04 --- /dev/null +++ b/docs/MIGRATION_PLAN.md @@ -0,0 +1,82 @@ +# Database Migration Plan + +## Migration tool + +Use Alembic for all schema migrations. + +## Migration principles + +- Every database schema change must be represented by an Alembic migration. +- Geometry columns must use PostGIS types. +- Migrations must be safe to rerun in development after database reset. +- Never modify old migrations after they have been used as a release baseline. + +## Initial migration sequence + +### 0001_enable_postgis + +Creates the PostGIS extension. + +```sql +CREATE EXTENSION IF NOT EXISTS postgis; +``` + +### 0002_core_tables + +Creates: + +- projects +- areas +- datasets +- layers +- analysis_runs +- metrics +- exports + +### 0003_geoai_tables + +Creates: + +- detections +- segmentations +- quality_checks +- quality_check_items + +### 0004_jobs_and_events + +Creates: + +- jobs +- domain_events + +## Geometry columns + +Use SRID 4326 for persisted canonical geometries unless a specific processing table requires another CRS. + +Recommended columns: + +```text +areas.geometry: POLYGON/MULTIPOLYGON, SRID 4326 +detections.geometry: POLYGON, SRID 4326 +quality_check_items.geometry: GEOMETRY, SRID 4326 +``` + +Area measurements must not be calculated directly in EPSG:4326. Use projected CRS transformations in services. + +## Indexes + +Every geometry column that is queried spatially needs a GIST index. + +Core indexes: + +- projects.created_at +- areas.project_id +- datasets.project_id +- layers.dataset_id +- analysis_runs.project_id +- detections.analysis_run_id +- quality_checks.analysis_run_id + +## Development reset + +Development reset may drop and recreate the database. Production-style migrations must still remain valid. diff --git a/docs/MODEL_REGISTRY_SEED.md b/docs/MODEL_REGISTRY_SEED.md new file mode 100644 index 00000000..23082206 --- /dev/null +++ b/docs/MODEL_REGISTRY_SEED.md @@ -0,0 +1,55 @@ +# Model Registry Seed + +The application must ship with a deterministic demo model registry before real ML models are integrated. + +## Seed Models + +### demo-yolo-buildings-v1 +- framework: demo +- task: object_detection +- classes: building +- purpose: deterministic demo detections for UI and QA/QC development +- production_ready: false + +### demo-seg-vegetation-v1 +- framework: demo +- task: segmentation +- classes: vegetation, water, impervious_surface +- purpose: deterministic segmentation polygons for UI and metrics development +- production_ready: false + +### ultralytics-yolo-placeholder +- framework: ultralytics +- task: object_detection +- classes: configurable +- purpose: real YOLO integration slot +- production_ready: false until implemented and tested + +### sam-placeholder +- framework: segment-anything +- task: segmentation +- classes: prompt_based +- purpose: real SAM integration slot +- production_ready: false until implemented and tested + +## Registry Fields +```json +{ + "id": "demo-yolo-buildings-v1", + "name": "Demo YOLO Buildings", + "version": "1.0.0", + "framework": "demo", + "task": "object_detection", + "classes": ["building"], + "artifact_path": null, + "metadata": { + "deterministic": true, + "portfolio_demo": true + } +} +``` + +## Rules +- Demo models must be clearly labeled as non-production. +- Real model integration must not silently replace demo behavior. +- Model outputs must always include model id and version. diff --git a/docs/MODEL_REGISTRY_SPEC.md b/docs/MODEL_REGISTRY_SPEC.md new file mode 100644 index 00000000..208e022c --- /dev/null +++ b/docs/MODEL_REGISTRY_SPEC.md @@ -0,0 +1,52 @@ +# Model Registry Specification + +## Purpose + +The model registry tracks which AI models are available, what they can detect or segment, and how outputs should be interpreted. + +## V1 model record + +```yaml +id: yolov8n-building-demo +name: YOLO Building Demo +framework: ultralytics +runtime: pytorch +task: object_detection +version: 0.1.0 +classes: + - building +input: + tile_size: 640 + channels: 3 +outputs: + type: bounding_boxes +metrics: + mAP50: null + precision: null + recall: null +status: experimental +``` + +## Required metadata + +- id +- name +- framework +- task +- version +- classes +- artifact_path or provider +- input constraints +- output schema +- status + +## Status values + +- experimental +- validated +- deprecated +- disabled + +## Rule + +Detection and segmentation services must request model info through the registry. They must not hardcode model paths in API routes. diff --git a/docs/MODULES.md b/docs/MODULES.md new file mode 100644 index 00000000..55e47f85 --- /dev/null +++ b/docs/MODULES.md @@ -0,0 +1,175 @@ +# Modules + +## 1. Project Manager + +### Doel + +Beheert onderzoeken en demo-cases. + +### Input + +- projectnaam +- beschrijving +- regio +- analysegebieden + +### Output + +- project workspace +- gekoppelde datasets +- analysis runs +- exports + +## 2. Dataset Manager + +### Doel + +Alle datasets beheren en valideren. + +### Ondersteuning + +- raster upload +- vector upload +- referentiedata ophalen +- metadata tonen +- status tonen + +### Belangrijke statussen + +- uploaded +- metadata_extracted +- needs_crs_review +- ready +- processing +- failed + +## 3. Raster Workbench + +### Doel + +Rasterdata inspecteren en voorbereiden voor analyse en AI. + +### Functies + +- metadata lezen +- preview tonen +- bandselectie +- clipping +- resampling +- tiling +- histogram +- NDVI/NDWI/NDBI later + +## 4. Vector Workbench + +### Doel + +Vectorlagen verwerken en analyseren. + +### Functies + +- GeoJSON/shapefile/GPKG import +- CRS transformeren +- geometrievalidatie +- clipping +- buffers +- overlays +- spatial joins +- area/perimeter berekenen + +## 5. Detection Lab + +### Doel + +Objectdetectie op rasterbeelden. + +### Functies + +- model kiezen +- confidence threshold instellen +- tile inference starten +- bounding boxes tonen +- class statistics tonen +- GeoJSON exporteren + +## 6. Segmentation Lab + +### Doel + +Maskers maken voor gebouwen, groen, water, verharding of andere klassen. + +### Functies + +- segmentation model kiezen +- mask preview +- polygonize +- oppervlakteberekening +- mask export +- vector export + +## 7. QA/QC Lab + +### Doel + +AI-output controleren tegenover referentiedata. + +### Functies + +- referentielaag kiezen +- IoU berekenen +- precision/recall/F1 berekenen +- false positives tonen +- false negatives tonen +- kwaliteitsrapport exporteren + +## 8. Change Detection Lab + +### Doel + +Twee datasets of analysis runs vergelijken. + +### Functies + +- dataset A/B kiezen +- detectie/segmentatie vergelijken +- added/removed/changed polygons tonen +- statistieken berekenen + +## 9. Annotation Studio + +### Doel + +Trainingsdata maken. + +### Functies + +- bounding boxes tekenen +- polygonen tekenen +- class label kiezen +- export naar YOLO/COCO/GeoJSON + +## 10. Training Studio + +### Doel + +Latere module voor modeltraining/fine-tuning. + +### Functies + +- dataset split +- training starten +- metrics tonen +- model artifact opslaan + +## 11. LiDAR Workbench + +### Doel + +Latere module voor puntenwolkanalyse. + +### Functies + +- LAS/LAZ metadata +- clipping +- DEM/DSM generatie +- hoogteprofielen diff --git a/docs/MODULE_BUILD_CONTRACTS.md b/docs/MODULE_BUILD_CONTRACTS.md new file mode 100644 index 00000000..5b46cd42 --- /dev/null +++ b/docs/MODULE_BUILD_CONTRACTS.md @@ -0,0 +1,81 @@ +# Module Build Contracts + +Every module must follow this contract before it is considered build-complete. + +## Common Contract +Each module must define: +- Purpose. +- Inputs. +- Outputs. +- API calls. +- Database entities touched. +- Background jobs used. +- Empty, loading, error and success UI states. +- Tests. +- Export behavior if applicable. + +## Project Workspace +Purpose: manage a geospatial investigation. +Inputs: name, description, optional region preset. +Outputs: project record, workspace route. +API: `POST /api/projects`, `GET /api/projects`, `GET /api/projects/{id}`. +DB: projects. +Tests: create/list/detail. + +## Area Manager +Purpose: create reusable analysis geometries. +Inputs: GeoJSON polygon or drawn polygon. +Outputs: area geometry, area size, bounds. +API: `POST /api/projects/{project_id}/areas`, `GET /api/projects/{project_id}/areas`. +DB: areas. +Validation: polygon must be valid, non-empty and inside supported broad region unless override is enabled. + +## Dataset Manager +Purpose: ingest source data. +Inputs: uploaded files or source connector request. +Outputs: dataset record with extracted metadata. +API: `POST /api/projects/{project_id}/datasets`, `GET /api/projects/{project_id}/datasets`, `GET /api/datasets/{id}`. +DB: datasets, dataset_versions. +Storage: originals, processed, cache. + +## Raster Lab +Purpose: inspect and prepare raster data for analysis and AI. +Inputs: raster dataset, area polygon, band config. +Outputs: metadata, preview, clipped raster, tiles. +API: `/api/raster/*`. +DB: datasets, processing_jobs, raster_outputs. + +## Vector Lab +Purpose: inspect and process vector data. +Inputs: vector dataset, area polygon, operation parameters. +Outputs: clipped/intersected/buffered derived layers. +API: `/api/vector/*`. +DB: layers, derived_layers. + +## Detection Lab +Purpose: run object detection and georeference outputs. +Inputs: raster dataset, model id, threshold, tile size. +Outputs: detections as PostGIS geometries and GeoJSON export. +API: `POST /api/analysis/detection-runs`. +DB: analysis_runs, detections, metrics. + +## Segmentation Lab +Purpose: run segmentation and polygonize masks. +Inputs: raster dataset, model id, classes, threshold. +Outputs: segmentation polygons, masks and area metrics. +API: `POST /api/analysis/segmentation-runs`. +DB: analysis_runs, segmentations, metrics. + +## QA/QC Lab +Purpose: compare AI output against reference geodata such as GRB. +Inputs: analysis run, reference dataset, class mapping, IoU threshold. +Outputs: precision, recall, F1, matched/unmatched features. +API: `POST /api/qaqc/runs`. +DB: quality_checks, quality_findings, metrics. + +## Exports +Purpose: provide reusable outputs for GIS tools and portfolio demo. +Inputs: project, analysis, dataset or layer id. +Outputs: GeoJSON, CSV, report files. +API: `POST /api/exports`. +DB: exports. diff --git a/docs/MODULE_CONTRACTS.md b/docs/MODULE_CONTRACTS.md new file mode 100644 index 00000000..33ba50a8 --- /dev/null +++ b/docs/MODULE_CONTRACTS.md @@ -0,0 +1,108 @@ +# Module Contracts + +## Dataset Manager Contract + +### Inputs + +- project id +- uploaded file +- declared dataset type when auto-detection is ambiguous + +### Outputs + +- dataset record +- metadata record +- processing status + +### Required states + +- empty: no datasets yet +- uploading +- processing metadata +- ready +- failed + +## Raster Lab Contract + +### Inputs + +- raster dataset id +- optional area id + +### Outputs + +- metadata panel +- preview bounds +- clip job +- tile job + +### Required states + +- no raster datasets +- raster selected +- unsupported raster +- metadata failed + +## Vector Lab Contract + +### Inputs + +- vector dataset id +- optional area id + +### Outputs + +- feature count +- geometry type summary +- bounds +- clip result +- derived vector layer + +## Detection Lab Contract + +### Inputs + +- raster dataset id +- area id +- model id +- confidence threshold +- tile size + +### Outputs + +- analysis run +- detection features +- confidence summary +- GeoJSON export + +## QA/QC Contract + +### Inputs + +- prediction layer or detection run +- reference layer, preferably GRB buildings +- IoU threshold + +### Outputs + +- precision +- recall +- F1 +- matched features +- false positives +- false negatives +- QA layer export + +## Export Contract + +### Inputs + +- project id +- dataset id or analysis run id +- export type + +### Outputs + +- export record +- downloadable file +- metadata about source and creation date diff --git a/docs/OBSERVABILITY_PLAN.md b/docs/OBSERVABILITY_PLAN.md new file mode 100644 index 00000000..8c5e2efa --- /dev/null +++ b/docs/OBSERVABILITY_PLAN.md @@ -0,0 +1,19 @@ +# Observability Plan + +## Doel +GeoIntel moet tijdens lange AI/GIS-jobs duidelijk tonen wat er gebeurt. + +## Logcategorieën +- Application logs: API requests, validatiefouten, service lifecycle. +- Geospatial logs: raster metadata, CRS-detectie, clip/reproject/tiling, geometrievalidatie. +- AI logs: model load, tile inference, merge, polygonization, filtering. +- Job logs: queued, started, progress, completed, failed. + +## Correlation IDs +Elke job en API-call die een job start krijgt een `correlation_id`. + +## Minimale job events +`queued`, `started`, `metadata_extracted`, `preprocessing_completed`, `inference_started`, `inference_completed`, `postprocessing_completed`, `persisted`, `completed`, `failed`. + +## Frontend observability +De UI toont status, progress, laatste logregel, link naar details en foutmelding met herstelactie. diff --git a/docs/PERFORMANCE_BUDGETS.md b/docs/PERFORMANCE_BUDGETS.md new file mode 100644 index 00000000..089f4544 --- /dev/null +++ b/docs/PERFORMANCE_BUDGETS.md @@ -0,0 +1,34 @@ +# Performance Budgets + +GeoIntel will eventually process large geospatial files. V1 must make limits explicit. + +## V1 file limits + +- Default max upload: 500 MB. +- Raster preview should not load full large raster into frontend memory. +- Backend metadata extraction should stream or sample where possible. + +## Processing rules + +- Long operations must become jobs after foundation is complete. +- V1 can run small fixture operations synchronously if documented. +- Tile generation should avoid keeping all tiles in memory. + +## Frontend budgets + +- Avoid rendering thousands of features directly without simplification/clustering. +- Use map layers for GeoJSON, not huge DOM lists. +- Paginate dataset and analysis lists if needed. + +## Database indexes + +Required indexes: + +- spatial index on `areas.geometry` +- spatial index on `vector_features.geometry` +- spatial index on `detections.geometry` +- foreign key indexes for project/dataset/analysis lookups + +## Known V1 limitation + +Full production-scale remote sensing processing is not required in V1. The product must demonstrate correct architecture and working small/medium workflows. diff --git a/docs/PRODUCT_BLUEPRINT.md b/docs/PRODUCT_BLUEPRINT.md new file mode 100644 index 00000000..7c744013 --- /dev/null +++ b/docs/PRODUCT_BLUEPRINT.md @@ -0,0 +1,121 @@ +# GeoIntel Kempen — Product Blueprint v1.0 + +## 1. Positionering + +GeoIntel Kempen is een GeoAI Workbench voor regionale remote sensing en geospatiale AI-analyse in de Belgische Kempen. + +Het project is afgestemd op een GeoAI Engineer-profiel. Het moet aantonen dat de ontwikkelaar kan werken met GIS, remote sensing, rasterdata, vectordata, computer vision, PyTorch, objectdetectie, segmentatie, kwaliteitscontrole en geospatiale data engineering. + +GeoIntel is dus niet primair een beleidsdashboard of rapportagetool. Rapportage is een exportlaag. De kern is: + +```text +data → preprocessing → AI/model → geospatial output → QA/QC → export +``` + +## 2. Productbelofte + +GeoIntel zet luchtfoto's, satellietbeelden, vectorlagen en referentiedata om in controleerbare AI-gegenereerde GIS-lagen. + +## 3. Primaire gebruiker + +De primaire gebruiker is een technische GIS/GeoAI-gebruiker: + +- GeoAI Engineer +- GIS-analist +- remote sensing onderzoeker +- data engineer met geodata-focus +- ruimtelijk analist + +Secundaire gebruikers zoals beleidsmakers kunnen later via rapporten en dashboards bediend worden. + +## 4. Kernmodules + +### Project Manager + +Beheert analyseprojecten, gebieden, datasets en runs. + +### Dataset Manager + +Beheert raster-, vector- en later LiDAR-datasets. + +### Raster Workbench + +Leest rastermetadata, toont previews, ondersteunt bandselectie, clipping, tiling, indexberekeningen en AI-ready preprocessing. + +### Vector Workbench + +Laadt vectorlagen, valideert geometrieën, transformeert CRS, voert clipping, buffers, overlays en spatial joins uit. + +### Detection Lab + +Voert objectdetectie uit op getilede rasters. Resultaten worden teruggezet naar geografische coördinaten en als GeoJSON/PostGIS-laag opgeslagen. + +### Segmentation Lab + +Voert semantische of instance-segmentatie uit. Masks worden gepolygoniseerd en als geospatiale output opgeslagen. + +### QA/QC Lab + +Vergelijkt AI-resultaten met referentielagen zoals GRB. Berekent IoU, precision, recall, F1, false positives en false negatives. + +### Change Detection Lab + +Vergelijkt datasets of analysis runs doorheen tijd en detecteert toegevoegde, verdwenen of gewijzigde objecten. + +### Annotation Studio + +Laat gebruikers bounding boxes, polygonen of masks tekenen en exporteren naar YOLO, COCO, GeoJSON of mask-formaat. + +### Training Studio + +Latere module voor finetuning/training van modellen met metrics zoals mAP, precision, recall, IoU en confusion matrix. + +### LiDAR Workbench + +Latere module voor LAS/LAZ, PDAL, laspy, DEM, DSM en hoogteprofielen. + +## 5. MVP-scope + +V1 moet technisch sterk, maar haalbaar zijn. + +V1 bevat: + +- project aanmaken +- analysegebied tekenen +- raster upload +- vector upload +- metadata extractie +- kaartviewer +- raster/vector overlay +- raster clipping +- raster tiling +- OSM of GRB-referentie integratie +- objectdetectie via YOLO demo-model +- georeferencing van detecties +- detecties als kaartlaag +- GeoJSON export +- QA/QC tegenover referentielaag + +V1 bevat niet: + +- volledige LiDAR verwerking +- volledige modeltraining +- perfecte Sentinel-pipeline +- multi-user systeem +- complexe beleidsrapporten + +## 6. Demo-strategie + +De belangrijkste portfolio-demo: + +1. Kies een gebied in Geel, Mol of Turnhout. +2. Laad een luchtfoto of raster. +3. Haal GRB/OSM-gebouwen op als referentielaag. +4. Tile het raster. +5. Run YOLO-gebouwdetectie. +6. Projecteer detecties naar kaartcoördinaten. +7. Vergelijk detecties met GRB. +8. Toon false positives, false negatives en IoU. +9. Exporteer detecties als GeoJSON. + +Dit demonstreert GIS, raster, vector, AI, QA/QC en export in één verhaal. diff --git a/docs/PRODUCT_VISION.md b/docs/PRODUCT_VISION.md new file mode 100644 index 00000000..d1804250 --- /dev/null +++ b/docs/PRODUCT_VISION.md @@ -0,0 +1,44 @@ +# Product Vision + +GeoIntel Kempen bestaat om open en lokale geospatiale data om te zetten in bruikbare, controleerbare GeoAI-resultaten. + +## Geen klassieke GIS-viewer + +Een klassieke GIS-viewer toont lagen. GeoIntel verwerkt lagen. + +De kaart is noodzakelijk, maar niet het eindproduct. Het eindproduct is een betrouwbare geospatiale output: + +- detectielaag +- segmentatielaag +- change layer +- kwaliteitsrapport +- exporteerbare dataset + +## Portfolio-doel + +GeoIntel moet bewijzen dat de ontwikkelaar de volledige keten begrijpt: + +1. data verzamelen +2. data valideren +3. CRS en geometrieën correct behandelen +4. rasterdata voorbereiden +5. AI-inference uitvoeren +6. modeloutputs geospatiaal correct maken +7. resultaten vergelijken met referentiedata +8. kwaliteit meten +9. resultaten exporteren + +## Regionale identiteit + +De Kempen vormen de afgebakende regio. Dit maakt het project concreet, realistisch en demo-baar. Voorbeelden: + +- gebouwdetectie in Geel +- infrastructuurdruk rond Mol +- verstedelijking rond Turnhout +- natuurfragmentatie rond Kasterlee of Retie + +## Richting + +De definitieve richting is: + +> GeoIntel Kempen is een GeoAI Workbench waarmee je open geodata, luchtfoto's en satellietbeelden verwerkt tot detecties, segmentaties, veranderingen en controleerbare GIS-lagen. diff --git a/docs/PROPOSED_IMPROVEMENTS.md b/docs/PROPOSED_IMPROVEMENTS.md new file mode 100644 index 00000000..e5a0854d --- /dev/null +++ b/docs/PROPOSED_IMPROVEMENTS.md @@ -0,0 +1,45 @@ +# Proposed Improvements Backlog + +Codex should add ideas here instead of silently expanding scope. + +## Format + +```md +## Title + +Status: proposed +Area: backend/frontend/geospatial/AI/devops/docs +Reason: +Impact: +Risk: +Recommended milestone: +``` + +## Initial Suggested Future Improvements + +### STAC-Based Sentinel Integration + +Status: proposed +Area: data/raster +Reason: A STAC workflow would make Sentinel integration cleaner and more reproducible. +Impact: High for V2 remote sensing. +Risk: Medium dependency complexity. +Recommended milestone: V2. + +### QGIS Export Package + +Status: proposed +Area: export/GIS +Reason: Pack GeoJSON, styles and metadata so results open cleanly in QGIS. +Impact: High portfolio relevance. +Risk: Low. +Recommended milestone: V1.1. + +### Model Registry UI + +Status: proposed +Area: AI/MLOps +Reason: Allows tracking model versions, classes and evaluation metrics. +Impact: High for V3. +Risk: Medium. +Recommended milestone: V3. diff --git a/docs/QA_QC_ENGINE.md b/docs/QA_QC_ENGINE.md new file mode 100644 index 00000000..c0c944b6 --- /dev/null +++ b/docs/QA_QC_ENGINE.md @@ -0,0 +1,77 @@ +# QA/QC Engine + +## 1. Doel + +De QA/QC Engine maakt GeoIntel professioneel. AI-resultaten zijn pas waardevol als ze controleerbaar zijn tegenover referentiedata zoals GRB. + +## 2. Primaire use case + +Gebouwdetecties vergelijken met GRB-gebouwpolygonen. + +## 3. Input + +- analysis_run_id met detecties of segmentaties +- reference_dataset_id met referentiepolygonen +- IoU threshold +- class mapping + +## 4. Output + +- true positives +- false positives +- false negatives +- precision +- recall +- F1 +- mean IoU +- kaartlagen met fouten + +## 5. Matching-regels + +1. Bereken intersects tussen AI-output en referentieobjecten. +2. Bereken IoU per kandidaatpaar. +3. Match hoogste IoU boven threshold. +4. Voorkom dubbele matches. +5. AI-output zonder match = false positive. +6. Referentie zonder match = false negative. +7. Matched objecten = true positive. + +## 6. IoU + +```text +IoU = area(intersection) / area(union) +``` + +## 7. Metrics + +```text +precision = TP / (TP + FP) +recall = TP / (TP + FN) +F1 = 2 * precision * recall / (precision + recall) +``` + +## 8. Kaartlagen + +QA/QC moet minstens drie lagen kunnen tonen: + +- true positives +- false positives +- false negatives + +Later: + +- low IoU matches +- confidence vs IoU scatter +- spatial error clusters + +## 9. Rapportage + +QA/QC-output moet exporteerbaar zijn als: + +- GeoJSON +- JSON summary +- later PDF/HTML report + +## 10. Belangrijk + +GRB is een referentie, geen perfecte waarheid. UI moet spreken over “referentievergelijking” of “ground-truth proxy”, niet absolute waarheid. diff --git a/docs/QA_QC_SPECIFICATION.md b/docs/QA_QC_SPECIFICATION.md new file mode 100644 index 00000000..91590be5 --- /dev/null +++ b/docs/QA_QC_SPECIFICATION.md @@ -0,0 +1,176 @@ +# GeoIntel Kempen — QA/QC Specification v1.0 + +QA/QC is a core showcase feature. It must compare model-derived geospatial outputs against reference data such as GRB or user-supplied labels. + +## Primary use case + +Compare AI building detections or segmentations against GRB building footprints. + +## Inputs + +### Prediction layer + +- Polygons converted from object detections or segmentation masks. +- Required fields: `id`, `class_name`, `confidence`, `geometry`, `analysis_run_id`. + +### Reference layer + +- GRB building polygons, OSM buildings, or user annotation polygons. +- Required fields: `id`, `class_name` or canonical class, `geometry`, `source`. + +## Geometry preparation + +1. Reproject both layers to the metric CRS. +2. Repair invalid geometries. +3. Clip both layers to the analysis area. +4. Optionally simplify only for visualization, not for metric calculation. +5. Filter classes if class-specific QA is requested. + +## Matching strategy + +Default matching is polygon IoU-based matching. + +### IoU + +```text +IoU = area(intersection(prediction, reference)) / area(union(prediction, reference)) +``` + +### Match threshold + +Default: `IoU >= 0.50`. + +Additional thresholds for reporting: + +- lenient: `0.30` +- standard: `0.50` +- strict: `0.75` + +### Matching algorithm + +1. Build spatial index. +2. For every prediction, find intersecting reference candidates. +3. Calculate IoU for candidates. +4. Select the highest IoU candidate. +5. Enforce one-to-one matching: one reference can match only one prediction. +6. Resolve conflicts by highest IoU, then highest confidence. + +## Classification of outcomes + +| Outcome | Definition | +|---|---| +| True Positive | prediction matched to a reference above threshold | +| False Positive | prediction not matched to any reference | +| False Negative | reference not matched by any prediction | +| Low-IoU Match | prediction overlaps reference but below selected threshold | +| Class Mismatch | geometry match but class differs | + +## Metrics + +```text +precision = TP / (TP + FP) +recall = TP / (TP + FN) +f1 = 2 * precision * recall / (precision + recall) +mean_iou = average IoU of matched pairs +``` + +Also calculate: + +- false positive count +- false negative count +- low confidence false positives +- confidence distribution by outcome +- area-weighted recall + +## Outputs + +### QA summary JSON + +```json +{ + "threshold": 0.5, + "true_positive_count": 120, + "false_positive_count": 8, + "false_negative_count": 13, + "precision": 0.9375, + "recall": 0.9022, + "f1": 0.9195, + "mean_iou": 0.72, + "class_name": "building" +} +``` + +### QA layers + +- `qa_true_positives` +- `qa_false_positives` +- `qa_false_negatives` +- `qa_low_iou_matches` +- `qa_match_lines` connecting prediction centroids to reference centroids + +### QA dashboard cards + +- Precision +- Recall +- F1 +- Mean IoU +- False positives +- False negatives +- Top issue areas + +## UI requirements + +The QA/QC page must show: + +1. Reference layer selector. +2. Prediction layer selector. +3. Class filter. +4. IoU threshold slider. +5. Run QA button. +6. Metrics cards. +7. Map with color-coded TP/FP/FN layers. +8. Table of individual findings. +9. Export buttons for GeoJSON and CSV. + +## V1 implementation target + +V1 must support polygon-vs-polygon QA for building detections against GRB or OSM building footprints. Raster mask QA and pixel-level IoU can be added later. + +## Sprint 12 golden QA/QC benchmark + +Sprint 12 adds a deterministic benchmark package for regression detection: + +- `fixtures/golden/reference_buildings.geojson` +- `fixtures/golden/predicted_buildings.geojson` +- `fixtures/golden/expected_qa_metrics.json` +- `scripts/run_golden_qa_benchmark.py` + +The benchmark uses two reference building polygons and two candidate building polygons: + +- one candidate polygon matches one reference polygon at IoU `0.8339768339761133`; +- one candidate polygon is a false positive; +- one reference polygon is a false negative. + +Expected baseline at IoU threshold `0.5`: + +```json +{ + "matches": 1, + "false_positive_count": 1, + "false_negative_count": 1, + "precision": 0.5, + "recall": 0.5, + "f1": 0.5, + "mean_iou": 0.8339768339761133 +} +``` + +Run from the repository root: + +```bash +python scripts/run_golden_qa_benchmark.py +``` + +The script uses the existing `QaService` and `QualityService` paths. It fails if metrics drift outside the documented tolerance and verifies that one `QualityCheck` plus metric rows for precision, recall, F1, mean IoU, false positives and false negatives are produced. + +This benchmark is explicit fixture/demo data only. It does not require live providers, real AI models, Docker or PostGIS. diff --git a/docs/QUEUE_ARCHITECTURE.md b/docs/QUEUE_ARCHITECTURE.md new file mode 100644 index 00000000..3fdd07c2 --- /dev/null +++ b/docs/QUEUE_ARCHITECTURE.md @@ -0,0 +1,42 @@ +# Queue Architecture + +## Decision + +Use Redis + RQ for V1 background jobs. + +## Why + +- Simple to run locally. +- Easy to understand. +- Good enough for raster processing and AI inference jobs. +- Avoids Celery complexity in the first implementation. + +## Queue names + +- `default` for lightweight jobs. +- `processing` for raster/vector processing. +- `ai` for detection and segmentation. +- `exports` for GeoJSON/report exports. + +## Job lifecycle + +1. API validates request. +2. API creates `analysis_run` with status `queued`. +3. API enqueues job with `analysis_run_id`. +4. Worker sets status `running`. +5. Worker writes artifacts and metrics. +6. Worker sets status `completed` or `failed`. +7. Frontend polls analysis run endpoint. + +## Failure handling + +Failures must store: + +- error code, +- error message, +- stack trace in internal logs only, +- user-safe explanation. + +## No silent failures + +A failed job must be visible in the UI and queryable through the API. diff --git a/docs/RASTER_OPERATIONS_SPEC.md b/docs/RASTER_OPERATIONS_SPEC.md new file mode 100644 index 00000000..b2418b83 --- /dev/null +++ b/docs/RASTER_OPERATIONS_SPEC.md @@ -0,0 +1,255 @@ +# GeoIntel Kempen — Raster Operations Specification v1.0 + +Raster operations are implemented with Rasterio, GDAL-compatible tooling, NumPy and optional OpenCV. + +## Global requirements + +- Never assume band order without metadata or explicit user selection. +- Preserve georeferencing for all derived rasters. +- Use Cloud Optimized GeoTIFF when generating large outputs if practical. +- Store previews separately from analytical rasters. +- Keep nodata values and masks explicit. + +## Required V1 operations + +## 1. Metadata extraction + +Input: raster file. + +Output: + +```json +{ + "driver": "GTiff", + "width": 4096, + "height": 4096, + "band_count": 4, + "crs": "EPSG:31370", + "bounds": [xmin, ymin, xmax, ymax], + "resolution": [0.25, 0.25], + "dtype": ["uint8"], + "nodata": [null], + "transform": [..] +} +``` + +## 2. Band statistics + +For each band: + +- min +- max +- mean +- std +- nodata ratio +- histogram bins +- nodata count +- valid pixel count +- dtype +- optional histogram bins approximation + +## 3. Preview generation + +Generate browser-friendly previews: + +- PNG preview for full raster or overview. +- Optional tile pyramid later. +- Support contrast stretch. + +## 4. Clip by polygon + +Input: + +- raster dataset +- area polygon + +Output: + +- clipped GeoTIFF +- metadata +- preview + +## 5. Reproject + +Input: + +- raster dataset +- target CRS +- target CRS default: `EPSG:31370` when not explicitly set in request +- resampling method: nearest, bilinear, cubic + +Output: + +- reprojected GeoTIFF + +Failure behavior: + +- if rasterio is unavailable, return dependency-aware error (`RASTER_PROCESSING_UNAVAILABLE`) +- if source CRS is missing, return invalid CRS error + +## 6. Tile generation for AI + +Input: + +- raster dataset +- tile size, default 512 (configured in runtime defaults) +- overlap, default 64 +- optional area polygon + +Output: + +```json +{ + "tile_set_id": "uuid", + "source_dataset_id": "uuid", + "source_raster_id": "uuid", + "bounds": [xmin, ymin, xmax, ymax], + "tile_size": 512, + "overlap": 64, + "parameters": {}, + "count": 16, + "tile_paths": ["storage/tiles/.../tile_0001.tif"], + "tiles": [ + { + "path": "storage/tiles/.../tile_0001.tif", + "pixel_window": [x, y, width, height], + "bounds": [xmin, ymin, xmax, ymax], + "transform": [] + } + ], + "ai_inference": false, + "tile_server": null +} + +``` + +Failure behavior: + +- if rasterio is unavailable: dependency-aware error +- if no raster tiles are generated: empty-result failure + +## 7. Index calculations + +### NDVI + +Required inputs: + +- red band index +- NIR band index +- output name (optional) +- formula: `(nir - red) / (nir + red)` +- nodata/NaN handling: invalid pixels and division-by-zero values are written as `NaN` +- output dtype: `float32` +- provenance fields: + - `source_dataset_id` + - `operation` + - `band_mapping` + - `formula` + - `output_dtype` + - `nodata_strategy` + - `value_range_note` +- failure behavior: + - band indices must be positive integers + - band indices must exist in source dataset + - `RASTER_PROCESSING_UNAVAILABLE` when rasterio/numpy missing + +Formula: + +```text +NDVI = (NIR - Red) / (NIR + Red) +``` + +### NDWI + +Required inputs: + +- NIR band index +- green band index +- output name (optional) +- formula: `(nir - green) / (nir + green)` +- same provenance strategy as NDVI + +Formula: + +```text +NDWI = (NIR - Green) / (NIR + Green) +``` + +### NDBI + +Required inputs: + +- SWIR band index +- NIR band index +- output name (optional) +- formula: `(swir - nir) / (swir + nir)` +- same provenance strategy as NDVI + +Formula: + +```text +NDBI = (SWIR - NIR) / (SWIR + NIR) +``` + +Derived metadata contract: + +- `operation`: one of `raster.ndvi`, `raster.ndwi`, `raster.ndbi` +- `source_dataset_id` +- `band_mapping` +- `formula` +- `output_dtype` +- `nodata_strategy` +- `value_range_note` +- `created_at` +- `output_dataset_id` +- `path` + +## 8. Threshold to mask + +Input: + +- raster/index +- threshold operator +- threshold value + +Output: + +- binary mask raster +- optional vector polygons + +## Storage output policy + +Derived rasters go to: + +```text +storage/rasters/derived/{project_id}/{dataset_id}/ +``` + +Tiles go to: + +```text +storage/tiles/{project_id}/{dataset_id}/{tile_set_id}/ +``` + +Previews go to: + +```text +storage/previews/{project_id}/{dataset_id}/ +``` + +## API shape + +```http +GET /datasets/{id}/raster/metadata +POST /datasets/{id}/raster/clip +POST /datasets/{id}/raster/reproject +POST /datasets/{id}/raster/tile +POST /datasets/{id}/raster/indices/ndvi +POST /datasets/{id}/raster/indices/ndwi +POST /datasets/{id}/raster/indices/ndbi +POST /datasets/{id}/raster/threshold +``` + +## V1 target + +V1 must implement metadata, preview, clip and tile generation. NDVI/NDWI/NDBI are implemented on local raster files with explicit band selection. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..42d8fc93 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,49 @@ +# GeoIntel Docs Index + +Start here when preparing an implementation pass. + +## Product foundation +- PRODUCT_VISION.md +- PRODUCT_BLUEPRINT.md +- SPECIFICATION_FREEZE_M0.md +- V1_SCOPE_FREEZE.md +- M0_HANDOFF_SUMMARY.md + +## Data and processing +- DATASET_STRATEGY.md +- DATA_CATALOG.md +- DATA_SPECIFICATION.md +- DATA_SOURCES.md +- RASTER_OPERATIONS_SPEC.md +- VECTOR_OPERATIONS_SPEC.md +- DETECTION_PIPELINE_SPEC.md +- SEGMENTATION_PIPELINE_SPEC.md +- CHANGE_DETECTION_SPEC.md +- ANALYSIS_SPECIFICATIONS.md +- QA_QC_SPECIFICATION.md +- FIXTURE_STRATEGY.md + +## Architecture +- ARCHITECTURE.md +- SERVICE_ARCHITECTURE.md +- DATABASE_SCHEMA.md +- API_SPECIFICATION.md +- STORAGE_ARCHITECTURE.md +- REPOSITORY_CONVENTIONS.md + +## UI and modules +- MODULES.md +- UI_UX_SPEC.md +- UI_PAGE_SPECIFICATIONS.md +- COMPONENT_BREAKDOWN.md + +## Execution +- CODEX_MASTER_PROMPT.md +- CODEX_EXECUTION_PLAN.md +- CODEX_BUILD_PLAN.md +- CODEX_EXECUTION_LOG.md +- ACCEPTANCE_CRITERIA.md +- TODO.md +- IMPLEMENTATION_BACKLOG.md +- DEVELOPMENT_RULES.md +- AGENTS.md diff --git a/docs/RELEASE_PROCESS.md b/docs/RELEASE_PROCESS.md new file mode 100644 index 00000000..a726fff0 --- /dev/null +++ b/docs/RELEASE_PROCESS.md @@ -0,0 +1,22 @@ +# Release Process + +## Versies +- `v0.1-foundation` +- `v0.2-dataset-manager` +- `v0.3-raster-vector-core` +- `v0.4-detection-lab` +- `v0.5-qaqc-lab` +- `v0.6-change-detection` +- `v1.0-geoai-workbench` + +## Release checklist +- [ ] `CHANGELOG.md` bijgewerkt. +- [ ] Release note toegevoegd. +- [ ] TODO bijgewerkt. +- [ ] Bekende beperkingen bijgewerkt. +- [ ] Quick smoke checks groen. +- [ ] API-contracten niet ongedocumenteerd gewijzigd. +- [ ] Database migrations beschreven. + +## ZIP-regel +Elke milestone krijgt update zip en full zip. diff --git a/docs/REPOSITORY_CONVENTIONS.md b/docs/REPOSITORY_CONVENTIONS.md new file mode 100644 index 00000000..698fa636 --- /dev/null +++ b/docs/REPOSITORY_CONVENTIONS.md @@ -0,0 +1,134 @@ +# Repository Conventions + +## Canonical structure + +```text +GeoIntel/ + backend/ + app/ + api/ + routes/ + deps.py + core/ + config.py + errors.py + logging.py + db/ + session.py + base.py + migrations/ + models/ + schemas/ + services/ + repositories/ + workers/ + providers/ + utils/ + tests/ + pyproject.toml + README.md + + frontend/ + src/ + app/ + pages/ + components/ + features/ + hooks/ + lib/ + services/ + stores/ + types/ + styles/ + package.json + README.md + + docs/ + datasets/ + raw/ + processed/ + cache/ + storage/ + uploads/ + tiles/ + masks/ + reports/ + exports/ + models/ + scripts/ + tests/ + fixtures/ +``` + +## Backend naming +- SQLAlchemy models: singular class names, e.g. `Project`, `Dataset`, `AnalysisRun`. +- Pydantic schemas: `ProjectCreate`, `ProjectRead`, `DatasetRead`. +- Services: `ProjectService`, `RasterService`. +- Repositories: `ProjectRepository`. +- Route files: plural noun, e.g. `projects.py`, `datasets.py`, `analysis_runs.py`. + +## Frontend naming +- Pages end with `Page`, e.g. `DatasetManagerPage.tsx`. +- Components are PascalCase. +- Hooks start with `use`, e.g. `useProjects.ts`. +- API clients live under `src/services/api`. +- Domain types live under `src/types`. +- Feature-specific components live under `src/features/`. + +## Documentation rule +When Codex implements or changes a module, it must update at least one of: +- docs/TODO.md +- docs/IMPLEMENTATION_BACKLOG.md +- docs/CODEX_EXECUTION_LOG.md +- relevant technical spec if a documented decision changes + +## No generated noise +Do not commit: +- node_modules +- .venv +- __pycache__ +- large raw datasets +- model weights unless explicitly requested +- generated tiles/masks/reports except small fixtures + +## Environment variables +All configurable external sources must use environment variables: +- DATABASE_URL +- REDIS_URL +- STORAGE_ROOT +- GRB_WFS_URL +- OSM_OVERPASS_URL +- OPENAI_API_KEY optional later +- MODEL_STORAGE_ROOT + +## Testing conventions +Backend: +- pytest +- fixture datasets under tests/fixtures +- geometry tests must use small artificial polygons + +Frontend: +- typecheck must pass +- lint must pass if configured +- components must handle loading/error/empty states + +## Build conventions +Every build pass should report: +- files changed +- modules completed +- tests run +- known limitations +- next recommended pass + +## Geospatial correctness conventions +- Never compute area/length in EPSG:4326. +- Always store CRS metadata. +- Always warn on missing CRS. +- Never export geospatial outputs from non-georeferenced rasters unless the output is explicitly marked image-space only. + +## Mocking rule +Mock providers are allowed only when: +- they are named `Development...Provider` or `Fixture...Provider`; +- they are clearly isolated; +- UI labels say development/demo provider; +- real provider interfaces are already defined. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 00000000..d28a8bb3 --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,88 @@ +# Roadmap + +## V0.1 — Foundation + +- repo scaffold +- FastAPI backend +- React frontend +- PostGIS setup +- project model +- area model +- basic map viewer + +## V0.2 — Dataset Manager + +- raster upload +- vector upload +- metadata extraction +- storage structure +- dataset status + +## V0.3 — Raster & Vector Core + +- raster clipping +- raster tiling +- vector import +- vector clipping +- CRS handling +- geometry validation + +## V0.4 — Reference Data + +- OSM fallback fetcher +- GRB integration design +- PostGIS reference cache +- reference layer viewer + +## V0.5 — Object Detection + +- YOLO service wrapper +- tile inference +- geospatial reprojection +- detections table +- detection map layer +- GeoJSON export + +## V0.6 — QA/QC + +- compare detections with reference layer +- IoU +- precision/recall/F1 +- false positive/negative layers + +## V0.7 — Segmentation + +- segmentation service +- masks +- polygonize +- area metrics + +## V0.8 — Change Detection + +- compare analysis runs +- added/removed/changed objects +- change map layer + +## V0.9 — Annotation Studio + +- bbox/polygon annotation +- class labels +- export YOLO/COCO/GeoJSON + +## V1.0 — Portfolio Release + +- stable demo workflow +- docs complete +- tests green +- sample project +- export package +- portfolio README section + +## Later + +- Sentinel indices +- DHMV hoogte +- LiDAR Workbench +- Training Studio +- model registry +- QGIS integration diff --git a/docs/ROLLBACK_AND_RECOVERY.md b/docs/ROLLBACK_AND_RECOVERY.md new file mode 100644 index 00000000..060737bf --- /dev/null +++ b/docs/ROLLBACK_AND_RECOVERY.md @@ -0,0 +1,13 @@ +# Rollback and Recovery + +## Basisregels +Maak per milestone een full zip en update zip. Geen destructieve migrations zonder rollback-notitie. + +## Database rollback +Elke migration beschrijft upgrade en downgrade. Als downgrade onveilig is, noteer dit expliciet. + +## Storage rollback +Bestanden worden niet stilzwijgend verwijderd. Gebruik soft-delete waar mogelijk. + +## AI/model rollback +Elke analysis run bewaart model id, version, parameters, confidence threshold en pipeline version. diff --git a/docs/SECURITY_AND_DATA_BOUNDARIES.md b/docs/SECURITY_AND_DATA_BOUNDARIES.md new file mode 100644 index 00000000..f0068d31 --- /dev/null +++ b/docs/SECURITY_AND_DATA_BOUNDARIES.md @@ -0,0 +1,35 @@ +# Security and Data Boundaries + +V1 is a local/single-user portfolio application. Still, it must avoid unsafe defaults. + +## Upload safety + +- Enforce upload size limit. +- Store uploads outside source directories. +- Sanitize filenames. +- Do not execute uploaded files. +- Validate file type by content where possible, not only extension. + +## Path safety + +- All storage paths must be resolved under `STORAGE_ROOT`. +- No user-controlled absolute paths. +- No `../` traversal. + +## Secrets + +- No API keys in git. +- Use `.env.example` only. +- External service credentials stay in environment variables. + +## Network calls + +- External data fetchers must have timeouts. +- Cache results where appropriate. +- Surface failures clearly. + +## AI model execution + +- Do not download arbitrary model weights at runtime without explicit configuration. +- Model paths must be configured or stored under `storage/models`. +- If unavailable, return `not_configured`. diff --git a/docs/SECURITY_CHECKLIST.md b/docs/SECURITY_CHECKLIST.md new file mode 100644 index 00000000..b7926201 --- /dev/null +++ b/docs/SECURITY_CHECKLIST.md @@ -0,0 +1,14 @@ +# Security Checklist + +## Verplicht +- Geen secrets committen. +- `.env.example` zonder echte keys. +- Uploadvalidatie op extensie/MIME waar mogelijk. +- Bestandspaden normaliseren; geen path traversal. +- Geen stacktraces naar frontend. +- CORS expliciet. +- Maximale uploadgrootte. +- Lange jobs via queue. + +## Uploads V1 +Toegestaan: GeoTIFF/TIFF, GeoJSON/JSON, GPKG later, Shapefile alleen als gevalideerde zip later. diff --git a/docs/SEED_DATA_PLAN.md b/docs/SEED_DATA_PLAN.md new file mode 100644 index 00000000..72afe90e --- /dev/null +++ b/docs/SEED_DATA_PLAN.md @@ -0,0 +1,57 @@ +# Seed Data Plan + +Seed data exists only to make the local developer experience and Codex implementation deterministic. + +## Seed data rules + +- Seed data must be clearly labeled as demo or fixture data. +- Seed data must never be presented as real GRB, OSM or official source data unless it actually comes from that source. +- Fixture geometries should be small and simple. +- Fixtures should support automated tests. + +## Required seed entities + +### Project + +```json +{ + "name": "Demo — Geel Building QA", + "description": "Fixture project for building detection and QA/QC workflow." +} +``` + +### Area + +A small polygon near the Kempen demo viewport. + +### Reference buildings + +Use `fixtures/geojson/reference_buildings_fixture.geojson`. + +### Predicted buildings + +Use `fixtures/geojson/predicted_buildings_fixture.geojson`. + +## Required seed command + +Codex should add a command similar to: + +```bash +python -m app.scripts.seed_demo +``` + +or a Make target: + +```bash +make seed-demo +``` + +## Seed acceptance + +After running seed: + +- one project exists +- one area exists +- reference building dataset exists +- predicted building dataset or analysis output exists +- QA/QC can run against the two fixture layers diff --git a/docs/SEGMENTATION_CLASSES_CATALOG.md b/docs/SEGMENTATION_CLASSES_CATALOG.md new file mode 100644 index 00000000..b9e746cd --- /dev/null +++ b/docs/SEGMENTATION_CLASSES_CATALOG.md @@ -0,0 +1,29 @@ +# Segmentation Classes Catalog + +## V1/V2 target classes + +### vegetation +Used for green cover analysis and NDVI comparison. + +### water +Used for water surface extraction and NDWI comparison. + +### building +Used for footprint segmentation if a suitable model is available. + +### impervious_surface +Used for paved/urban surface estimation. + +## Output requirements + +Segmentation outputs must support: + +- raster mask path, +- polygonized vector output, +- area per class, +- confidence or quality score when available, +- overlay visualization. + +## Rule + +Segmentation outputs must become geospatial layers, not just image overlays. diff --git a/docs/SEGMENTATION_PIPELINE_SPEC.md b/docs/SEGMENTATION_PIPELINE_SPEC.md new file mode 100644 index 00000000..a6297915 --- /dev/null +++ b/docs/SEGMENTATION_PIPELINE_SPEC.md @@ -0,0 +1,126 @@ +# GeoIntel Kempen — Segmentation Pipeline Specification v1.0 + +The segmentation pipeline converts raster imagery into georeferenced masks and polygons. + +## Goal + +Produce class masks for buildings, vegetation, water, or other targets and convert those masks to GIS layers. + +## Model families + +V1/early: + +- YOLO segmentation model if available. +- Segment Anything for prompt-based or automatic masks. + +Later: + +- U-Net +- DeepLab +- custom PyTorch semantic segmentation + +## Pipeline + +```text +Raster dataset +↓ +Raster validation +↓ +Tile generation +↓ +Segmentation inference +↓ +Mask stitching +↓ +Georeferenced mask output +↓ +Polygonization +↓ +Geometry cleanup +↓ +PostGIS storage +↓ +Map overlay + statistics +``` + +## Mask types + +### Instance segmentation + +Each object has its own mask. + +### Semantic segmentation + +Each pixel has a class value. + +V1 may support instance segmentation first. + +## Output storage + +- Raw mask files: `storage/masks/{project_id}/{analysis_run_id}/` +- Polygonized outputs in PostGIS `segmentations` table. +- Derived vector layer for map rendering. + +## Polygonization rules + +1. Convert binary/class mask to shapes using raster transform. +2. Discard polygons below minimum area threshold. +3. Repair geometries. +4. Simplify for display only, keep analysis geometry if possible. +5. Calculate area. + +Default thresholds: + +```yaml +minimum_area_m2: 1.0 +simplify_tolerance_m: 0.10 +``` + +## Required segmentation record + +```yaml +analysis_run_id +class_name +geometry +area_m2 +confidence +raster_mask_path +source_tile_id +model_name +metadata_json +``` + +## Metrics + +- total segmented area by class +- polygon count by class +- mean confidence by class +- largest polygon by class +- area ratio against selected area + +## API + +```http +POST /analysis/segmentation +GET /analysis/{id}/segmentations +POST /analysis/{id}/polygonize +POST /analysis/{id}/exports/masks +POST /analysis/{id}/exports/geojson +``` + +## UI + +Segmentation Lab must show: + +- raster selector +- model selector +- segmentation mode +- class list +- threshold controls +- mask preview +- polygon overlay +- export buttons + +## V1 target + +Implement the data structures, job pipeline contracts, mask storage, polygonization utilities and UI shell. Full SAM integration can follow after raster/detection foundation is stable. diff --git a/docs/SERVICE_ARCHITECTURE.md b/docs/SERVICE_ARCHITECTURE.md new file mode 100644 index 00000000..232c424d --- /dev/null +++ b/docs/SERVICE_ARCHITECTURE.md @@ -0,0 +1,213 @@ +# Backend Service Architecture + +## Goal +Define service boundaries so Codex does not mix API controllers, database models, geospatial processing and AI inference into unmaintainable code. + +## Service map + +### ProjectService +Responsibilities: +- create/list/read/update projects +- project summary aggregation +- project-level validation + +Dependencies: +- ProjectRepository +- AreaRepository +- DatasetRepository + +### AreaService +Responsibilities: +- create/read/update/delete project areas +- validate geometry +- calculate area and perimeter in metric CRS +- convert API geometry to persisted PostGIS geometry + +Dependencies: +- GeometryService + +### DatasetService +Responsibilities: +- handle upload metadata +- create dataset records +- dispatch raster/vector import based on detected type +- manage dataset status +- store file paths and checksums + +Dependencies: +- StorageService +- RasterService +- VectorService + +### StorageService +Responsibilities: +- determine storage paths +- save uploaded files +- generate checksums +- create derived artifact paths +- prevent path traversal + +Storage roots: +- storage/uploads +- storage/tiles +- storage/masks +- storage/reports +- exports + +### RasterService +Responsibilities: +- open raster safely +- extract metadata +- clip raster by geometry +- generate tiles for inference +- calculate raster statistics/histograms +- calculate remote-sensing indices later + +Libraries: +- rasterio +- numpy +- GDAL-compatible tooling if needed + +### VectorService +Responsibilities: +- import GeoJSON/shapefile/GPKG where supported +- validate geometry +- fix simple invalid geometry where safe +- reproject +- clip +- buffer +- intersect +- calculate metrics + +Libraries: +- geopandas +- shapely +- pyproj + +### ReferenceLayerService +Responsibilities: +- fetch/cache professional reference layers +- support provider abstraction +- normalize provider output +- store cached layers in PostGIS + +Providers: +- LocalFixtureReferenceProvider +- GrbWfsReferenceProvider later/configured +- OsmOverpassProvider fallback/context + +### DetectionService +Responsibilities: +- create detection analysis run +- prepare raster tiles +- call detection provider +- georeference detections +- merge or suppress duplicate tile-edge detections +- persist detections +- produce layer metadata + +Providers: +- DevelopmentDetectionProvider for deterministic fixtures +- YoloDetectionProvider + +### SegmentationService +Responsibilities: +- create segmentation analysis run +- prepare input raster/tile/mask workflow +- call segmentation provider +- persist mask artifacts +- polygonize masks where possible +- persist segmentation geometries + +Providers: +- DevelopmentSegmentationProvider +- SamSegmentationProvider later +- YoloSegmentationProvider later + +### QaqcService +Responsibilities: +- compare prediction geometries with reference geometries +- calculate IoU and matching +- calculate precision/recall/F1 +- persist quality check summary and findings +- generate layers for matched/false-positive/false-negative outputs + +Dependencies: +- GeometryService +- AnalysisRunRepository + +### ChangeDetectionService +Responsibilities: +- compare baseline and comparison layers +- identify added/removed/changed geometries +- persist change features and metrics + +### ExportService +Responsibilities: +- export vectors as GeoJSON +- export run package with metadata +- export simple Markdown/HTML report +- record export history + +### JobService +Responsibilities: +- enqueue long-running operations +- track status/progress/logs +- support retry/failure status + +V1 may implement synchronous execution first for small fixtures, but service boundaries must allow async jobs later. + +### GeometryService +Responsibilities: +- CRS conversion helpers +- area/length calculations in metric CRS +- geometry validation +- IoU calculation +- spatial matching helpers + +## API layer rule +API routes may: +- validate request schema +- call service methods +- return response schema + +API routes may not: +- perform direct geospatial processing +- access files directly +- contain model inference logic +- contain PostGIS SQL beyond simple repository usage + +## Repository layer rule +Repositories handle database access only. They should not know about Rasterio, YOLO, MapLibre or UI concerns. + +## Error handling +All services must raise typed application errors: +- NotFoundError +- ValidationError +- ProcessingError +- UnsupportedDatasetError +- ExternalProviderError +- GeospatialError + +API layer maps these to proper HTTP responses. + +## Logging +Every analysis run should record: +- start time +- end time +- parameters +- status +- failure reason if failed +- output artifact paths + +## V1 service acceptance +V1 backend foundation is acceptable when the following services exist even if some methods are minimal: +- ProjectService +- AreaService +- DatasetService +- StorageService +- RasterService +- VectorService +- DetectionService +- QaqcService +- ExportService diff --git a/docs/SERVICE_IO_CONTRACTS.md b/docs/SERVICE_IO_CONTRACTS.md new file mode 100644 index 00000000..c2179fde --- /dev/null +++ b/docs/SERVICE_IO_CONTRACTS.md @@ -0,0 +1,47 @@ +# Backend Service IO Contracts + +## ProjectService +`create_project(input: ProjectCreate) -> ProjectRead` +`list_projects() -> list[ProjectRead]` +`get_project(project_id: str) -> ProjectRead` + +## AreaService +`create_area(project_id: str, input: AreaCreate) -> AreaRead` +`list_areas(project_id: str) -> FeatureCollection` +`validate_geometry(geojson: dict) -> GeometryValidationResult` + +## DatasetService +`register_upload(project_id: str, file) -> DatasetRead` +`extract_metadata(dataset_id: str) -> DatasetMetadataResult` +`list_datasets(project_id: str) -> list[DatasetRead]` +`get_dataset(dataset_id: str) -> DatasetRead` + +## RasterService +`read_metadata(path: Path) -> RasterMetadata` +`clip(dataset_id: str, area_id: str) -> JobRead` +`tile(dataset_id: str, parameters: TileParameters) -> JobRead` +`calculate_index(dataset_id: str, index_type: str, bands: dict) -> JobRead` + +## VectorService +`read_metadata(path: Path) -> VectorMetadata` +`clip(dataset_id: str, area_id: str) -> JobRead` +`buffer(dataset_id: str, distance_m: float) -> JobRead` +`intersect(left_dataset_id: str, right_dataset_id: str) -> JobRead` + +## DetectionService +`create_run(project_id: str, input: DetectionRunCreate) -> AnalysisRunRead` +`execute_run(run_id: str) -> DetectionRunResult` + +## SegmentationService +`create_run(project_id: str, input: SegmentationRunCreate) -> AnalysisRunRead` +`execute_run(run_id: str) -> SegmentationRunResult` + +## QAQCService +`create_quality_check(input: QualityCheckCreate) -> QualityCheckRead` +`calculate_iou_matrix(candidate_layer_id: str, reference_layer_id: str) -> IoUMatrix` +`calculate_metrics(matches: list[Match]) -> QualityMetrics` + +## ExportService +`create_export(input: ExportCreate) -> ExportRead` +`write_geojson(layer_id: str) -> Path` +`write_metrics_csv(analysis_run_id: str) -> Path` diff --git a/docs/SPECIFICATION_FREEZE_M0.md b/docs/SPECIFICATION_FREEZE_M0.md new file mode 100644 index 00000000..3c13355c --- /dev/null +++ b/docs/SPECIFICATION_FREEZE_M0.md @@ -0,0 +1,66 @@ +# GeoIntel Kempen — Specification Freeze M0 + +Status: frozen for first Codex build +Purpose: remove architecture ambiguity before implementation + +## 1. Product decision +GeoIntel Kempen is a GeoAI Workbench, not a generic GIS viewer and not a policy-report dashboard. The primary value is the conversion of raster, vector and AI outputs into geospatially correct layers, metrics, QA findings and exports. + +## 2. Region decision +The default operating region is the Belgian Kempen, with Flemish open geodata as the professional reference layer. The initial demo area should be Geel, Mol or Turnhout because these are recognisable Kempen cases with buildings, vegetation, water and infrastructure. + +## 3. First-user decision +The first target user is a technical geospatial analyst or GeoAI engineer candidate. UI copy may be understandable to non-specialists, but implementation must prioritise correct geodata handling, reproducible pipelines and inspectable outputs. + +## 4. V1 priority order +1. Project workspace and area selection. +2. Dataset intake and metadata extraction. +3. Raster and vector workbench foundation. +4. GRB/OSM reference-layer ingestion or caching. +5. Object detection pipeline. +6. Segmentation pipeline. +7. QA/QC against GRB or reference vector layers. +8. Export of outputs as GeoJSON, masks and simple report artifacts. + +## 5. Explicit non-goals for V1 +- No multi-tenant user management. +- No payment, sharing or organisation management. +- No QGIS plugin yet. +- No full LiDAR workbench yet. +- No full model training studio yet. +- No heavy MLOps platform yet. +- No beautiful but empty dashboard replacing actual processing. +- No fake “analysis complete” states without persisted outputs. + +## 6. Data source decisions +- GRB is the first professional Flemish reference source. +- OSM is the fallback and supplementary open context source. +- Sentinel-2 is the first remote-sensing source for NDVI/NDWI/NDBI, but only after raster/vector foundation is stable. +- DHMV/DEM/DSM are later height sources. +- LAS/LAZ point clouds are later, not V1. + +## 7. AI decisions +- V1 may use pre-trained YOLO/Ultralytics models and SAM-style segmentation where practical. +- The system architecture must isolate model inference behind services so models can be replaced later. +- AI results must be georeferenced back into map coordinates. +- AI results must never be treated as ground truth without QA/QC. + +## 8. QA/QC decisions +The signature V1 portfolio feature is comparing AI-generated detections or segmentations against GRB or another reference vector layer. The QA engine must calculate at least: +- matched features +- unmatched predictions +- unmatched reference objects +- IoU distribution +- precision +- recall +- F1 + +## 9. Export decisions +V1 exports must be useful to GIS workflows: +- GeoJSON for vector outputs +- zipped result package with metadata JSON +- mask raster path or exported raster artifact where segmentation is used +- simple HTML or Markdown report summary; advanced PDF can come later + +## 10. Implementation rule +If a choice is not explicitly frozen here, Codex must prefer the simplest implementation that preserves geospatial correctness, testability and later extensibility. diff --git a/docs/SPRINT_BOARD_M4.md b/docs/SPRINT_BOARD_M4.md new file mode 100644 index 00000000..a1c68f43 --- /dev/null +++ b/docs/SPRINT_BOARD_M4.md @@ -0,0 +1,110 @@ +# GeoIntel M4 Sprint Board + +## Sprint 0 — Repo Bootstrapping +- [ ] Verify repo root structure. +- [ ] Add backend package skeleton. +- [ ] Add frontend package skeleton. +- [ ] Ensure Docker Compose starts PostGIS and Redis. +- [ ] Add `.env.example` parity with backend settings. +- [ ] Add smoke command documentation. + +Acceptance: +- A new developer can run the documented startup commands. +- Missing optional GIS binaries degrade gracefully with explicit warnings. + +## Sprint 1 — Backend Foundation +- [ ] FastAPI app factory. +- [ ] Health endpoint. +- [ ] Settings loader. +- [ ] Structured error response middleware. +- [ ] Database session management. +- [ ] Alembic migration baseline. + +Acceptance: +- `/health` returns app, db, redis and storage status. +- Failed validation returns the standard error envelope. + +## Sprint 2 — Domain Models +- [ ] Project model. +- [ ] Area model with PostGIS geometry. +- [ ] Dataset model. +- [ ] AnalysisRun model. +- [ ] Detection model. +- [ ] Segmentation model. +- [ ] Metric model. +- [ ] Export model. + +Acceptance: +- Migration creates all core tables. +- Geometry columns use SRID 31370 internally where applicable. +- API schemas never expose internal file-system paths unless explicitly an export path. + +## Sprint 3 — Dataset Manager +- [ ] Dataset upload endpoint. +- [ ] Dataset metadata extractor dispatcher. +- [ ] Raster metadata extractor. +- [ ] Vector metadata extractor. +- [ ] Dataset list/detail endpoints. +- [ ] Dataset status transitions. + +Acceptance: +- Uploading GeoJSON fixture produces vector metadata. +- Uploading placeholder/demo raster either extracts metadata or returns a clear unsupported-fixture status. + +## Sprint 4 — Area & Map Workspace +- [ ] Create area from polygon. +- [ ] Validate area geometry. +- [ ] Calculate area in square meters. +- [ ] Return areas as GeoJSON. +- [ ] Frontend map shell. +- [ ] Draw polygon flow. + +Acceptance: +- A user can create a project and add at least one polygon area through the UI. + +## Sprint 5 — Raster/Vector Labs +- [ ] Raster metadata page. +- [ ] Vector metadata page. +- [ ] Vector clip operation. +- [ ] Raster operation contract endpoints. +- [ ] Operation job creation. + +Acceptance: +- Vector fixture clipped to an area returns a new derived layer record. + +## Sprint 6 — AI Pipeline Shells +- [ ] Model registry seed. +- [ ] Detection run endpoint. +- [ ] Segmentation run endpoint. +- [ ] Deterministic demo-mode inference. +- [ ] Store detections/segmentations as geometry outputs. + +Acceptance: +- Detection run on demo area creates predictable detections with classes and confidence. +- Segmentation run creates polygons and area metrics. + +## Sprint 7 — QA/QC +- [ ] Reference dataset selection. +- [ ] IoU matching. +- [ ] Precision/recall/F1. +- [ ] False positive/false negative outputs. +- [ ] QA dashboard panel. + +Acceptance: +- Demo detections compared to demo reference produce stable metrics. + +## Sprint 8 — Exports +- [ ] GeoJSON export. +- [ ] CSV metrics export. +- [ ] Report stub export. +- [ ] Export list/detail page. + +Acceptance: +- User can download detection results as GeoJSON. + +## Sprint 9 — Codex RC Pass +- [ ] Run tests. +- [ ] Update TODO. +- [ ] Update CHANGELOG. +- [ ] Remove dead placeholders. +- [ ] Verify all pages have empty/loading/error/success states. diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md new file mode 100644 index 00000000..77193478 --- /dev/null +++ b/docs/STORAGE_ARCHITECTURE.md @@ -0,0 +1,143 @@ +# GeoIntel Kempen — Storage Architecture v1.0 + +GeoIntel stores metadata in PostgreSQL/PostGIS and binary/geospatial files on filesystem storage or object storage. + +## Principles + +- Database stores metadata, relationships and vector geometries. +- Filesystem/object storage stores original rasters, derived rasters, tiles, masks, reports and model artifacts. +- Every stored file must have a dataset/export/model record in the database. +- Never store large raster binary data directly in regular application tables in V1. + +## Root storage layout + +```text +storage/ + uploads/ + {project_id}/ + rasters/ + vectors/ + lidar/ + rasters/ + derived/ + {project_id}/{dataset_id}/ + tiles/ + {project_id}/{dataset_id}/{tile_set_id}/ + masks/ + {project_id}/{analysis_run_id}/ + previews/ + {project_id}/{dataset_id}/ + exports/ + {project_id}/ + geojson/ + csv/ + reports/ + coco/ + yolo/ + models/ + detection/ + segmentation/ + training-runs/ + cache/ + grb/ + osm/ + sentinel/ + dhmv/ +``` + +## Upload policy + +When a file is uploaded: + +1. Save original file unchanged. +2. Compute checksum. +3. Extract metadata. +4. Create dataset record. +5. Create preview if applicable. + +Required file metadata: + +```yaml +path +original_filename +mime_type +size_bytes +checksum_sha256 +created_at +storage_backend +``` + +## Derived data policy + +Derived files must record: + +- source dataset id(s) +- analysis run id or processing job id +- processing parameters +- software component version +- created_at + +## Segmentation mask artifacts + +Sprint 9 stores segmentation masks as filesystem artifacts and segmentation polygons as authoritative PostGIS records. + +Default mask path convention: + +```text +storage/masks/{project_id}/{analysis_run_id}/tile_{tile_index}/mask_{segmentation_id}.png +``` + +Optional run manifest convention: + +```text +storage/masks/{project_id}/{analysis_run_id}/manifest.json +``` + +Mask files are provenance/debug artifacts. QA, map display and GeoJSON output must use persisted `segmentations.geometry` rather than mask files. + +## Cleanup policy + +Do not delete originals automatically. Derived outputs may be cleaned through explicit cache management. + +## Model storage + +Model artifacts live under: + +```text +storage/models/ +``` + +Database model registry records: + +```yaml +model_id +name +task_type +framework +path +classes_json +version +created_at +metrics_json +``` + +## Exports + +Every export is reproducible and linked to project/analysis. + +Export record fields: + +```yaml +id +project_id +analysis_run_id +export_type +path +format +created_at +parameters_json +``` + +## Local development default + +Use local filesystem paths. Keep MinIO/object storage as future extension. diff --git a/docs/TEST_CATALOG.md b/docs/TEST_CATALOG.md new file mode 100644 index 00000000..594ec71f --- /dev/null +++ b/docs/TEST_CATALOG.md @@ -0,0 +1,70 @@ +# Test Catalog + +## Backend tests + +### Project tests + +- Create project. +- List projects. +- Read project detail. +- Reject invalid project payload. + +### Area tests + +- Create valid polygon area. +- Reject invalid geometry. +- Calculate area in square meters. +- Return GeoJSON. + +### Dataset tests + +- Register dataset. +- Upload allowed file type. +- Reject unsupported file type. +- Extract vector metadata. +- Extract raster metadata where fixture is available. + +### Raster tests + +- Read CRS. +- Read bounds. +- Clip raster by area. +- Generate tiles. + +### Vector tests + +- Import GeoJSON. +- Reproject geometry. +- Clip vector by area. +- Calculate area/length. + +### Detection tests + +- Create detection run. +- Store detection geometry. +- Filter detections by confidence. +- Export detections to GeoJSON. + +### QA/QC tests + +- Match predicted and reference polygons. +- Calculate IoU. +- Calculate precision. +- Calculate recall. +- Calculate F1. +- Identify false positives. +- Identify false negatives. + +## Frontend tests + +- App shell renders. +- Project list loads from API. +- Dataset page handles loading/empty/error states. +- Map page loads fixture geometry. +- Detection results display confidence and classes. + +## Smoke tests + +- Docker compose starts core services. +- Backend health endpoint returns OK. +- Frontend can call backend health endpoint. diff --git a/docs/TEST_STRATEGY.md b/docs/TEST_STRATEGY.md new file mode 100644 index 00000000..4636b52c --- /dev/null +++ b/docs/TEST_STRATEGY.md @@ -0,0 +1,85 @@ +# Test Strategy + +GeoIntel is data-heavy. Tests must focus on contracts, geospatial correctness, and regression safety. + +## Test layers + +### 1. Unit tests + +Backend: + +- Geometry validation. +- Area calculation. +- Dataset metadata extraction wrappers. +- Analysis formulas. +- QA/QC metrics. + +Frontend: + +- API client methods. +- Basic component rendering. +- Empty/loading/error states. + +### 2. Integration tests + +- FastAPI endpoint + database session. +- Project CRUD. +- Area creation with valid GeoJSON. +- Dataset metadata refresh with fixture files. +- Vector clipping using small fixture polygons. +- QA/QC using fixture detection/reference polygons. + +### 3. Smoke tests + +Minimum commands: + +```bash +# backend +pytest + +# frontend +npm run build +npm run lint +``` + +### 4. Geospatial correctness tests + +Use small deterministic fixtures: + +- One AOI polygon. +- Three building polygons. +- Two detection polygons. +- Known intersection results. +- Expected precision/recall/F1. + +No huge geospatial datasets in the repo. + +## Fixtures + +Place lightweight fixtures in: + +```text +tests/fixtures/ + geojson/ + rasters/ + vectors/ +``` + +Large datasets are not committed. Document download/setup in `docs/FIXTURE_STRATEGY.md`. + +## Required V1 acceptance tests + +- Create project. +- Create area from GeoJSON. +- Upload or register vector fixture. +- Extract vector metadata. +- Calculate building stats. +- Create fake detection fixture through service/test-only path. +- Run QA/QC against reference polygons. +- Export QA output as GeoJSON or JSON. + +## Regression rules + +- Do not remove existing tests to make a pass green. +- When an endpoint changes, update API contract and frontend client together. +- If a feature is stubbed because dependency is unavailable, it must return an explicit `not_configured` state, not silent success. diff --git a/docs/TODO.md b/docs/TODO.md new file mode 100644 index 00000000..8406a945 --- /dev/null +++ b/docs/TODO.md @@ -0,0 +1,283 @@ +# GeoIntel TODO + +This file now starts with the current implementation status. Older preparation/backlog sections are preserved below as historical planning context and should not be treated as the live sprint board without checking `docs/CODEX_EXECUTION_LOG.md`. + +## Release hardening status + +- [x] Remove Python `datetime.utcnow()` deprecation warnings from backend service paths. +- [x] Split frontend production build into app, React vendor and MapLibre vendor chunks. +- [x] Enforce Python deprecation warnings as release-readiness failures. +- [x] Fix Docker backend package install order and remove mandatory root `.env` dependency. +- [x] Add Docker build context ignores for backend and frontend. +- [ ] Run Docker/PostGIS live validation on a machine where Docker is available. + +## Current implementation status + +- [x] Backend FastAPI foundation, health endpoint and service structure. +- [x] React/TypeScript frontend foundation and MapLibre workbench. +- [x] SQLAlchemy/PostGIS ORM models and Alembic migration chain through Sprint 9. +- [x] Dataset upload, storage metadata and vector feature persistence. +- [x] Raster metadata and raster operation service boundaries. +- [x] Vector operation service boundaries and fixture-backed tests. +- [x] Provider registry skeleton for GRB, OSM, manual and fixture providers. +- [x] Detection Lab foundation, persistence, GeoJSON output and QA integration. +- [x] Configured-YOLO optional dependency strategy and local preflight. +- [x] Segmentation Lab foundation, persistence, GeoJSON output and QA integration. +- [x] QA/QC golden benchmark fixtures and script. +- [x] Explicit offline demo workflow seed for project, AOI, fixture datasets and persisted QA metrics. +- [x] Project-scoped QA/QC result listing and frontend QA/QC Results panel. +- [x] Persisted export foundation for vector/detection/segmentation GeoJSON and project metadata JSON. +- [x] Lightweight HTML project report artifact export. +- [x] Browser-facing demo/export workflow smoke script. +- [ ] Live Docker/PostGIS validation in this execution environment. +- [ ] Real YOLO compatibility smoke with optional AI extras and local model file. +- [ ] Further frontend state/module decomposition beyond Sprint 10 extraction. + +## Sprint 8 status + +- [x] Detection foundation ORM and migration +- [x] Detection model registry capability stubs +- [x] Detection run service boundary +- [x] Detection API foundation +- [x] Detection Lab UI foundation +- [x] Segmentation Lab foundation +- [x] Configured YOLO local preflight +- [ ] Real YOLO/PyTorch model compatibility smoke + +## 0. Repository Foundation + +- [x] Repo mappenstructuur voorbereiden +- [x] Productdocumentatie voorbereiden +- [x] Architectuurdocumentatie voorbereiden +- [x] Codex build plan voorbereiden +- [x] Masterprompt voorbereiden +- [ ] Init git repository +- [x] Voeg echte backend scaffold toe +- [x] Voeg echte frontend scaffold toe + +## 1. Backend Foundation + +- [x] FastAPI app aanmaken +- [x] Config systeem aanmaken +- [x] Database connectie voorbereiden +- [x] SQLAlchemy models toevoegen +- [x] Alembic migrations toevoegen +- [x] Health endpoint toevoegen +- [x] Tests voor health endpoint toevoegen + +## 2. Database / PostGIS + +- [x] Docker compose met PostgreSQL/PostGIS +- [x] PostGIS extensie activeren +- [x] projects tabel +- [x] areas tabel +- [x] datasets tabel +- [x] analysis_runs tabel +- [x] detections tabel +- [x] quality_checks tabel +- [x] exports tabel +- [x] spatial indexes + +## 3. Frontend Foundation + +- [x] React + TypeScript scaffold +- [x] Routing +- [x] Layout met sidebar +- [x] API client +- [x] Project pages +- [x] Map Workbench basis + +## 4. Project & Area API + +- [ ] POST /projects +- [ ] GET /projects +- [ ] GET /projects/{id} +- [ ] POST /projects/{id}/areas +- [ ] GET /projects/{id}/areas +- [ ] GeoJSON validatie + +## 5. Dataset Manager + +- [ ] Upload endpoint +- [ ] Storage paths +- [ ] Raster metadata extraction +- [ ] Vector metadata extraction +- [ ] Dataset list UI +- [ ] Dataset detail UI + +## 6. Raster Core + +- [ ] Rasterio metadata reader +- [ ] Raster preview generation +- [ ] Clip raster by area +- [ ] Tile raster by area +- [ ] Save tile metadata + +## 7. Vector Core + +- [ ] GeoPandas importer +- [ ] CRS detection +- [ ] CRS transformation +- [ ] Geometry validation +- [ ] Clip vector by area +- [ ] Store features in PostGIS + +## 8. Reference Data + +- [ ] OSM fetcher als fallback +- [ ] GRB integration research verwerken in code +- [ ] Reference dataset cache +- [ ] Reference layer viewer + +## 9. Detection Lab + +- [ ] YOLO wrapper +- [ ] Model config +- [ ] Inference job +- [ ] Pixel bbox naar geo polygon +- [ ] Detections opslaan +- [ ] Detection UI +- [ ] GeoJSON export + +## 10. QA/QC Lab + +- [ ] Spatial matching +- [ ] IoU berekening +- [ ] Precision/recall/F1 +- [ ] False positive layer +- [ ] False negative layer +- [x] QA dashboard +- [x] QA export foundation + +## 11. Segmentation Lab + +- [x] Segmentation service design +- [x] Mask artifact path convention +- [ ] Polygonize masks +- [x] Segmentatiekaartlaag + +## 12. Change Detection + +- [ ] Compare two runs +- [ ] Added/removed objects +- [ ] Change stats +- [ ] Change layer + +## 13. Tests + +- [x] Unit tests GIS helpers +- [x] API tests +- [x] DB/migration smoke tests +- [x] Raster fixture tests +- [x] Vector fixture tests +- [x] QA/QC tests + +## 14. Portfolio Release + +- [x] Demo dataset voorbereiden +- [x] Demo workflow documenteren +- [ ] Screenshots toevoegen +- [ ] README portfolio sectie +- [ ] Full smoke test + +# Repo preparation additions + +- [x] Add Data Catalog. +- [x] Add Analysis Specifications. +- [x] Add QA/QC Specification. +- [x] Add Raster Operations Specification. +- [x] Add Vector Operations Specification. +- [x] Add Detection Pipeline Specification. +- [x] Add Segmentation Pipeline Specification. +- [x] Add Change Detection Specification. +- [x] Add UI Page Specifications. +- [x] Add Storage Architecture. +- [x] Add Demo Scenarios. +- [x] Add Development Rules. +- [x] Add Codex phase prompts. + +# Recommended first Codex build sequence + +- [ ] Phase 1: Backend foundation using `docs/CODEX_PHASE_1_PROMPT.md`. +- [ ] Phase 2: Dataset Manager using `docs/CODEX_PHASE_2_PROMPT.md`. +- [ ] Phase 3: Detection + QA/QC skeleton using `docs/CODEX_PHASE_3_PROMPT.md`. + +## M2 Engineering Package + +- [x] Add ADR decision records. +- [x] Add RFC placeholders for future modules. +- [x] Add API/database/event contracts. +- [x] Add model registry and class catalogs. +- [x] Add queue architecture. +- [x] Add acceptance matrix and test catalog. +- [x] Add Codex M2 build prompts. +- [ ] Start Codex Pass 01 backend foundation. + +# M4 Autonomous Build Readiness + +- [x] Add M4 autonomous build readiness document. +- [x] Add M4 sprint board. +- [x] Add module build contracts. +- [x] Add acceptance test catalog. +- [x] Add API example responses. +- [x] Add job lifecycle contract. +- [x] Add frontend state and route contracts. +- [x] Add backend service IO contracts. +- [x] Add model registry seed specification. +- [x] Add demo fixture manifest. +- [x] Add Codex autonomous runbook. +- [x] Add Codex pass prompts. +- [x] Add Geel demo fixtures. + +# Next M5 Preparation + +- [ ] Add concrete SQL migration snippets for every core table. +- [ ] Add OpenAPI YAML draft. +- [ ] Add frontend component prop contracts. +- [ ] Add backend unit-test skeleton files. +- [ ] Add frontend test skeleton files. +- [ ] Add live data connector research notes with verified endpoints. + +# M5 Operational Readiness Checklist + +- [x] CI/CD specification added. +- [x] Healthcheck contracts added. +- [x] Observability plan added. +- [x] Troubleshooting runbook added. +- [x] Rollback and recovery plan added. +- [x] Security checklist added. +- [x] Geospatial validation rules added. +- [x] Build governance added. +- [x] Codex pass documents added. +- [x] Smoke scripts added. +- [ ] M6: implement actual backend foundation. +- [ ] M6: implement database migrations. +- [ ] M6: implement frontend shell. + +## M8 Codex Day-1 Readiness + +- [x] Add Day 1 master prompt. +- [x] Add pass-by-pass Day 1 prompts. +- [x] Add autonomy boundaries. +- [x] Add failure recovery playbook. +- [x] Add quality gate matrix. +- [x] Add operator checklist. +- [x] Add smoke script scaffold. +- [ ] Let Codex execute Day 1 implementation passes. + + +# M10 Ultra Preparation + +- [x] Add autonomous build charter. +- [x] Add Codex start-here guide. +- [x] Add M10 master autonomous prompt. +- [x] Add pass sequence. +- [x] Add geometry and CRS contracts. +- [x] Add error taxonomy. +- [x] Add feature flag strategy. +- [x] Add model adapter guide. +- [x] Add QA/QC matching algorithm. +- [x] Add frontend state machine. +- [x] Add implementation ticket index and tickets. +- [x] Add API example payloads. +- [x] Add final pre-Codex checklist. diff --git a/docs/TROUBLESHOOTING_RUNBOOK.md b/docs/TROUBLESHOOTING_RUNBOOK.md new file mode 100644 index 00000000..df37c3c8 --- /dev/null +++ b/docs/TROUBLESHOOTING_RUNBOOK.md @@ -0,0 +1,22 @@ +# Troubleshooting Runbook + +## Backend start niet +Controleer `.env`, database, Redis, dependencies en importeerbaarheid. Draai `scripts/smoke_backend_import.sh`. + +## PostGIS werkt niet +Controleer `SELECT postgis_full_version();` en migration met `CREATE EXTENSION IF NOT EXISTS postgis;`. + +## Rasterio/GDAL problemen +Gebruik Docker-profiel voor GIS dependencies. Voeg geen willekeurige pip-fixes toe zonder dependency policy. + +## Vector import geeft lege resultaten +Controleer CRS, bounds overlap, geometry validity en clip-operatie. + +## YOLO/SAM inference te traag +Gebruik kleine fixture, `ai-light` mode, lagere tile size of hogere confidence threshold. + +## QA/QC scores lijken fout +Controleer projectie, units, valid geometry, IoU threshold, matchingmethode en duplicaten. + +## Frontend toont lege kaart +Controleer MapLibre style, API URL, GeoJSON-validiteit, WGS84-coördinaten en layer visibility. diff --git a/docs/UI_PAGE_SPECIFICATIONS.md b/docs/UI_PAGE_SPECIFICATIONS.md new file mode 100644 index 00000000..fb327a71 --- /dev/null +++ b/docs/UI_PAGE_SPECIFICATIONS.md @@ -0,0 +1,201 @@ +# GeoIntel Kempen — UI Page Specifications v1.0 + +The UI must feel like a modern GeoAI workbench, not a generic admin dashboard. + +## Global layout + +- Left navigation: product sections. +- Center: map/workbench content. +- Right panel: context, analysis settings, result details. +- Bottom or top status area: job status and notifications. +- Dark professional theme by default, with enough contrast. + +## Main pages + +## 1. Home + +### Purpose +Start or resume work. + +### Components + +- Product intro: GeoAI Workbench for the Kempen. +- New project button. +- Open demo scenario button. +- Recent projects. +- System readiness cards: backend, database, storage, AI models. + +## 2. Project Workspace + +### Purpose +Project overview. + +### Components + +- Project summary. +- Areas list. +- Datasets list. +- Recent analysis runs. +- Quick actions: upload raster, upload vector, draw area, run detection. + +## 3. Map Workbench + +### Purpose +Spatial exploration and layer inspection. + +### Components + +- MapLibre map. +- Layer manager. +- Draw controls. +- Feature inspector. +- Opacity controls. +- CRS/bounds information. + +### Required actions + +- Draw area. +- Save area. +- Toggle layers. +- Click feature for attributes. + +## 4. Dataset Manager + +### Purpose +Upload, inspect and manage raster/vector datasets. + +### Components + +- Upload zone. +- Dataset table. +- Metadata detail panel. +- Processing status. +- Preview button. + +### API calls + +- `POST /projects/{id}/datasets/upload` +- `GET /projects/{id}/datasets` +- `GET /datasets/{id}/metadata` + +## 5. Raster Lab + +### Purpose +Inspect and process rasters. + +### Components + +- Raster selector. +- Metadata cards. +- Band selector. +- Histogram. +- Preview. +- Clip controls. +- Tile generation controls. + +## 6. Vector Lab + +### Purpose +Inspect and process vector layers. + +### Components + +- Vector selector. +- Attribute schema. +- Geometry summary. +- Operation buttons: clip, buffer, intersect, difference, dissolve. +- Result layer list. + +## 7. Detection Lab + +### Purpose +Run object detection on raster imagery. + +### Components + +- Raster selector. +- Model selector. +- Class selector. +- Confidence threshold. +- Tile size/overlap. +- Run button. +- Detection results table. +- Map overlay. + +## 8. Segmentation Lab + +### Purpose +Create masks and polygons from imagery. + +### Components + +- Raster selector. +- Model selector. +- Mode selector. +- Threshold controls. +- Mask preview. +- Polygonized output table. + +## 9. QA/QC Lab + +### Purpose +Validate predictions against reference layers such as GRB. + +### Components + +- Prediction layer selector. +- Reference layer selector. +- Class filter. +- IoU threshold slider. +- Metrics cards. +- TP/FP/FN map overlays. +- Findings table. + +## 10. Change Detection Lab + +### Purpose +Compare timepoints or analysis outputs. + +### Components + +- Dataset/layer A selector. +- Dataset/layer B selector. +- Method selector. +- Threshold controls. +- Added/removed/modified layers. +- Summary cards. + +## 11. Exports + +### Purpose +Download results. + +### Supported exports + +- GeoJSON +- CSV +- COCO later +- YOLO later +- masks +- report + +## 12. Settings + +### Purpose +System configuration. + +### Components + +- Data source settings. +- Storage paths. +- AI model paths. +- Processing defaults. +- Cache management. + +## UX rules + +- Every empty state must explain what the user should do next. +- Every long operation must create a visible job. +- Every analysis result must have export options. +- Every map layer must be inspectable. +- Do not create fake results to make pages look full. diff --git a/docs/UI_ROUTE_CONTRACTS.md b/docs/UI_ROUTE_CONTRACTS.md new file mode 100644 index 00000000..dee2a2e3 --- /dev/null +++ b/docs/UI_ROUTE_CONTRACTS.md @@ -0,0 +1,45 @@ +# UI Route Contracts + +## `/` +Purpose: landing dashboard. +Must show: create project action, recent projects, demo launcher, documentation links. + +## `/projects/:projectId` +Purpose: project workspace. +Must show: project summary, areas, datasets, latest analysis runs, exports. + +## `/projects/:projectId/map` +Purpose: map workbench. +Must show: map, layer tree, drawing tools, selected feature inspector. + +## `/projects/:projectId/datasets` +Purpose: dataset manager. +Must show: upload area, dataset table, status badges, metadata access. + +## `/projects/:projectId/raster` +Purpose: raster lab. +Must show: raster selector, metadata, band controls, operations panel. + +## `/projects/:projectId/vector` +Purpose: vector lab. +Must show: vector selector, geometry summary, operations panel. + +## `/projects/:projectId/detection` +Purpose: object detection lab. +Must show: raster selector, model selector, threshold controls, run button, result layers. + +## `/projects/:projectId/segmentation` +Purpose: segmentation lab. +Must show: raster selector, model selector, class controls, mask/result preview. + +## `/projects/:projectId/qaqc` +Purpose: quality control. +Must show: AI run selector, reference dataset selector, IoU threshold, metrics and findings map. + +## `/projects/:projectId/exports` +Purpose: generated files. +Must show: export actions, export history, download buttons. + +## `/settings` +Purpose: local app configuration. +Must show: storage paths, data connectors, model registry, API keys where required. diff --git a/docs/UI_UX_SPEC.md b/docs/UI_UX_SPEC.md new file mode 100644 index 00000000..213d09a2 --- /dev/null +++ b/docs/UI_UX_SPEC.md @@ -0,0 +1,136 @@ +# UI/UX Specification + +## 1. UX-principe + +GeoIntel moet aanvoelen als een moderne technische workbench, niet als een oude GIS-desktopkloon. + +De gebruiker moet altijd zien: + +- welk project actief is +- welke dataset actief is +- welke analyse loopt +- welke lagen zichtbaar zijn +- welke outputs exporteerbaar zijn + +## 2. Globale layout + +```text +Left sidebar: navigatie +Center: kaart / lab workspace +Right panel: context, parameters, results +Bottom/toast: job status +``` + +## 3. Hoofdpagina's + +### Home + +- nieuw project +- demo-case openen +- recente projecten +- status van lokale omgeving + +### Project Workspace + +- projectoverzicht +- areas +- datasets +- analysis runs +- exports + +### Map Workbench + +- kaart +- layer manager +- draw tools +- inspect tool +- area selection + +### Dataset Manager + +- uploads +- brondata +- metadata +- validatiestatus + +### Raster Lab + +- raster preview +- metadata +- band controls +- clip/tile acties + +### Vector Lab + +- feature table +- geometry info +- vector operations + +### Detection Lab + +- dataset kiezen +- model kiezen +- threshold +- run button +- resultaten + +### Segmentation Lab + +- model/prompt settings +- mask preview +- polygonize + +### QA/QC Lab + +- AI-run kiezen +- referentielaag kiezen +- metrics +- foutenkaart + +### Change Lab + +- run A/B +- methode +- resultaten + +### Exports + +- exportlijst +- downloadknoppen +- formaatkeuze + +## 4. Kaartlagen + +Elke laag moet minimaal hebben: + +- naam +- type +- zichtbaarheid +- opacity +- bron +- feature count indien vector +- style + +## 5. Jobstatus + +Langlopende acties moeten duidelijke status geven: + +- queued +- running +- succeeded +- failed + +Bij failed altijd: + +- foutmelding +- technische details +- retry indien mogelijk + +## 6. Designrichting + +- clean dark/light compatible +- technical but readable +- geen overbodige animaties +- geen demo-looking placeholders +- dense genoeg voor professionals +- duidelijke empty states diff --git a/docs/V1_SCOPE_FREEZE.md b/docs/V1_SCOPE_FREEZE.md new file mode 100644 index 00000000..fbe742b3 --- /dev/null +++ b/docs/V1_SCOPE_FREEZE.md @@ -0,0 +1,98 @@ +# V1 Scope Freeze + +## V1 identity +GeoIntel V1 is the first usable GeoAI Workbench foundation. It must prove the full data-processing loop from project to geospatial output. + +## V1 must include + +### Project foundation +- Create/list/open projects. +- Add one or more areas. +- Persist project metadata. +- Persist area geometry. + +### Dataset manager +- Upload raster datasets. +- Upload vector datasets. +- Extract metadata. +- Persist dataset records. +- Show import status. + +### Map workbench +- Display selected area. +- Display vector layers. +- Display raster preview or placeholder tile/extent if full tiling is not ready. +- Toggle layer visibility. +- Adjust opacity. +- Inspect feature properties. + +### Raster foundation +- Read metadata using Rasterio/GDAL-compatible stack. +- Clip raster by area if georeferenced. +- Generate tiles or tile records for inference. +- Store derived artifact paths. + +### Vector foundation +- Import GeoJSON and zipped shapefiles. +- Normalize CRS. +- Validate/fix simple geometries. +- Clip by project area. +- Calculate area/length in metric CRS. + +### Reference data foundation +- Support GRB/reference building layer through provider abstraction. +- Support local fixture provider for offline demos. +- Support OSM provider as fallback/context if external access is available. + +### Detection foundation +- Implement detection run entity. +- Implement inference provider interface. +- Add a simple YOLO/Ultralytics provider or stub provider only if clearly marked as development provider. +- Convert outputs to persisted detection geometries. +- Export detections as GeoJSON. + +### Segmentation foundation +- Implement segmentation run entity. +- Implement segmentation provider interface. +- Support mask artifact path. +- Polygonize segmentation masks when georeferenced and feasible. + +### QA/QC foundation +- Compare prediction geometries with reference geometries. +- Calculate IoU-based matching. +- Calculate precision, recall, F1. +- Persist quality-check record. +- Show matched/unmatched layers. + +### Exports +- GeoJSON export for detections/segmentations/change outputs. +- JSON metadata export. +- Simple report artifact. + +### Tests +- Backend unit tests for geometry metrics. +- Backend unit tests for QA matching. +- API smoke tests for projects/datasets/analysis runs. +- At least one fixture-based workflow test for Demo 1 if possible. + +## V1 should include if time permits +- Basic NDVI/NDWI/NDBI calculation for suitable rasters. +- Simple vector change detection. +- Simple dashboard cards for metrics. +- Demo fixture loader. + +## V1 must not include +- Full LiDAR workbench. +- Model training studio. +- Full MLOps registry. +- QGIS plugin. +- Multi-user authentication. +- Complex PDF designer. +- Real-time monitoring. +- Production deployment hardening beyond local Docker. + +## V1 UX rule +Every page must have a useful empty state and a real connected state. No page should exist only as a design placeholder. + +## V1 quality rule +If a feature cannot be made geospatially correct yet, it must explicitly show a warning and avoid exporting misleading coordinates. diff --git a/docs/VECTOR_OPERATIONS_SPEC.md b/docs/VECTOR_OPERATIONS_SPEC.md new file mode 100644 index 00000000..d6c7acf9 --- /dev/null +++ b/docs/VECTOR_OPERATIONS_SPEC.md @@ -0,0 +1,162 @@ +# GeoIntel Kempen — Vector Operations Specification v1.0 + +Vector operations are implemented with GeoPandas/Shapely in the backend and optionally accelerated through PostGIS. + +## Global requirements + +- Always validate CRS before spatial operations. +- Use metric CRS for area, length and distance calculations. +- Repair invalid geometries before operations. +- Return operation metadata with input dataset ids and parameters. +- Persist important outputs as derived layers. + +## Required V1 operations + +## 1. Metadata extraction + +Input: vector dataset. + +Output: + +```json +{ + "feature_count": 1234, + "geometry_types": ["Polygon"], + "crs": "EPSG:31370", + "bounds": [xmin, ymin, xmax, ymax], + "columns": [{"name":"type","dtype":"str"}] +} +``` + +## 2. Reproject + +Input: + +- vector dataset +- target CRS + +Output: + +- reprojected layer +- operation log + +## 3. Clip + +Input: + +- vector dataset +- area polygon + +Output: + +- clipped layer +- metrics: before/after feature count, retained area/length + +## 4. Buffer + +Input: + +- vector dataset +- distance in meters +- dissolve boolean + +Output: + +- buffer polygon layer + +## 5. Intersect + +Input: + +- layer A +- layer B + +Output: + +- intersection layer +- area/length metrics + +## 6. Difference + +Input: + +- layer A +- layer B + +Output: + +- geometries in A not covered by B + +Use case: changed features, missing reference coverage. + +## 7. Spatial join + +Input: + +- target layer +- join layer +- predicate: intersects, within, contains, nearest + +Output: + +- joined attributes +- match count + +## 8. Dissolve + +Input: + +- vector layer +- optional group field + +Output: + +- dissolved geometries + +Use case: combine green patches, merge detection masks. + +## 9. Polygonize from raster mask + +Input: + +- binary or class raster mask + +Output: + +- polygon layer with class values + +This is technically raster-to-vector but must produce vector layers compatible with all vector operations. + +## API shape + +Vector operations should be available as jobs: + +```http +POST /vector/operations/clip +POST /vector/operations/buffer +POST /vector/operations/intersect +POST /vector/operations/difference +POST /vector/operations/spatial-join +POST /vector/operations/dissolve +``` + +Each returns a job id and later a derived dataset/layer id. + +## Error handling + +Return clear validation errors for: + +- missing CRS +- unsupported geometry type +- invalid geometry repair failure +- empty output after clipping +- projection failure + +## Test fixtures + +Create small fixture GeoJSON files for: + +- building polygons +- road lines +- area polygon +- reference/prediction polygons for QA diff --git a/docs/governance/ARCHITECTURE_INVARIANTS.md b/docs/governance/ARCHITECTURE_INVARIANTS.md new file mode 100644 index 00000000..9286070b --- /dev/null +++ b/docs/governance/ARCHITECTURE_INVARIANTS.md @@ -0,0 +1,43 @@ +# Architecture Invariants + +These invariants must remain true throughout implementation. + +## Core invariants + +1. GeoIntel is backend/API-driven. Frontend does not own business logic. +2. PostGIS is the system of record for geometries and analysis outputs. +3. Filesystem/object storage stores binary artifacts, not authoritative feature state. +4. Heavy processing runs as jobs and reports state transitions. +5. Every analysis result belongs to an `analysis_run`. +6. Every detection/segmentation stores model metadata when produced by a model. +7. Every dataset stores source metadata, validation state and CRS information. +8. API responses use a consistent envelope. +9. Frontend state follows backend truth; it may cache but not invent completion states. +10. Exports are generated from persisted outputs, not transient frontend state. + +## Geospatial invariants + +1. API GeoJSON uses EPSG:4326 unless endpoint explicitly states otherwise. +2. Belgian metric calculations use EPSG:31370 where possible. +3. Area values are stored with units. +4. Length values are stored with units. +5. Invalid geometries are fixed, rejected or marked with a validation error; never silently accepted. +6. MultiPolygon/MultiLineString handling must be explicit. +7. Geometry simplification may not mutate authoritative geometry unless a derived output is created. + +## AI invariants + +1. AI inference is an analysis run. +2. Model outputs are candidates with confidence, not ground truth. +3. Thresholds are parameters and must be stored. +4. Tiling parameters are parameters and must be stored. +5. Model version is required for reproducibility. +6. QA/QC is separate from inference. + +## UI invariants + +1. Every page has empty/loading/error/success states. +2. Every destructive action requires confirmation. +3. Every unavailable feature must show an honest pending state. +4. The map is a workbench surface, not a decorative background. +5. Metrics must always indicate source and timestamp. diff --git a/docs/governance/DECISION_PRECEDENCE.md b/docs/governance/DECISION_PRECEDENCE.md new file mode 100644 index 00000000..351fa6d1 --- /dev/null +++ b/docs/governance/DECISION_PRECEDENCE.md @@ -0,0 +1,22 @@ +# Decision Precedence + +When documents conflict, use this precedence order. + +1. `docs/00-start/START_HERE.md` +2. `docs/governance/GEOINTEL_CONSTITUTION.md` +3. `docs/governance/ARCHITECTURE_INVARIANTS.md` +4. `docs/governance/FORBIDDEN_DECISIONS.md` +5. ADRs in `adr/` +6. Canonical specs in `docs/specs/` +7. Workflow specs in `docs/workflows/` +8. API/database contracts in `contracts/` and `docs/API_CONTRACTS.md` +9. Current build plans in `docs/build/` +10. Older milestone handoffs and prompts. + +## Rule for outdated documents + +Older milestone files are historical unless referenced by the current canonical start file. Do not delete them, but do not treat them as higher authority. + +## Rule for autonomous improvements + +If an improvement is compatible with constitution, invariants, contracts, state machines and golden paths, Codex may implement it. If not, document it as a proposal. diff --git a/docs/governance/FORBIDDEN_DECISIONS.md b/docs/governance/FORBIDDEN_DECISIONS.md new file mode 100644 index 00000000..c263c77e --- /dev/null +++ b/docs/governance/FORBIDDEN_DECISIONS.md @@ -0,0 +1,58 @@ +# Forbidden Decisions + +Codex and contributors must not make the following decisions without creating/updating an ADR and receiving explicit approval. + +## Technology changes + +Forbidden without ADR: + +- replacing FastAPI; +- replacing React/TypeScript; +- replacing PostgreSQL/PostGIS; +- replacing Redis + RQ as the default queue strategy; +- introducing a second ORM or data access pattern; +- introducing a second map rendering framework for the same purpose; +- replacing MapLibre/Deck.gl before V1; +- adding a non-approved AI framework as a core dependency. + +## Architecture changes + +Forbidden: + +- direct model-to-frontend coupling; +- frontend calling external geodata providers directly; +- AI services writing directly to core tables without an analysis run; +- storing geometries only as JSON files instead of PostGIS records; +- bypassing the response envelope; +- bypassing state machines; +- creating synchronous long-running endpoints for heavy operations; +- hardcoding demo data as if it were production data. + +## Product changes + +Forbidden: + +- making reporting the primary product; +- making chatbot the primary interface; +- removing QA/QC from the V1 critical path; +- moving LiDAR/training/MLOps into V1 critical path; +- changing the primary region away from the Kempen. + +## Data changes + +Forbidden: + +- dropping CRS metadata; +- accepting geospatial data without validation state; +- silently transforming CRS without recording it; +- treating OSM as more authoritative than GRB-like reference data for the V1 QA workflow; +- deleting original uploaded datasets when processing succeeds. + +## Quality changes + +Forbidden: + +- removing smoke scripts to pass CI; +- marking a pass complete without running relevant checks; +- adding `TODO`, `pass`, placeholder endpoints or fake success responses as completion substitutes; +- ignoring failing golden path tests. diff --git a/docs/governance/GEOINTEL_CONSTITUTION.md b/docs/governance/GEOINTEL_CONSTITUTION.md new file mode 100644 index 00000000..9e4dd1d7 --- /dev/null +++ b/docs/governance/GEOINTEL_CONSTITUTION.md @@ -0,0 +1,50 @@ +# GeoIntel Constitution + +This document defines the product laws. It has priority over ordinary specs when there is ambiguity. + +## Article 1 — GeoAI Workbench First + +GeoIntel is a GeoAI Workbench. It must prioritize geospatial processing, AI outputs, QA/QC and GIS exports over generic dashboards or visual polish. + +## Article 2 — Correct Data Beats Pretty UI + +When there is a tradeoff between visual presentation and correct geospatial processing, correct processing wins. + +## Article 3 — Every Output Must Be Traceable + +Every detection, segmentation, metric, export or QA score must be traceable to: + +- source dataset; +- processing parameters; +- model/version if applicable; +- analysis run; +- timestamp; +- CRS and geometry assumptions. + +## Article 4 — AI Never Owns the Truth + +AI models produce candidate observations. Reference data, QA/QC and explicit metrics determine confidence and usefulness. + +## Article 5 — Reproducibility Is Mandatory + +An analysis run must be reproducible from stored metadata, parameters and input references. + +## Article 6 — Async Processing Is the Default for Heavy Work + +Raster operations, vector batch operations, detection, segmentation, QA/QC and exports should be implemented as background jobs when non-trivial. + +## Article 7 — No Permanent Mock-Only Features + +Demo fixtures are allowed, but every feature must expose a path toward real data operation. UI must not pretend unavailable functionality is complete. + +## Article 8 — The Kempen Use Case Drives V1 + +Generic worldwide support is secondary. V1 must make the Geel/Mol/Turnhout/Kempen workflows excellent. + +## Article 9 — Architecture Must Stay Extensible + +Sentinel, DHMV, LiDAR, training, QGIS plugin and MLOps are future modules. V1 architecture may not block these. + +## Article 10 — Documentation Is Part of the Product + +Codex must update docs when contracts, behavior, workflows or limitations change. diff --git a/docs/specs/CANONICAL_DOMAIN_MODELS.md b/docs/specs/CANONICAL_DOMAIN_MODELS.md new file mode 100644 index 00000000..5889c7db --- /dev/null +++ b/docs/specs/CANONICAL_DOMAIN_MODELS.md @@ -0,0 +1,151 @@ +# Canonical Domain Models + +This document defines shared concepts. All backend services, frontend types and tests must use these definitions. + +## Project + +A Project is an investigation container. + +Required behavior: + +- contains Areas, Datasets, AnalysisRuns and Exports; +- has a human-readable name and optional description; +- may contain demo or real data, but the data mode must be visible; +- does not directly store geospatial features except through child entities. + +## Area + +An Area is a geospatial boundary used to clip, filter and analyze data. + +Required fields: + +- `id` +- `project_id` +- `name` +- `geometry` +- `geometry_crs` +- `area_m2` +- `bounds_4326` +- `created_at` + +Rules: + +- API representation is GeoJSON EPSG:4326. +- Internal metric operations should use EPSG:31370 for Kempen/Belgium. +- Area can be a free polygon, municipality-derived polygon or demo fixture. + +## Dataset + +A Dataset is a registered data source or uploaded file. + +Dataset types: + +- `raster` +- `vector` +- `reference_vector` +- `model_output` +- `mask` +- `export` + +Required behavior: + +- original artifact is preserved; +- metadata extraction produces a metadata record; +- validation state is explicit; +- derived datasets reference parents. + +## Layer + +A Layer is a map-renderable view of a dataset or analysis output. + +Rules: + +- layers have styling metadata; +- layers do not own authoritative geometry; +- layer visibility is frontend state, not processing state. + +## AnalysisRun + +An AnalysisRun is one execution of a processing pipeline. + +Examples: + +- raster metadata extraction; +- vector clipping; +- object detection; +- segmentation; +- QA/QC; +- export generation. + +Required fields: + +- `id` +- `project_id` +- `area_id` optional +- `analysis_type` +- `status` +- `parameters_json` +- `started_at` +- `finished_at` +- `error_code` optional +- `error_message` optional + +## Detection + +A Detection is a candidate object produced or imported as an analysis output. + +Required fields: + +- class name; +- confidence; +- geometry; +- bbox; +- source analysis run; +- model metadata if AI-produced; +- source tile if tiled inference was used. + +## Segmentation + +A Segmentation is a polygon or raster mask representing class coverage. + +Required fields: + +- class name; +- geometry or mask path; +- area_m2; +- confidence/score if available; +- source analysis run; +- model metadata if AI-produced. + +## ReferenceFeature + +A ReferenceFeature is an authoritative or semi-authoritative feature used for validation. + +V1 examples: + +- GRB-like building polygons; +- demo reference buildings; +- OSM fallback buildings. + +## QualityCheck + +A QualityCheck compares candidate outputs to reference data or validates dataset integrity. + +Required outputs: + +- metric values; +- method; +- thresholds; +- matched/unmatched features where applicable; +- pass/fail or warning status. + +## Export + +An Export is a generated artifact derived from persisted state. + +Allowed V1 exports: + +- GeoJSON; +- CSV metrics; +- simple HTML/Markdown report if trivial; +- not mandatory: complex PDF. diff --git a/docs/specs/DATA_LIFECYCLE.md b/docs/specs/DATA_LIFECYCLE.md new file mode 100644 index 00000000..2ef13d47 --- /dev/null +++ b/docs/specs/DATA_LIFECYCLE.md @@ -0,0 +1,46 @@ +# Data Lifecycle + +## Original dataset lifecycle + +1. User or system registers dataset. +2. Original file/reference is preserved. +3. Dataset enters `UPLOADED` or `CREATED`. +4. Validation extracts basic file and geospatial metadata. +5. Metadata is persisted. +6. Dataset becomes `READY`, `REQUIRES_CRS` or failure state. + +## Derived dataset lifecycle + +1. Operation is requested from a ready parent dataset. +2. AnalysisRun is created. +3. Job produces derived artifact. +4. Derived dataset references parent dataset and analysis run. +5. Derived metadata is extracted. +6. Derived layer can be rendered/exported. + +## Detection lifecycle + +1. Detection run is created. +2. Inputs and parameters are stored. +3. Model/import produces candidate features. +4. Features are georeferenced. +5. Features are validated. +6. Features are persisted as detections. +7. Optional QA/QC compares them to reference features. +8. Detections may be exported. + +## Segmentation lifecycle + +1. Segmentation run is created. +2. Raster and model parameters are stored. +3. Model/import produces masks. +4. Masks are georeferenced. +5. Optional polygonization creates vector features. +6. Features and/or mask paths are persisted. +7. QA/QC and export may run. + +## Deletion and archival + +- Deleting a project should not immediately physically delete files unless cleanup is requested. +- Archival removes items from active lists but keeps reproducibility. +- Physical cleanup must never leave dangling database references. diff --git a/docs/specs/ERROR_CATALOG.md b/docs/specs/ERROR_CATALOG.md new file mode 100644 index 00000000..5790ee1a --- /dev/null +++ b/docs/specs/ERROR_CATALOG.md @@ -0,0 +1,54 @@ +# Error Catalog + +Use stable error codes across backend and frontend. + +## Dataset errors + +- `DATASET_001_INVALID_FILE`: file cannot be read. +- `DATASET_002_UNSUPPORTED_TYPE`: file type is unsupported. +- `DATASET_003_MISSING_CRS`: CRS cannot be determined. +- `DATASET_004_INVALID_GEOMETRY`: geometry validation failed. +- `DATASET_005_METADATA_FAILED`: metadata extraction failed. +- `DATASET_006_STORAGE_FAILED`: artifact could not be stored. + +## Raster errors + +- `RASTER_001_NOT_GEOREFERENCED`: transform/CRS missing. +- `RASTER_002_BAND_OUT_OF_RANGE`: selected band does not exist. +- `RASTER_003_CLIP_FAILED`: clipping failed. +- `RASTER_004_TILE_FAILED`: tiling failed. +- `RASTER_005_REPROJECT_FAILED`: reprojection failed. + +## Vector errors + +- `VECTOR_001_EMPTY_LAYER`: no features found. +- `VECTOR_002_SCHEMA_UNSUPPORTED`: attribute schema unsupported. +- `VECTOR_003_CRS_TRANSFORM_FAILED`: reprojection failed. +- `VECTOR_004_OPERATION_FAILED`: spatial operation failed. + +## Analysis errors + +- `ANALYSIS_001_INVALID_INPUT_STATE`: input not ready. +- `ANALYSIS_002_JOB_FAILED`: background job failed. +- `ANALYSIS_003_TIMEOUT`: processing timed out. +- `ANALYSIS_004_PARTIAL_OUTPUT`: output incomplete. + +## Detection errors + +- `DETECTION_001_MODEL_UNAVAILABLE`: configured model unavailable. +- `DETECTION_002_INFERENCE_FAILED`: inference failed. +- `DETECTION_003_GEOREFERENCE_FAILED`: output could not be mapped to coordinates. +- `DETECTION_004_NO_DETECTIONS`: valid run produced no detections. + +## QA/QC errors + +- `QAQC_001_REFERENCE_MISSING`: reference layer missing. +- `QAQC_002_CANDIDATES_MISSING`: candidate features missing. +- `QAQC_003_MATCHING_FAILED`: matching algorithm failed. +- `QAQC_004_THRESHOLD_INVALID`: threshold invalid. + +## Export errors + +- `EXPORT_001_NO_OUTPUTS`: nothing to export. +- `EXPORT_002_FORMAT_UNSUPPORTED`: unsupported export format. +- `EXPORT_003_GENERATION_FAILED`: export failed. diff --git a/docs/specs/GIS_STANDARDS.md b/docs/specs/GIS_STANDARDS.md new file mode 100644 index 00000000..6f2b8c8c --- /dev/null +++ b/docs/specs/GIS_STANDARDS.md @@ -0,0 +1,58 @@ +# GIS Standards + +## CRS standards + +- API GeoJSON: EPSG:4326. +- Internal Belgian metric calculations: EPSG:31370. +- Raster native operations preserve source CRS unless reproject requested. +- Store both source CRS and normalized CRS. + +## Units + +- Area: square meters internally, hectares/km² as display derivatives. +- Length: meters internally, kilometers as display derivative. +- Resolution: meters/pixel where projected CRS allows it; otherwise state units clearly. + +## Geometry validity + +All vector geometries must pass validation before becoming `READY`. + +Allowed repair strategies: + +1. `make_valid` where available. +2. zero-width buffer only if documented. +3. reject and require user correction. + +Repairs must be recorded in metadata. + +## Geometry type normalization + +- Polygon inputs may become MultiPolygon. +- LineString inputs may become MultiLineString. +- GeometryCollections require explicit extraction or rejection. + +## Spatial operations + +All spatial operations must define: + +- input CRS; +- output CRS; +- unit assumptions; +- tolerance; +- geometry repair behavior; +- empty-result behavior. + +## Precision + +Store full precision in database. Simplify only derived visualization layers. + +## Boundary handling + +For clipping/intersections, include features that intersect area boundary. Record whether metrics use full feature geometry or clipped geometry. + +## Source priority for V1 reference workflows + +1. GRB-like reference fixtures / GRB when implemented. +2. Official Flemish datasets where available. +3. OSM fallback. +4. User-uploaded reference layer. diff --git a/docs/specs/PERFORMANCE_BUDGETS_CANONICAL.md b/docs/specs/PERFORMANCE_BUDGETS_CANONICAL.md new file mode 100644 index 00000000..df6f6eb1 --- /dev/null +++ b/docs/specs/PERFORMANCE_BUDGETS_CANONICAL.md @@ -0,0 +1,29 @@ +# Canonical Performance Budgets + +These are V1 local-development budgets. They are guidance, not hard production SLAs. + +## API responsiveness + +- simple GET list/detail: < 500 ms on demo data; +- project/area creation: < 1 s; +- metadata retrieval: < 500 ms once extracted; +- job status polling: < 300 ms. + +## Processing budgets on demo fixtures + +- vector metadata extraction: < 2 s; +- vector clip on demo area: < 5 s; +- raster metadata extraction on small GeoTIFF: < 3 s; +- tile manifest generation on demo raster: < 10 s; +- QA/QC on demo building fixtures: < 5 s; +- GeoJSON export on demo fixtures: < 5 s. + +## UI budgets + +- initial shell render: < 2 s in local dev; +- map layer toggle on demo data: < 500 ms; +- metrics panel update after API response: < 300 ms. + +## Scaling note + +Large real rasters or full municipality layers may exceed these budgets. In that case, the app must show async job state rather than blocking. diff --git a/docs/specs/RASTER_STANDARDS.md b/docs/specs/RASTER_STANDARDS.md new file mode 100644 index 00000000..23cacd2d --- /dev/null +++ b/docs/specs/RASTER_STANDARDS.md @@ -0,0 +1,76 @@ +# Raster Standards + +## Accepted V1 raster inputs + +- GeoTIFF preferred. +- TIFF only if georeferencing metadata exists or sidecar world file is provided. +- JPG/PNG only as non-georeferenced preview unless georeferencing is explicitly supplied. + +## Metadata required + +- width; +- height; +- band count; +- dtype; +- CRS; +- transform; +- bounds in source CRS; +- bounds in EPSG:4326 where possible; +- nodata value; +- resolution; +- file size; +- checksum/hash. + +## Tile defaults + +- tile size: 512x512 pixels; +- overlap: 64 pixels for detection/segmentation; +- padding strategy: reflect or constant nodata, recorded in parameters; +- tile IDs must be stable and reproducible. + +## Derived raster outputs + +Derived rasters must reference: + +- parent dataset; +- operation; +- parameters; +- timestamp; +- CRS; +- nodata handling. + +## Raster statistics + +V1 statistics: + +- min; +- max; +- mean; +- std where feasible; +- nodata count; +- histogram bins for display. + +## Reprojection + +Reprojection must not overwrite originals. It creates a derived dataset. + +## Clipping + +Raster clipping to Area creates a derived raster and stores the clipping geometry reference. + +## AI inference preparation + +Before inference: + +- dataset must be georeferenced; +- CRS must be known; +- transform must be available; +- tiling parameters must be stored; +- band selection must be explicit. + +## Non-goals for V1 + +- perfect Sentinel processing pipeline; +- cloud optimized GeoTIFF production; +- large-scale distributed raster processing; +- LiDAR-derived raster generation. diff --git a/docs/specs/STATE_MACHINES.md b/docs/specs/STATE_MACHINES.md new file mode 100644 index 00000000..227e9acf --- /dev/null +++ b/docs/specs/STATE_MACHINES.md @@ -0,0 +1,108 @@ +# State Machines + +Use these states exactly. Do not invent ad-hoc alternatives. + +## Dataset state machine + +```text +CREATED + ↓ +UPLOADING + ↓ +UPLOADED + ↓ +VALIDATING + ↓ +METADATA_EXTRACTED + ↓ +READY +``` + +Failure states: + +```text +VALIDATION_FAILED +METADATA_FAILED +PROCESSING_FAILED +REQUIRES_CRS +ARCHIVED +DELETED +``` + +Rules: + +- `READY` means metadata and validation are good enough for compatible operations. +- `REQUIRES_CRS` blocks geospatial operations but may allow file inspection. +- `ARCHIVED` keeps data but hides it from active workflows. +- `DELETED` means logical deletion unless physical cleanup is explicitly run. + +## AnalysisRun state machine + +```text +QUEUED + ↓ +RUNNING + ↓ +SUCCEEDED +``` + +Failure/cancel states: + +```text +FAILED +CANCELLED +PARTIAL +``` + +Rules: + +- `PARTIAL` is allowed only when outputs are explicitly incomplete and marked as such. +- Failed runs must retain logs/error code. +- Retrying creates a new run unless retry semantics are explicitly implemented. + +## Job state machine + +```text +PENDING + ↓ +STARTED + ↓ +FINISHED +``` + +Failure states: + +```text +FAILED +RETRYING +CANCELLED +TIMEOUT +``` + +## Export state machine + +```text +REQUESTED + ↓ +GENERATING + ↓ +READY +``` + +Failure states: + +```text +FAILED +EXPIRED +``` + +## Frontend page state model + +Every data-driven page must handle: + +- `empty` +- `loading` +- `ready` +- `error` +- `partial` +- `offline/unavailable` where external service is involved. diff --git a/docs/workflows/GOLDEN_PATHS.md b/docs/workflows/GOLDEN_PATHS.md new file mode 100644 index 00000000..deab428e --- /dev/null +++ b/docs/workflows/GOLDEN_PATHS.md @@ -0,0 +1,98 @@ +# Golden Paths + +Golden paths are protected workflows. Every major Codex pass must avoid breaking them. + +## Golden Path 1 — Project Area Dataset Foundation + +Goal: prove that the app can create a geospatial workspace. + +Steps: + +1. Create project. +2. Create an Area from GeoJSON. +3. Upload/register a vector dataset. +4. Extract metadata. +5. Persist dataset and geometry metadata. +6. Display project, area and dataset in frontend. + +Pass criteria: + +- project exists via API; +- area geometry validates; +- dataset reaches `READY` or honest failure state; +- frontend can show the state. + +## Golden Path 2 — Reference Building QA + +Goal: prove core GeoAI value without requiring real model inference yet. + +Steps: + +1. Load demo reference buildings. +2. Load demo predicted buildings. +3. Run matching algorithm. +4. Produce precision, recall, F1 and IoU summary. +5. Store QA run. +6. Export matched/unmatched features as GeoJSON. + +Pass criteria: + +- metrics are deterministic on fixtures; +- thresholds are configurable; +- false positives/false negatives are visible; +- export is valid GeoJSON. + +## Golden Path 3 — Raster Metadata and Tiling Readiness + +Goal: prove raster handling foundation. + +Steps: + +1. Register/upload GeoTIFF. +2. Extract metadata. +3. Validate CRS/transform. +4. Generate tile manifest without necessarily running AI. +5. Store tile parameters. + +Pass criteria: + +- metadata is persisted; +- tile count is deterministic; +- georeferencing is preserved; +- unsupported rasters fail honestly. + +## Golden Path 4 — Detection Result Lifecycle + +Goal: prove detection outputs can travel through the system. + +Steps: + +1. Create or import detection run. +2. Store detections with class/confidence/geometry. +3. Render detections as layer. +4. Run QA against reference. +5. Export detections. + +Pass criteria: + +- no frontend-only detections; +- every detection belongs to analysis run; +- geometries are valid; +- export works. + +## Golden Path 5 — V1 Recruiter Demo + +Goal: show portfolio value. + +Narrative: + +> GeoIntel loads a Kempen area, compares predicted building detections with reference building polygons, reports QA metrics, shows spatial errors and exports usable GIS output. + +Minimum demo artifacts: + +- map layer with reference buildings; +- map layer with predicted detections; +- QA metrics panel; +- unmatched features layer; +- GeoJSON export; +- concise explanation of methods and limitations. diff --git a/exports/.gitkeep b/exports/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/fixtures/geojson/predicted_buildings_fixture.geojson b/fixtures/geojson/predicted_buildings_fixture.geojson new file mode 100644 index 00000000..29f24004 --- /dev/null +++ b/fixtures/geojson/predicted_buildings_fixture.geojson @@ -0,0 +1,76 @@ +{ + "type": "FeatureCollection", + "name": "predicted_buildings_fixture", + "features": [ + { + "type": "Feature", + "properties": { + "id": "pred-1", + "class": "building", + "confidence": 0.86 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.99002, + 51.16002 + ], + [ + 4.99052, + 51.16002 + ], + [ + 4.99052, + 51.16038 + ], + [ + 4.99002, + 51.16038 + ], + [ + 4.99002, + 51.16002 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "pred-2", + "class": "building", + "confidence": 0.63 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.992, + 51.1608 + ], + [ + 4.9923, + 51.1608 + ], + [ + 4.9923, + 51.1611 + ], + [ + 4.992, + 51.1611 + ], + [ + 4.992, + 51.1608 + ] + ] + ] + } + } + ] +} \ No newline at end of file diff --git a/fixtures/geojson/reference_buildings_fixture.geojson b/fixtures/geojson/reference_buildings_fixture.geojson new file mode 100644 index 00000000..f801cd6e --- /dev/null +++ b/fixtures/geojson/reference_buildings_fixture.geojson @@ -0,0 +1,76 @@ +{ + "type": "FeatureCollection", + "name": "reference_buildings_fixture", + "features": [ + { + "type": "Feature", + "properties": { + "id": "ref-1", + "source": "fixture_grb", + "class": "building" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.99, + 51.16 + ], + [ + 4.9905, + 51.16 + ], + [ + 4.9905, + 51.1604 + ], + [ + 4.99, + 51.1604 + ], + [ + 4.99, + 51.16 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "ref-2", + "source": "fixture_grb", + "class": "building" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.991, + 51.1602 + ], + [ + 4.9914, + 51.1602 + ], + [ + 4.9914, + 51.1606 + ], + [ + 4.991, + 51.1606 + ], + [ + 4.991, + 51.1602 + ] + ] + ] + } + } + ] +} \ No newline at end of file diff --git a/fixtures/golden/expected_qa_metrics.json b/fixtures/golden/expected_qa_metrics.json new file mode 100644 index 00000000..b315d884 --- /dev/null +++ b/fixtures/golden/expected_qa_metrics.json @@ -0,0 +1,17 @@ +{ + "benchmark_id": "golden-buildings-partial-match-v1", + "description": "Two reference building polygons and two candidate building polygons: one candidate matches ref-1, one candidate is a false positive, and ref-2 is a false negative.", + "candidate_fixture": "fixtures/golden/predicted_buildings.geojson", + "reference_fixture": "fixtures/golden/reference_buildings.geojson", + "iou_threshold": 0.5, + "candidate_feature_count": 2, + "reference_feature_count": 2, + "matches": 1, + "false_positive_count": 1, + "false_negative_count": 1, + "precision": 0.5, + "recall": 0.5, + "f1": 0.5, + "mean_iou": 0.8339768339761133, + "tolerance": 1e-9 +} diff --git a/fixtures/golden/predicted_buildings.geojson b/fixtures/golden/predicted_buildings.geojson new file mode 100644 index 00000000..efc8e8b0 --- /dev/null +++ b/fixtures/golden/predicted_buildings.geojson @@ -0,0 +1,48 @@ +{ + "type": "FeatureCollection", + "name": "golden_predicted_buildings", + "features": [ + { + "type": "Feature", + "id": "pred-1", + "properties": { + "id": "pred-1", + "class": "building", + "confidence": 0.86 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [4.99002, 51.16002], + [4.99052, 51.16002], + [4.99052, 51.16038], + [4.99002, 51.16038], + [4.99002, 51.16002] + ] + ] + } + }, + { + "type": "Feature", + "id": "pred-2", + "properties": { + "id": "pred-2", + "class": "building", + "confidence": 0.63 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [4.992, 51.1608], + [4.9923, 51.1608], + [4.9923, 51.1611], + [4.992, 51.1611], + [4.992, 51.1608] + ] + ] + } + } + ] +} diff --git a/fixtures/golden/reference_buildings.geojson b/fixtures/golden/reference_buildings.geojson new file mode 100644 index 00000000..7dfb39bb --- /dev/null +++ b/fixtures/golden/reference_buildings.geojson @@ -0,0 +1,48 @@ +{ + "type": "FeatureCollection", + "name": "golden_reference_buildings", + "features": [ + { + "type": "Feature", + "id": "ref-1", + "properties": { + "id": "ref-1", + "source": "golden_fixture", + "class": "building" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [4.99, 51.16], + [4.9905, 51.16], + [4.9905, 51.1604], + [4.99, 51.1604], + [4.99, 51.16] + ] + ] + } + }, + { + "type": "Feature", + "id": "ref-2", + "properties": { + "id": "ref-2", + "source": "golden_fixture", + "class": "building" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [4.991, 51.1602], + [4.9914, 51.1602], + [4.9914, 51.1606], + [4.991, 51.1606], + [4.991, 51.1602] + ] + ] + } + } + ] +} diff --git a/frontend/.dockerignore b/frontend/.dockerignore new file mode 100644 index 00000000..dfebf087 --- /dev/null +++ b/frontend/.dockerignore @@ -0,0 +1,7 @@ +node_modules +dist +__pycache__ +*.pyc +.pytest_cache +.vite +.env diff --git a/frontend/.gitkeep b/frontend/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 00000000..237e7597 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,16 @@ +FROM node:20-alpine AS build + +WORKDIR /app + +COPY package.json package-lock.json* ./ +RUN npm install + +COPY . . +RUN npm run build + +FROM nginx:1.27-alpine AS runtime + +COPY nginx.conf /etc/nginx/conf.d/default.conf +COPY --from=build /app/dist /usr/share/nginx/html + +EXPOSE 80 diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 00000000..38cefcb4 --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,189 @@ +# GeoIntel Frontend (Sprint 4) + +React + TypeScript + MapLibre foundation for project/area/dataset workflow. + +## Scope implemented +- API client layer (`src/services/api`) +- Project and area list/create flows +- Vector and raster dataset upload + metadata display +- MapLibre map with uploaded feature preview + +## Sprint 2 additions +- Dataset manager details now shows: + - dataset type + - processing status + - file size + - vector feature count + - vector bounds + - file metadata (original/stored filename, MIME type, SHA256) + - raster metadata preview when available +- Vector inspection and raster metadata endpoint support in API client +- Readiness indicator for uploaded datasets + +## Sprint 3 additions +- Dataset detail panel now includes: + - available operations + - operation run actions (`clip`, `buffer`, `intersect`) + - linked job list and status details + - derived dataset navigation from job output +- Raster operation callouts for unavailable processing states +- Vector operation summaries integrated in selected dataset view + +## Sprint 4 additions +- Dataset detail panel now includes raster-specific runtime metadata: + - driver, dimensions, band count, bounds, CRS + - storage details (`size_bytes`, `checksum_sha256`) +- Added action buttons for raster operations: + - regenerate/inspect metadata + - generate preview + - generate tiles + - clip by selected area +- Added clear unavailable states for raster ops when backend returns `RASTER_PROCESSING_UNAVAILABLE` +- Added operation/job result visibility for raster runs with derived dataset navigation + +## Sprint 5 additions +- Added raster band statistics display in the dataset detail panel (min/max/mean/std/nodata ratio/valid pixel count). +- Added raster reproject workflow in UI (target CRS + resampling) with visible errors for invalid CRS/dependency gaps. +- Added stronger raster tile/clip result context with consistent job status display and derived output links where produced. +- Added tile manifest-aware controls for raster tile generation parameters. + +## Sprint 6 additions +- Added spectral index controls in dataset detail panel: + - NDVI with NIR/Red band inputs + - NDWI with NIR/Green band inputs + - NDBI with SWIR/NIR band inputs +- Added job-driven execution for local spectral index operations and result dataset linking. +- Added clear error surfacing for dependency-unavailable index execution (`RASTER_PROCESSING_UNAVAILABLE`). +- Added CRS/bounds/resolution context visibility for raster index source inspection. + +## Sprint 7B additions +- Added a lightweight Provider Capabilities panel. +- The panel lists GRB, OSM, manual and fixture provider status, configured state, authority level, supported layers, supported geometry types, query modes and limitation messages. +- GRB and OSM are shown as `not_configured`; the UI does not expose a live import/download action for them. +- Existing dataset, reference and QA/QC UI remains unchanged. + +## Sprint 8 additions +- Added a minimal Detection Lab panel. +- The panel lists detection model capabilities and clearly shows configured/not_configured status. +- Users can select a raster dataset, choose a confidence threshold and request a detection run. +- Unavailable model responses are shown honestly with the backend error code/message. +- The UI does not claim real YOLO/PyTorch inference is enabled. + +## Sprint 8B additions +- Detection Lab now exposes the `yolo-configured` capability reported by the backend. +- When `yolo-configured` is selected, users can provide an existing raster tile manifest path. +- The UI still does not download models or create fake detections; backend status and error codes remain the source of truth. + +## Sprint 8C additions +- Detection Lab now lists detection analysis runs and persisted detections. +- Users can filter detections by class and minimum confidence. +- Selected detection GeoJSON is rendered on the existing MapLibre workbench map. +- Detection QA compares a selected detection run against a reference dataset and displays persisted QA metrics. +- No segmentation UI is introduced in Sprint 8C. + +## Sprint 9 additions +- Added a minimal Segmentation Lab panel. +- The panel lists segmentation model capabilities and clearly distinguishes placeholders, fixture/demo mode, SAM placeholder and YOLO-seg placeholder states. +- Users can select a raster dataset, select a configured segmentation model, list segmentation runs and load persisted segmentation results. +- Segmentation results display class, confidence, area, model, tile and mask path fields. +- Selected segmentation GeoJSON is rendered through the existing MapLibre workbench map. +- Segmentation QA compares a selected segmentation run against a reference dataset and displays persisted QA metrics. +- Real SAM/YOLO-seg inference, model downloads and new AI dependencies are not introduced in Sprint 9. + +## Sprint 10 maintainability updates +- Split large workbench sections out of `src/App.tsx` without changing UI behavior: + - `src/components/project/ProjectPanel.tsx` + - `src/components/project/AreaPanel.tsx` + - `src/components/providers/ProviderPanel.tsx` + - `src/components/detection/DetectionLab.tsx` + - `src/components/segmentation/SegmentationLab.tsx` +- `App.tsx` still owns shared state orchestration and API calls; extracted components receive the same state and callbacks as props. +- Existing MapLibre overlay behavior, dataset/reference flows, Detection Lab flows and Segmentation Lab flows are unchanged. + +## Sprint 15 additions + +- Added a Projects panel action to load the explicit offline demo workflow. +- The action calls `POST /api/v1/demo/workflow` and refreshes projects, areas, datasets and run lists. +- Demo data is labelled fixture/demo data and does not represent live GRB/OSM data or AI inference. + +## Sprint 16 additions + +- Added a QA/QC Results panel that lists persisted project quality checks and metric rows. +- The panel calls `GET /api/v1/projects/{project_id}/quality-checks`. +- Demo workflow loading and QA actions refresh the persisted QA/QC result list. + +## Sprint 17 additions + +- Added an Export Center panel. +- The panel can create persisted exports for: + - project metadata JSON + - project report HTML + - selected vector dataset GeoJSON + - selected detection run GeoJSON + - selected segmentation run GeoJSON +- Export records are listed from `GET /api/v1/exports/projects/{project_id}/exports`. +- JSON artifact preview uses `GET /api/v1/exports/{export_id}/content`. +- Artifact downloads use `GET /api/v1/exports/{export_id}/download`. +- The UI does not introduce live provider downloads, a report designer or new AI behavior. +- The HTML report is a lightweight artifact built from persisted project, dataset, QA/QC summary and export history state; it is not a PDF/report designer. + +## Release hardening updates + +- Production builds split application code, React vendor code and MapLibre vendor code into separate chunks. +- The MapLibre chunk is intentionally larger than generic app chunks because it contains the GIS map runtime; the Vite warning threshold is set to keep this known vendor dependency visible without warning on every release build. + +## Raster dependency visibility + +Raster metadata and raster ops may remain unavailable when backend raster stack is missing. In that case: + +- raster uploads are still stored and listed +- status becomes `failed` +- backend returns explicit `RASTER_PROCESSING_UNAVAILABLE` responses for metadata/preview/clip/tile + +## Run locally + +### Prerequisites + +- Node.js 18+ + +### Install dependencies + +```bash +cd frontend +npm install +``` + +### Run locally + +```bash +npm run start +``` + +### Type check and build + +```bash +npm run typecheck +npm run build +``` + +### Dockerized frontend + +```bash +docker compose up --build frontend +``` + +When using the repository Docker Compose stack, the frontend is published on host port `1202`: `http://localhost:1202`. + +The frontend API client uses same-origin requests by default. In Docker Compose, nginx serves the built frontend and reverse proxies `/api` and `/health` to the backend service, so browser clients on LAN hosts do not call their own `localhost:8000`. + +## Useful repository scripts + +- `bash scripts/frontend_install.sh` +- `bash scripts/frontend_typecheck.sh` +- `bash scripts/frontend_build.sh` +- `bash scripts/frontend_dev.sh` + +## Key docs +- `docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md` +- `docs/API_CONTRACTS.md` +- `docs/REPOSITORY_CONVENTIONS.md` diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 00000000..4b95ca2c --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,12 @@ + + + + + + GeoIntel Kempen + + +
+ + + diff --git a/frontend/nginx.conf b/frontend/nginx.conf new file mode 100644 index 00000000..f87142f4 --- /dev/null +++ b/frontend/nginx.conf @@ -0,0 +1,29 @@ +server { + listen 80; + server_name _; + + root /usr/share/nginx/html; + index index.html; + + location /api/ { + proxy_pass http://backend:8000/api/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /health { + proxy_pass http://backend:8000/health; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location / { + try_files $uri $uri/ /index.html; + } +} diff --git a/frontend/package-lock.json b/frontend/package-lock.json new file mode 100644 index 00000000..413c97f1 --- /dev/null +++ b/frontend/package-lock.json @@ -0,0 +1,2109 @@ +{ + "name": "geointel-frontend", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "geointel-frontend", + "version": "0.1.0", + "dependencies": { + "maplibre-gl": "^4.7.1", + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@types/react": "^18.2.0", + "@types/react-dom": "^18.2.0", + "@vitejs/plugin-react": "^4.3.2", + "typescript": "^5.5.4", + "vite": "^5.4.1" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", + "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", + "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.7" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.29.7.tgz", + "integrity": "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.29.7.tgz", + "integrity": "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", + "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@mapbox/geojson-rewind": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/@mapbox/geojson-rewind/-/geojson-rewind-0.5.2.tgz", + "integrity": "sha512-tJaT+RbYGJYStt7wI3cq4Nl4SXxG8W7JDG5DMJu97V25RnbNg3QtQtf+KD+VLjNpWKYsRvXDNmNrBgEETr1ifA==", + "license": "ISC", + "dependencies": { + "get-stream": "^6.0.1", + "minimist": "^1.2.6" + }, + "bin": { + "geojson-rewind": "geojson-rewind" + } + }, + "node_modules/@mapbox/jsonlint-lines-primitives": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@mapbox/jsonlint-lines-primitives/-/jsonlint-lines-primitives-2.0.2.tgz", + "integrity": "sha512-rY0o9A5ECsTQRVhv7tL/OyDpGAoUB4tTvLiW1DSzQGq4bvTPhNw1VpSNjDJc5GFZ2XuyOtSWSVN05qOtcD71qQ==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@mapbox/point-geometry": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@mapbox/point-geometry/-/point-geometry-0.1.0.tgz", + "integrity": "sha512-6j56HdLTwWGO0fJPlrZtdU/B13q8Uwmo18Ck2GnGgN9PCFyKTZ3UbXeEdRFh18i9XQ92eH2VdtpJHpBD3aripQ==", + "license": "ISC" + }, + "node_modules/@mapbox/tiny-sdf": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@mapbox/tiny-sdf/-/tiny-sdf-2.2.0.tgz", + "integrity": "sha512-LVL4wgI9YAum5V+LNVQO6QgFBPw7/MIIY4XJPNsPDMrjEwcE+JfKk1LuIl8GnF197ejVdC9QdPaxrx5gfgdGXg==", + "license": "BSD-2-Clause" + }, + "node_modules/@mapbox/unitbezier": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@mapbox/unitbezier/-/unitbezier-0.0.1.tgz", + "integrity": "sha512-nMkuDXFv60aBr9soUG5q+GvZYL+2KZHVvsqFCzqnkGEf46U2fvmytHaEVc1/YZbiLn8X+eR3QzX1+dwDO1lxlw==", + "license": "BSD-2-Clause" + }, + "node_modules/@mapbox/vector-tile": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@mapbox/vector-tile/-/vector-tile-1.3.1.tgz", + "integrity": "sha512-MCEddb8u44/xfQ3oD+Srl/tNcQoqTw3goGk2oLsrFxOTc3dUp+kAnby3PvAeeBYSMSjSPD1nd1AJA6W49WnoUw==", + "license": "BSD-3-Clause", + "dependencies": { + "@mapbox/point-geometry": "~0.1.0" + } + }, + "node_modules/@mapbox/whoots-js": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@mapbox/whoots-js/-/whoots-js-3.1.0.tgz", + "integrity": "sha512-Es6WcD0nO5l+2BOQS4uLfNPYQaNDfbot3X1XUoloz+x0mPDS3eeORZJl06HXjwBG1fOGwCRnzK88LMdxKRrd6Q==", + "license": "ISC", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@maplibre/maplibre-gl-style-spec": { + "version": "20.4.0", + "resolved": "https://registry.npmjs.org/@maplibre/maplibre-gl-style-spec/-/maplibre-gl-style-spec-20.4.0.tgz", + "integrity": "sha512-AzBy3095fTFPjDjmWpR2w6HVRAZJ6hQZUCwk5Plz6EyfnfuQW1odeW5i2Ai47Y6TBA2hQnC+azscjBSALpaWgw==", + "license": "ISC", + "dependencies": { + "@mapbox/jsonlint-lines-primitives": "~2.0.2", + "@mapbox/unitbezier": "^0.0.1", + "json-stringify-pretty-compact": "^4.0.0", + "minimist": "^1.2.8", + "quickselect": "^2.0.0", + "rw": "^1.3.3", + "tinyqueue": "^3.0.0" + }, + "bin": { + "gl-style-format": "dist/gl-style-format.mjs", + "gl-style-migrate": "dist/gl-style-migrate.mjs", + "gl-style-validate": "dist/gl-style-validate.mjs" + } + }, + "node_modules/@maplibre/maplibre-gl-style-spec/node_modules/quickselect": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/quickselect/-/quickselect-2.0.0.tgz", + "integrity": "sha512-RKJ22hX8mHe3Y6wH/N3wCM6BWtjaxIyyUIkpHOvfFnxdI4yD4tBXEBKSbriGujF6jnSVkJrffuo6vxACiSSxIw==", + "license": "ISC" + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.27", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", + "integrity": "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.61.1.tgz", + "integrity": "sha512-JnBB8MdXj45cajvTuO5FmPlvFVJRQgvrz1uSEl3NwqFnReAPGwb8EanbGi4z2nRaqLzjJSv5/JmycoTKlRZxHA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.61.1.tgz", + "integrity": "sha512-Jx2g7iSjw4AOT0HDPHM9RV3GNjRXwybWtSFZiZAYUTjUwjVrYIwq3kBf+LnhqJlzXFAqTAh2F7IGI+O568exPw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.61.1.tgz", + "integrity": "sha512-0F1L/Z3Eqv8mT2n3dCpeO8GcTvHvVqkP5/t6DMsn0KzhYVcg+s7Ncl5DS8qjKYEeio6Az0Gt6nyBORay5qIlCA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.61.1.tgz", + "integrity": "sha512-qLttcH871ujY4YcVfUSShhOw+CsoTatYz8gRbHO7Bb92QH059/P0y5do1KMs41fY0BpD2x4AJH/gID0zFiqVKQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.61.1.tgz", + "integrity": "sha512-fUI4RapGE0Oh3mb8mgfvC1O2nU1RpDZUKnDQm3xB1Ipg7C2wTs5Kstz7G2uWK99a8S2yTMq8/P4uycwNa0nJyw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.61.1.tgz", + "integrity": "sha512-H5YrdvJaDtI/U9/emrD4b++xkvp3y/JvOe4rizHbxvkyMfRS/CiRYdji+Pl8D0brEaNFWUh1drQxgAGIl6Xudw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.61.1.tgz", + "integrity": "sha512-Q8CBCCQtDFrYtXoeUXSrnFXKOnyUhx6bz+SkL6A0E7V8kAiCJ5pamq1WtbfpVGhR5TSpXY6ak3avmDc5fHTyJA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.61.1.tgz", + "integrity": "sha512-nwnhk1581l0FBVellGcVCAT0Oi06onEA3WB53sf01VO3I0UPBkMH9sXONYME2K0ovXcNayJfNtHfm6mpJElatQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.61.1.tgz", + "integrity": "sha512-x5Xr49hwt3hdW75UOZm3395YwwzPyauktslv29KpWL/T+vVAzoT3azLcTWv0eMciBNrx+DYjH4paehHoLpPvpg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.61.1.tgz", + "integrity": "sha512-unMS3H73DpaoPyyEVPjGKleM/s0mkmsauTENpw4INQY8y4+IuLNjkueQ5QCtC0D3N38Y38yhAU8OoZ20S2Tm6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.61.1.tgz", + "integrity": "sha512-zNZzGRnAhwjFEYmvphJRV5XaQGjs62cCmeYYHUT//NbvEnHauw+I85nGG+SiVg5ld4GX8D1IbKIX+ozITQnhMQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.61.1.tgz", + "integrity": "sha512-LdpWGL8X209B2SIvWjqlc8VZgM6PKfontSerGepuldQmHYrAOtnMCXeJkxXGbC+PPZVOuu5czJo7fNV6aeW8rQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.61.1.tgz", + "integrity": "sha512-EC5kTtNaNGOmbMGqar8dvJy6y/hg99GAwjfBz++pxZhQATXGcRjd6c5en5wcbru0vkRmiMGsQKdMJOOf6sza4g==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.61.1.tgz", + "integrity": "sha512-8hiwp6D4acEcNK78I4rP0/XtS1sknWIAMJBPdR4l6zUtyTm5KiTDr5bXmWt4foY7nAN7AThDHgkLIEZOWKbzWw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.61.1.tgz", + "integrity": "sha512-10dh/h/BqA7DuMPWSxkR8uks18FRwnwOEqr5zOTEl+NOwP/OMzKX8OFR/Of9xxDA7D5qef1Nzar5WDD2kCCr1g==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.61.1.tgz", + "integrity": "sha512-YKJ5lg35DP17gcAOggnihe+APw9HLyj1Xn7gsmGumBJAUDa6NGXNixJzmkWLhcK9TOuuyQjdamzvJefkO7qHZQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.61.1.tgz", + "integrity": "sha512-Mlil5G2Jj6a7B3LWGctg+XPL9vdXYuzCtNXfxOQ0nPjc2m6ueUktocPGH9bnAM0bNRKb/bAWTujUU7IJQdQA+g==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.61.1.tgz", + "integrity": "sha512-bVWIOIk6pV01p4CdUbPP7CJ/434z+OooYjDuFcR+44N35YvKUC66G8MGnvcWx5mWKW3g61J+t74l3Kj15Kwn2Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.61.1.tgz", + "integrity": "sha512-qy5pBvZbqNFheBz61R1rzsezjm0J7O2oNGoWtGoY89SZYLUfxAJTBAqDChqAIdB4rCiIbi9nF7yZ83GnNiLwSw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.61.1.tgz", + "integrity": "sha512-E83TXjI4zm0+5f2qO+UOudaCYIhYwpJ5jq6YCZNIZ+6CbfhKrkAGezeiASBL9ElxAxFsRS9ZhESv8mfnj6TKeg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.61.1.tgz", + "integrity": "sha512-fbWnKqVkjrJN38vNe3ahkbk6iejS/3b0Nt7EEtPpE6RBacZcGXNKbzfHN3GUUlXOPghUg0j6XUGrtjX9z1sIvA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.61.1.tgz", + "integrity": "sha512-ArMl38iVAbk0New1ogihQNY6iphLi4ZaRsa037gUzv5yeKPY8TD3Dmy4x2RNC1VztU/uqm+G+/RwFrSka3Oy2g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.61.1.tgz", + "integrity": "sha512-0mYtjHS9ucAbcATycCNK9IGBk/cCe/ma7EmSLGZdsxnOA8cjRIyU04wDpVAD9NiOfLUR9KTxdiO53uOkherqjQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.61.1.tgz", + "integrity": "sha512-gK1iCEPfpoSG9wfBihXxvBMi8ZfcWffYkEsC/Eih+iFENTaewvNcrEQ69lIOWYO5pePHKLHHO7nq5AILGO/HQQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.61.1.tgz", + "integrity": "sha512-X+zaP2x+j4RXGfbp/seSoRHWnPxzApilDszisZxbYH5C/jTxFhCtDNdPGZb9lJyYPs24wGxruPF7Y+sIXt9Gzw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "license": "MIT" + }, + "node_modules/@types/geojson-vt": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/@types/geojson-vt/-/geojson-vt-3.2.5.tgz", + "integrity": "sha512-qDO7wqtprzlpe8FfQ//ClPV9xiuoh2nkIgiouIptON9w5jvD/fA4szvP9GBlDVdJ5dldAl0kX/sy3URbWwLx0g==", + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, + "node_modules/@types/mapbox__point-geometry": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@types/mapbox__point-geometry/-/mapbox__point-geometry-0.1.4.tgz", + "integrity": "sha512-mUWlSxAmYLfwnRBmgYV86tgYmMIICX4kza8YnE/eIlywGe2XoOxlpVnXWwir92xRLjwyarqwpu2EJKD2pk0IUA==", + "license": "MIT" + }, + "node_modules/@types/mapbox__vector-tile": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@types/mapbox__vector-tile/-/mapbox__vector-tile-1.3.4.tgz", + "integrity": "sha512-bpd8dRn9pr6xKvuEBQup8pwQfD4VUyqO/2deGjfpe6AwC8YRlyEipvefyRJUSiCJTZuCb8Pl1ciVV5ekqJ96Bg==", + "license": "MIT", + "dependencies": { + "@types/geojson": "*", + "@types/mapbox__point-geometry": "*", + "@types/pbf": "*" + } + }, + "node_modules/@types/pbf": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/pbf/-/pbf-3.0.5.tgz", + "integrity": "sha512-j3pOPiEcWZ34R6a6mN07mUkM4o4Lwf6hPNt8eilOeZhTFbxFXmKhvXl9Y28jotFPaI1bpPDJsbCprUoNke6OrA==", + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.31", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.31.tgz", + "integrity": "sha512-vfEqpXTvwT91yhmwdfouStN2hSKwTvyRs8qpLfADyrq/kxDw0hZM7Wk9Ug1FELj8hIby+S/+kQCSRFF32nv2Qw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", + "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, + "node_modules/@types/supercluster": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@types/supercluster/-/supercluster-7.1.3.tgz", + "integrity": "sha512-Z0pOY34GDFl3Q6hUFYf3HkTwKEE02e7QgtJppBt+beEAxnyOpJua+voGFvxINBHa06GwLFFym7gRPY2SiKIfIA==", + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz", + "integrity": "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.28.0", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.27", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.17.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/baseline-browser-mapping": { + "version": "2.10.36", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.36.tgz", + "integrity": "sha512-lVq/Df7LXlO79MVaaUHztSwWiG9oXoWHlgvNS51v8Dpd4+G4/VIy6qYePTw31nAVls33nUtnfezYeLkYAak9dg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.28.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", + "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.10.12", + "caniuse-lite": "^1.0.30001782", + "electron-to-chromium": "^1.5.328", + "node-releases": "^2.0.36", + "update-browserslist-db": "^1.2.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001799", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001799.tgz", + "integrity": "sha512-hG1bReV+OUU+MOqK4t/ZWI0tZOyz3rqS9XuhOUz1cIcbwBKjOyJEJuw9ER5JuNyqxNk8u/JUVbGibBOL1yrjFw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/earcut": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/earcut/-/earcut-3.0.2.tgz", + "integrity": "sha512-X7hshQbLyMJ/3RPhyObLARM2sNxxmRALLKx1+NVFFnQ9gKzmCrxm9+uLIAdBcvc8FNLpctqlQ2V6AE92Ol9UDQ==", + "license": "ISC" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.371", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.371.tgz", + "integrity": "sha512-e9htk9mAYL6AzmkEhSvVVw7IWGSBJ/Bqdn2eRyRLrj1g6sncN4WbFt5qnILYoCktktr45pyjIrOiRvBThQ808w==", + "dev": true, + "license": "ISC" + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/geojson-vt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/geojson-vt/-/geojson-vt-4.0.3.tgz", + "integrity": "sha512-jR1MwkLaZGa8Zftct9ZFruyWFrdl9ZyD2OliXNy9Qq5bBPeg5wHVpBQF9p5GjnicSDQqvBVpysxTPKmWdsfWMA==", + "license": "ISC" + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gl-matrix": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/gl-matrix/-/gl-matrix-3.4.4.tgz", + "integrity": "sha512-latSnyDNt/8zYUB6VIJ6PCh2jBjJX6gnDsoCZ7LyW7GkqrD51EWwa9qCoGixj8YqBtETQK/xY7OmpTF8xz1DdQ==", + "license": "MIT" + }, + "node_modules/global-prefix": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-4.0.0.tgz", + "integrity": "sha512-w0Uf9Y9/nyHinEk5vMJKRie+wa4kR5hmDbEhGGds/kG1PwGLLHKRoNMeJOyCQjjBkANlnScqgzcFwGHgmgLkVA==", + "license": "MIT", + "dependencies": { + "ini": "^4.1.3", + "kind-of": "^6.0.3", + "which": "^4.0.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/ini": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ini/-/ini-4.1.3.tgz", + "integrity": "sha512-X7rqawQBvfdjS10YU1y1YVreA3SsLrW9dX2CewP2EbBJM4ypVNLDkO5y04gejPwKIY9lR+7r9gn3rFPt/kmWFg==", + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/isexe": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.5.tgz", + "integrity": "sha512-6B3tLtFqtQS4ekarvLVMZ+X+VlvQekbe4taUkf/rhVO3d/h0M2rfARm/pXLcPEsjjMsFgrFgSrhQIxcSVrBz8w==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-stringify-pretty-compact": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/json-stringify-pretty-compact/-/json-stringify-pretty-compact-4.0.0.tgz", + "integrity": "sha512-3CNZ2DnrpByG9Nqj6Xo8vqbjT4F6N+tb4Gb28ESAZjYZ5yqvmc56J+/kuIwkaAMOyblTQhUW7PxMkUb8Q36N3Q==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kdbush": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.1.0.tgz", + "integrity": "sha512-e9vurzrXJQrFX6ckpHP3bvj5l+9CnYzkxDNnNQ1h2QTqdWsUAJgXiKdGNcOa1EY85dU8KbQ+z/FdQdB7P+9yfQ==", + "license": "ISC" + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/maplibre-gl": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-4.7.1.tgz", + "integrity": "sha512-lgL7XpIwsgICiL82ITplfS7IGwrB1OJIw/pCvprDp2dhmSSEBgmPzYRvwYYYvJGJD7fxUv1Tvpih4nZ6VrLuaA==", + "license": "BSD-3-Clause", + "dependencies": { + "@mapbox/geojson-rewind": "^0.5.2", + "@mapbox/jsonlint-lines-primitives": "^2.0.2", + "@mapbox/point-geometry": "^0.1.0", + "@mapbox/tiny-sdf": "^2.0.6", + "@mapbox/unitbezier": "^0.0.1", + "@mapbox/vector-tile": "^1.3.1", + "@mapbox/whoots-js": "^3.1.0", + "@maplibre/maplibre-gl-style-spec": "^20.3.1", + "@types/geojson": "^7946.0.14", + "@types/geojson-vt": "3.2.5", + "@types/mapbox__point-geometry": "^0.1.4", + "@types/mapbox__vector-tile": "^1.3.4", + "@types/pbf": "^3.0.5", + "@types/supercluster": "^7.1.3", + "earcut": "^3.0.0", + "geojson-vt": "^4.0.2", + "gl-matrix": "^3.4.3", + "global-prefix": "^4.0.0", + "kdbush": "^4.0.2", + "murmurhash-js": "^1.0.0", + "pbf": "^3.3.0", + "potpack": "^2.0.0", + "quickselect": "^3.0.0", + "supercluster": "^8.0.1", + "tinyqueue": "^3.0.0", + "vt-pbf": "^3.1.3" + }, + "engines": { + "node": ">=16.14.0", + "npm": ">=8.1.0" + }, + "funding": { + "url": "https://github.com/maplibre/maplibre-gl-js?sponsor=1" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/murmurhash-js": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/murmurhash-js/-/murmurhash-js-1.0.0.tgz", + "integrity": "sha512-TvmkNhkv8yct0SVBSy+o8wYzXjE4Zz3PCesbfs8HiCXXdcTuocApFv11UWlNFWKYsP2okqrhb7JNlSm9InBhIw==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.12", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", + "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.47", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.47.tgz", + "integrity": "sha512-Uzmd6LXpouKo8EUK68IjH4+E01w/hXyV3R3g/geCJo+rXLNfh1xucB+LOzYEOQPSiUK3h/xZf0cQGcSsmyL2Og==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/pbf": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/pbf/-/pbf-3.3.0.tgz", + "integrity": "sha512-XDF38WCH3z5OV/OVa8GKUNtLAyneuzbCisx7QUCF8Q6Nutx0WnJrQe5O+kOtBlLfRNUws98Y58Lblp+NJG5T4Q==", + "license": "BSD-3-Clause", + "dependencies": { + "ieee754": "^1.1.12", + "resolve-protobuf-schema": "^2.1.0" + }, + "bin": { + "pbf": "bin/pbf" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.5.15", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", + "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/potpack": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/potpack/-/potpack-2.1.0.tgz", + "integrity": "sha512-pcaShQc1Shq0y+E7GqJqvZj8DTthWV1KeHGdi0Z6IAin2Oi3JnLCOfwnCo84qc+HAp52wT9nK9H7FAJp5a44GQ==", + "license": "ISC" + }, + "node_modules/protocol-buffers-schema": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.6.1.tgz", + "integrity": "sha512-VG2K63Igkiv9p76tk1lilczEK1cT+kCjKtkdhw1dQZV3k3IXJbd3o6Ho8b9zJZaHSnT2hKe4I+ObmX9w6m5SmQ==", + "license": "MIT" + }, + "node_modules/quickselect": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/quickselect/-/quickselect-3.0.0.tgz", + "integrity": "sha512-XdjUArbK4Bm5fLLvlm5KpTFOiOThgfWWI4axAZDWg4E/0mKdZyI9tNEfds27qCi1ze/vwTR16kvmmGhRra3c2g==", + "license": "ISC" + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/react-refresh": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", + "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-protobuf-schema": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/resolve-protobuf-schema/-/resolve-protobuf-schema-2.1.0.tgz", + "integrity": "sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ==", + "license": "MIT", + "dependencies": { + "protocol-buffers-schema": "^3.3.1" + } + }, + "node_modules/rollup": { + "version": "4.61.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.61.1.tgz", + "integrity": "sha512-I4KW6iuRpuu2uHBLraZ1wNZe0DP7lnRha+VJ9tNaYVaVgKhW0aI3h4RYnoRPeql0flHm/Co55b7snEDcOfOJrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.61.1", + "@rollup/rollup-android-arm64": "4.61.1", + "@rollup/rollup-darwin-arm64": "4.61.1", + "@rollup/rollup-darwin-x64": "4.61.1", + "@rollup/rollup-freebsd-arm64": "4.61.1", + "@rollup/rollup-freebsd-x64": "4.61.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.61.1", + "@rollup/rollup-linux-arm-musleabihf": "4.61.1", + "@rollup/rollup-linux-arm64-gnu": "4.61.1", + "@rollup/rollup-linux-arm64-musl": "4.61.1", + "@rollup/rollup-linux-loong64-gnu": "4.61.1", + "@rollup/rollup-linux-loong64-musl": "4.61.1", + "@rollup/rollup-linux-ppc64-gnu": "4.61.1", + "@rollup/rollup-linux-ppc64-musl": "4.61.1", + "@rollup/rollup-linux-riscv64-gnu": "4.61.1", + "@rollup/rollup-linux-riscv64-musl": "4.61.1", + "@rollup/rollup-linux-s390x-gnu": "4.61.1", + "@rollup/rollup-linux-x64-gnu": "4.61.1", + "@rollup/rollup-linux-x64-musl": "4.61.1", + "@rollup/rollup-openbsd-x64": "4.61.1", + "@rollup/rollup-openharmony-arm64": "4.61.1", + "@rollup/rollup-win32-arm64-msvc": "4.61.1", + "@rollup/rollup-win32-ia32-msvc": "4.61.1", + "@rollup/rollup-win32-x64-gnu": "4.61.1", + "@rollup/rollup-win32-x64-msvc": "4.61.1", + "fsevents": "~2.3.2" + } + }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "license": "BSD-3-Clause" + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/supercluster": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/supercluster/-/supercluster-8.0.1.tgz", + "integrity": "sha512-IiOea5kJ9iqzD2t7QJq/cREyLHTtSmUT6gQsweojg9WH2sYJqZK9SswTu6jrscO6D1G5v5vYZ9ru/eq85lXeZQ==", + "license": "ISC", + "dependencies": { + "kdbush": "^4.0.2" + } + }, + "node_modules/tinyqueue": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tinyqueue/-/tinyqueue-3.0.0.tgz", + "integrity": "sha512-gRa9gwYU3ECmQYv3lslts5hxuIa90veaEcxDYuu3QGOIAEM2mOZkVHp48ANJuu1CURtRdHKUBY5Lm1tHV+sD4g==", + "license": "ISC" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vite": { + "version": "5.4.21", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", + "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/vt-pbf": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/vt-pbf/-/vt-pbf-3.1.3.tgz", + "integrity": "sha512-2LzDFzt0mZKZ9IpVF2r69G9bXaP2Q2sArJCmcCgvfTdCCZzSyz4aCLoQyUilu37Ll56tCblIZrXFIjNUpGIlmA==", + "license": "MIT", + "dependencies": { + "@mapbox/point-geometry": "0.1.0", + "@mapbox/vector-tile": "^1.3.1", + "pbf": "^3.2.1" + } + }, + "node_modules/which": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", + "integrity": "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg==", + "license": "ISC", + "dependencies": { + "isexe": "^3.1.1" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^16.13.0 || >=18.0.0" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + } + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 00000000..6cf390d4 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,24 @@ +{ + "name": "geointel-frontend", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "start": "vite", + "build": "tsc -b && vite build", + "preview": "vite preview", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "react": "^18.2.0", + "react-dom": "^18.2.0", + "maplibre-gl": "^4.7.1" + }, + "devDependencies": { + "@types/react": "^18.2.0", + "@types/react-dom": "^18.2.0", + "@vitejs/plugin-react": "^4.3.2", + "typescript": "^5.5.4", + "vite": "^5.4.1" + } +} diff --git a/frontend/src/.gitkeep b/frontend/src/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 00000000..32ffd9c0 --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,1763 @@ +import { FormEvent, useEffect, useMemo, useState } from 'react' +import './styles/app.css' +import GeoMap from './components/GeoMap' +import { areasApi } from './services/api/areas' +import { datasetsApi } from './services/api/datasets' +import { projectsApi } from './services/api/projects' +import { demoApi, detectionApi, jobsApi, externalApi, exportsApi, qaApi, segmentationApi } from './services/api' +import { DetectionLab } from './components/detection/DetectionLab' +import { ExportCenter } from './components/exports/ExportCenter' +import { AreaPanel } from './components/project/AreaPanel' +import { ProjectPanel } from './components/project/ProjectPanel' +import type { + ApiError, + DatasetCreateResponse, + DatasetListResponse, + DetectionQaResult, + DetectionRead, + DetectionModelCapability, + DetectionRunRead, + DetectionRunResponse, + JobRead, + QaComparisonRequest, + RasterMetadataResponse, + RasterInspectResponse, + RasterStatsResponse, + RasterPreviewResponse, + VectorSummary, + ProjectRead, + ProjectCreate, + QualityCheckRead, + ExportCreateResponse, + ExportRead, + AreaCreate, + AreaListResponse, + AreaRead, + ProviderCapability, + QaComparisonResult, + SegmentationModelCapability, + SegmentationQaResult, + SegmentationRead, + SegmentationRunRead, + SegmentationRunResponse, +} from './types' +import { ProviderPanel } from './components/providers/ProviderPanel' +import { SegmentationLab } from './components/segmentation/SegmentationLab' + +function isVectorDatasetType(datasetType: string): boolean { + return datasetType === 'vector' || datasetType === 'geojson' +} + +function formatBytes(value: number | null | undefined): string { + if (!value && value !== 0) { + return 'n/a' + } + const units = ['B', 'KB', 'MB', 'GB'] + let size = value + let index = 0 + while (size >= 1024 && index < units.length - 1) { + size /= 1024 + index += 1 + } + return `${size.toFixed(1)} ${units[index]}` +} + +function formatBounds(bounds: Record | null | undefined): string { + if (!bounds) { + return 'n/a' + } + const keys = ['min_x', 'min_y', 'max_x', 'max_y'] + if (!keys.every((key) => key in bounds)) { + return 'n/a' + } + return `${bounds.min_x?.toFixed(4)}, ${bounds.min_y?.toFixed(4)} -> ${bounds.max_x?.toFixed(4)}, ${bounds.max_y?.toFixed(4)}` +} + +function formatError(error: unknown, fallback: string): string { + if (error instanceof Error) { + const code = (error as { code?: string }).code + return code ? `${error.message} (${code})` : error.message + } + return fallback +} + +function App(): JSX.Element { + const [projects, setProjects] = useState([]) + const [selectedProjectId, setSelectedProjectId] = useState(null) + const [areas, setAreas] = useState([]) + const [datasets, setDatasets] = useState([]) + const [selectedDatasetId, setSelectedDatasetId] = useState(null) + const [selectedDataset, setSelectedDataset] = useState(null) + const [selectedDatasetSummary, setSelectedDatasetSummary] = useState(null) + const [selectedRasterMetadata, setSelectedRasterMetadata] = useState(null) + const [selectedRasterStats, setSelectedRasterStats] = useState(null) + const [datasetContent, setDatasetContent] = useState(null) + const [jobs, setJobs] = useState([]) + const [providerCapabilities, setProviderCapabilities] = useState([]) + const [loadingCapabilities, setLoadingCapabilities] = useState(false) + const [capabilitiesError, setCapabilitiesError] = useState(null) + const [detectionModels, setDetectionModels] = useState([]) + const [loadingDetectionModels, setLoadingDetectionModels] = useState(false) + const [detectionModelError, setDetectionModelError] = useState(null) + const [selectedDetectionDatasetId, setSelectedDetectionDatasetId] = useState('') + const [selectedDetectionModelId, setSelectedDetectionModelId] = useState('yolo-placeholder') + const [detectionTileManifestPath, setDetectionTileManifestPath] = useState('') + const [detectionConfidenceThreshold, setDetectionConfidenceThreshold] = useState(0.5) + const [runningDetection, setRunningDetection] = useState(false) + const [detectionRunResult, setDetectionRunResult] = useState(null) + const [detectionRunError, setDetectionRunError] = useState(null) + const [detectionRuns, setDetectionRuns] = useState([]) + const [selectedDetectionRunId, setSelectedDetectionRunId] = useState('') + const [detectionItems, setDetectionItems] = useState([]) + const [detectionGeoJson, setDetectionGeoJson] = useState(null) + const [detectionClassFilter, setDetectionClassFilter] = useState('') + const [detectionMinConfidenceFilter, setDetectionMinConfidenceFilter] = useState(0) + const [loadingDetectionResults, setLoadingDetectionResults] = useState(false) + const [detectionReferenceDatasetId, setDetectionReferenceDatasetId] = useState('') + const [detectionQaResult, setDetectionQaResult] = useState(null) + const [detectionQaError, setDetectionQaError] = useState(null) + const [runningDetectionQa, setRunningDetectionQa] = useState(false) + const [segmentationModels, setSegmentationModels] = useState([]) + const [loadingSegmentationModels, setLoadingSegmentationModels] = useState(false) + const [segmentationModelError, setSegmentationModelError] = useState(null) + const [selectedSegmentationDatasetId, setSelectedSegmentationDatasetId] = useState('') + const [selectedSegmentationModelId, setSelectedSegmentationModelId] = useState('segmentation-placeholder') + const [segmentationConfidenceThreshold, setSegmentationConfidenceThreshold] = useState(0.5) + const [runningSegmentation, setRunningSegmentation] = useState(false) + const [segmentationRunResult, setSegmentationRunResult] = useState(null) + const [segmentationRunError, setSegmentationRunError] = useState(null) + const [segmentationRuns, setSegmentationRuns] = useState([]) + const [selectedSegmentationRunId, setSelectedSegmentationRunId] = useState('') + const [segmentationItems, setSegmentationItems] = useState([]) + const [segmentationGeoJson, setSegmentationGeoJson] = useState(null) + const [segmentationClassFilter, setSegmentationClassFilter] = useState('') + const [segmentationMinConfidenceFilter, setSegmentationMinConfidenceFilter] = useState(0) + const [loadingSegmentationResults, setLoadingSegmentationResults] = useState(false) + const [segmentationReferenceDatasetId, setSegmentationReferenceDatasetId] = useState('') + const [segmentationQaResult, setSegmentationQaResult] = useState(null) + const [segmentationQaError, setSegmentationQaError] = useState(null) + const [runningSegmentationQa, setRunningSegmentationQa] = useState(false) + const [rasterPreview, setRasterPreview] = useState(null) + const [selectedIntersectTargetId, setSelectedIntersectTargetId] = useState('') + const [selectedClipAreaId, setSelectedClipAreaId] = useState('') + const [qaCandidateDatasetId, setQaCandidateDatasetId] = useState('') + const [qaReferenceDatasetId, setQaReferenceDatasetId] = useState('') + const [qaAreaId, setQaAreaId] = useState('') + const [qaIouThreshold, setQaIouThreshold] = useState(0.5) + const [qaRunning, setQaRunning] = useState(false) + const [qaResult, setQaResult] = useState(null) + const [qaError, setQaError] = useState(null) + const [qualityChecks, setQualityChecks] = useState([]) + const [qualityChecksError, setQualityChecksError] = useState(null) + const [exports, setExports] = useState([]) + const [latestExport, setLatestExport] = useState(null) + const [exportError, setExportError] = useState(null) + const [loadingExports, setLoadingExports] = useState(false) + const [exporting, setExporting] = useState(false) + const [exportPreview, setExportPreview] = useState | null>(null) + const [rasterTileSize, setRasterTileSize] = useState(512) + const [rasterTileOverlap, setRasterTileOverlap] = useState(64) + const [rasterTileOutputName, setRasterTileOutputName] = useState('') + const [rasterReprojectCrs, setRasterReprojectCrs] = useState('EPSG:31370') + const [rasterReprojectResampling, setRasterReprojectResampling] = useState('nearest') + const [ndviNirBand, setNdviNirBand] = useState(4) + const [ndviRedBand, setNdviRedBand] = useState(3) + const [ndwiGreenBand, setNdwiGreenBand] = useState(2) + const [ndwiNirBand, setNdwiNirBand] = useState(4) + const [ndbiSwirBand, setNdbiSwirBand] = useState(5) + const [ndbiNirBand, setNdbiNirBand] = useState(4) + + const [loadingProjects, setLoadingProjects] = useState(false) + const [loadingDemoWorkflow, setLoadingDemoWorkflow] = useState(false) + const [demoWorkflowMessage, setDemoWorkflowMessage] = useState(null) + const [loadingAreas, setLoadingAreas] = useState(false) + const [loadingDatasets, setLoadingDatasets] = useState(false) + const [loadingDatasetDetails, setLoadingDatasetDetails] = useState(false) + const [errorMessage, setErrorMessage] = useState(null) + const [datasetDetailError, setDatasetDetailError] = useState(null) + + const [projectForm, setProjectForm] = useState({ + name: '', + description: '', + region: 'Kempen', + }) + const [areaForm, setAreaForm] = useState({ + name: 'Demo AOI', + geometry: + '{"type":"MultiPolygon","coordinates":[[[[4.35,51.28],[4.55,51.28],[4.55,51.46],[4.35,51.46],[4.35,51.28]]]]}', + crs: 'EPSG:4326', + }) + const [datasetForm, setDatasetForm] = useState({ + datasetType: 'vector', + source: 'user_upload', + datasetRole: 'source', + sourceName: '', + referenceLayerName: '', + sourceMetadataJson: '', + provenanceMetadataJson: '', + areaId: '', + file: null as File | null, + }) + + const selectedProject = useMemo( + () => projects.find((project) => project.id === selectedProjectId) ?? null, + [projects, selectedProjectId], + ) + const rasterUnavailableMessage = useMemo(() => { + if (!selectedDataset || selectedDataset.dataset_type !== 'raster' || !selectedDataset.metadata_json) { + return null + } + const metadata = selectedDataset.metadata_json as Record + const processingCode = metadata['processing_code'] + if (processingCode === 'RASTER_PROCESSING_UNAVAILABLE') { + return String(metadata['processing_error'] ?? 'Raster processing unavailable.') + } + return null + }, [selectedDataset]) + + const availableVectorTargets = useMemo( + () => datasets.filter((item) => item.id !== selectedDatasetId && isVectorDatasetType(item.dataset_type)), + [datasets, selectedDatasetId], + ) + const availableVectorDatasets = useMemo(() => datasets.filter((item) => isVectorDatasetType(item.dataset_type)), [datasets]) + const referenceDatasets = useMemo( + () => availableVectorDatasets.filter((item) => item.dataset_role === 'reference'), + [availableVectorDatasets], + ) + const candidateDatasets = availableVectorDatasets + const providers = useMemo(() => providerCapabilities, [providerCapabilities]) + const rasterDatasets = useMemo(() => datasets.filter((item) => item.dataset_type === 'raster'), [datasets]) + const selectedSegmentationModel = useMemo( + () => segmentationModels.find((model) => model.model_id === selectedSegmentationModelId) ?? null, + [segmentationModels, selectedSegmentationModelId], + ) + const mapFeatureCollection = useMemo(() => segmentationGeoJson ?? detectionGeoJson ?? datasetContent, [segmentationGeoJson, detectionGeoJson, datasetContent]) + const isRasterTileInputValid = useMemo( + () => rasterTileSize > 0 && rasterTileOverlap >= 0 && rasterTileOverlap < rasterTileSize, + [rasterTileSize, rasterTileOverlap], + ) + + const toRasterMetadata = (metadata: Record | null | undefined): RasterMetadataResponse | null => { + if (!metadata) { + return null + } + return metadata as unknown as RasterMetadataResponse + } + + const formatRasterBounds = (bounds: number[] | undefined | null): string => { + if (!bounds || bounds.length < 4) { + return 'n/a' + } + const [minX, minY, maxX, maxY] = bounds + return `${minX.toFixed(4)}, ${minY.toFixed(4)} -> ${maxX.toFixed(4)}, ${maxY.toFixed(4)}` + } + + const loadProjects = async () => { + setLoadingProjects(true) + setErrorMessage(null) + try { + const response = await projectsApi.list() + setProjects(response.items) + if (!selectedProjectId && response.items.length > 0) { + setSelectedProjectId(response.items[0].id) + } + } catch (error) { + setErrorMessage(error instanceof Error ? error.message : 'Failed to load projects') + } finally { + setLoadingProjects(false) + } + } + + const loadProjectData = async (projectId: string) => { + setLoadingAreas(true) + setLoadingDatasets(true) + setErrorMessage(null) + try { + const [areaResponse, datasetResponse]: [AreaListResponse, DatasetListResponse] = await Promise.all([ + areasApi.list(projectId), + datasetsApi.list(projectId), + ]) + setAreas(areaResponse.items) + setDatasets(datasetResponse.items) + if (!selectedClipAreaId && areaResponse.items.length > 0) { + setSelectedClipAreaId(areaResponse.items[0].id) + } + } catch (error) { + setErrorMessage(error instanceof Error ? error.message : 'Failed to load project data') + } finally { + setLoadingAreas(false) + setLoadingDatasets(false) + } + } + + const loadCapabilities = async () => { + setLoadingCapabilities(true) + setCapabilitiesError(null) + try { + const providerResponse = await externalApi.listProviders() + setProviderCapabilities(providerResponse.providers) + } catch (error) { + setCapabilitiesError(error instanceof Error ? error.message : 'Failed to load external capabilities') + } finally { + setLoadingCapabilities(false) + } + } + + const loadDetectionModels = async () => { + setLoadingDetectionModels(true) + setDetectionModelError(null) + try { + const response = await detectionApi.listModels() + setDetectionModels(response.models) + if (!response.models.some((model) => model.model_id === selectedDetectionModelId) && response.models.length > 0) { + setSelectedDetectionModelId(response.models[0].model_id) + } + } catch (error) { + setDetectionModelError(formatError(error, 'Failed to load detection models')) + } finally { + setLoadingDetectionModels(false) + } + } + + const loadDetectionRuns = async (projectId = selectedProjectId) => { + if (!projectId) { + setDetectionRuns([]) + return + } + try { + const response = await detectionApi.listRuns({ project_id: projectId }) + setDetectionRuns(response.items) + if (!selectedDetectionRunId && response.items.length > 0) { + setSelectedDetectionRunId(response.items[0].id) + } + } catch (error) { + setDetectionRunError(formatError(error, 'Failed to load detection runs')) + } + } + + const loadDetectionResults = async (analysisRunId = selectedDetectionRunId) => { + if (!analysisRunId) { + setDetectionItems([]) + setDetectionGeoJson(null) + return + } + setLoadingDetectionResults(true) + setDetectionRunError(null) + try { + const params = { + class_name: detectionClassFilter || null, + min_confidence: detectionMinConfidenceFilter > 0 ? detectionMinConfidenceFilter : null, + } + const [detectionsResponse, geoJsonResponse] = await Promise.all([ + detectionApi.listDetections(analysisRunId, params), + detectionApi.getRunGeoJson(analysisRunId, params), + ]) + setDetectionItems(detectionsResponse.items) + setDetectionGeoJson(geoJsonResponse) + } catch (error) { + setDetectionRunError(formatError(error, 'Failed to load detection results')) + } finally { + setLoadingDetectionResults(false) + } + } + + const loadSegmentationModels = async () => { + setLoadingSegmentationModels(true) + setSegmentationModelError(null) + try { + const response = await segmentationApi.listModels() + setSegmentationModels(response.models) + if (!response.models.some((model) => model.model_id === selectedSegmentationModelId) && response.models.length > 0) { + setSelectedSegmentationModelId(response.models[0].model_id) + } + } catch (error) { + setSegmentationModelError(formatError(error, 'Failed to load segmentation models')) + } finally { + setLoadingSegmentationModels(false) + } + } + + const loadSegmentationRuns = async (projectId = selectedProjectId) => { + if (!projectId) { + setSegmentationRuns([]) + return + } + try { + const response = await segmentationApi.listRuns({ project_id: projectId }) + setSegmentationRuns(response.items) + if (!selectedSegmentationRunId && response.items.length > 0) { + setSelectedSegmentationRunId(response.items[0].id) + } + } catch (error) { + setSegmentationRunError(formatError(error, 'Failed to load segmentation runs')) + } + } + + const loadSegmentationResults = async (analysisRunId = selectedSegmentationRunId) => { + if (!analysisRunId) { + setSegmentationItems([]) + setSegmentationGeoJson(null) + return + } + setLoadingSegmentationResults(true) + setSegmentationRunError(null) + try { + const params = { + class_name: segmentationClassFilter || null, + min_confidence: segmentationMinConfidenceFilter > 0 ? segmentationMinConfidenceFilter : null, + } + const [segmentationsResponse, geoJsonResponse] = await Promise.all([ + segmentationApi.listSegmentations(analysisRunId, params), + segmentationApi.getRunGeoJson(analysisRunId, params), + ]) + setSegmentationItems(segmentationsResponse.items) + setSegmentationGeoJson(geoJsonResponse) + } catch (error) { + setSegmentationRunError(formatError(error, 'Failed to load segmentation results')) + } finally { + setLoadingSegmentationResults(false) + } + } + + const loadQualityChecks = async (projectId = selectedProjectId) => { + if (!projectId) { + setQualityChecks([]) + return + } + setQualityChecksError(null) + try { + const response = await qaApi.listQualityChecks(projectId) + setQualityChecks(response.items) + } catch (error) { + setQualityChecksError(formatError(error, 'Failed to load QA/QC results')) + } + } + + const loadExports = async (projectId = selectedProjectId) => { + if (!projectId) { + setExports([]) + return + } + setLoadingExports(true) + setExportError(null) + try { + const response = await exportsApi.listProjectExports(projectId) + setExports(response.items) + } catch (error) { + setExportError(formatError(error, 'Failed to load exports')) + } finally { + setLoadingExports(false) + } + } + + const exportSelectedDatasetGeoJson = async () => { + if (!selectedDataset || !isVectorDatasetType(selectedDataset.dataset_type)) { + setExportError('Select a vector dataset before exporting GeoJSON.') + return + } + setExporting(true) + setExportError(null) + try { + const response = await exportsApi.exportGeojson({ + dataset_id: selectedDataset.id, + export_kind: 'dataset', + name: selectedDataset.name.replace(/\.(geo)?json$/i, ''), + }) + setLatestExport(response) + await loadExports(selectedDataset.project_id) + } catch (error) { + setExportError(formatError(error, 'Failed to export selected dataset')) + } finally { + setExporting(false) + } + } + + const exportSelectedDetectionRunGeoJson = async () => { + if (!selectedDetectionRunId) { + setExportError('Select a detection run before exporting GeoJSON.') + return + } + setExporting(true) + setExportError(null) + try { + const response = await exportsApi.exportGeojson({ + analysis_run_id: selectedDetectionRunId, + export_kind: 'detection_run', + }) + setLatestExport(response) + await loadExports(selectedProjectId) + } catch (error) { + setExportError(formatError(error, 'Failed to export detection run')) + } finally { + setExporting(false) + } + } + + const exportSelectedSegmentationRunGeoJson = async () => { + if (!selectedSegmentationRunId) { + setExportError('Select a segmentation run before exporting GeoJSON.') + return + } + setExporting(true) + setExportError(null) + try { + const response = await exportsApi.exportGeojson({ + analysis_run_id: selectedSegmentationRunId, + export_kind: 'segmentation_run', + }) + setLatestExport(response) + await loadExports(selectedProjectId) + } catch (error) { + setExportError(formatError(error, 'Failed to export segmentation run')) + } finally { + setExporting(false) + } + } + + const exportProjectMetadata = async () => { + if (!selectedProjectId) { + setExportError('Select a project before exporting metadata.') + return + } + setExporting(true) + setExportError(null) + try { + const response = await exportsApi.exportProjectMetadata(selectedProjectId) + setLatestExport(response) + await loadExports(selectedProjectId) + } catch (error) { + setExportError(formatError(error, 'Failed to export project metadata')) + } finally { + setExporting(false) + } + } + + const exportProjectReport = async () => { + if (!selectedProjectId) { + setExportError('Select a project before exporting a report.') + return + } + setExporting(true) + setExportError(null) + try { + const response = await exportsApi.exportProjectReport(selectedProjectId) + setLatestExport(response) + await loadExports(selectedProjectId) + } catch (error) { + setExportError(formatError(error, 'Failed to export project report')) + } finally { + setExporting(false) + } + } + + const previewExportContent = async (exportId: string) => { + setExportError(null) + try { + const response = await exportsApi.getContent(exportId) + setExportPreview(response.content) + } catch (error) { + setExportError(formatError(error, 'Failed to load export content')) + } + } + + const downloadExportArtifact = (exportId: string) => { + window.open(exportsApi.downloadUrl(exportId), '_blank', 'noopener,noreferrer') + } + + const loadDatasetJobs = async (projectId: string, datasetId: string) => { + const response = await jobsApi.list(projectId, { dataset_id: datasetId, limit: 20, offset: 0 }) + setJobs(response.items) + } + + const loadDatasetDetails = async (projectId: string, dataset: DatasetCreateResponse) => { + setLoadingDatasetDetails(true) + setDatasetDetailError(null) + setSelectedDataset(dataset) + setSelectedDatasetSummary(null) + setSelectedRasterMetadata(null) + setSelectedRasterStats(null) + setDatasetContent(null) + setRasterPreview(null) + setSelectedDatasetId(dataset.id) + setJobs([]) + try { + if (isVectorDatasetType(dataset.dataset_type)) { + const [content, summary] = await Promise.all([ + datasetsApi.getContent(projectId, dataset.id), + datasetsApi.vectorSummary(projectId, dataset.id), + ]) + setDatasetContent(content) + setSelectedDatasetSummary(summary) + } else if (dataset.dataset_type === 'raster') { + try { + const rasterInspection = await datasetsApi.rasterInspect(projectId, dataset.id) + setSelectedRasterMetadata(toRasterMetadata(rasterInspection.metadata)) + } catch (error) { + setSelectedRasterMetadata(null) + setDatasetDetailError(formatError(error, 'Raster metadata unavailable')) + } + } + await loadDatasetJobs(projectId, dataset.id) + } catch (error) { + setDatasetDetailError(formatError(error, 'Unable to load dataset detail')) + } finally { + setLoadingDatasetDetails(false) + } + } + + useEffect(() => { + loadProjects().catch(() => null) + loadCapabilities().catch(() => null) + loadDetectionModels().catch(() => null) + loadSegmentationModels().catch(() => null) + }, []) + + useEffect(() => { + if (!selectedProjectId) { + setAreas([]) + setDatasets([]) + setSelectedDatasetId(null) + setSelectedDataset(null) + setSelectedDatasetSummary(null) + setSelectedRasterMetadata(null) + setSelectedRasterStats(null) + setJobs([]) + setSelectedDetectionDatasetId('') + setDetectionRuns([]) + setSelectedDetectionRunId('') + setDetectionItems([]) + setDetectionGeoJson(null) + setDetectionRunResult(null) + setSelectedSegmentationDatasetId('') + setSegmentationRuns([]) + setSelectedSegmentationRunId('') + setSegmentationItems([]) + setSegmentationGeoJson(null) + setSegmentationRunResult(null) + setExports([]) + setLatestExport(null) + setExportPreview(null) + return + } + loadProjectData(selectedProjectId).catch(() => null) + loadDetectionRuns(selectedProjectId).catch(() => null) + loadSegmentationRuns(selectedProjectId).catch(() => null) + loadQualityChecks(selectedProjectId).catch(() => null) + loadExports(selectedProjectId).catch(() => null) + }, [selectedProjectId]) + + useEffect(() => { + loadDetectionResults().catch(() => null) + }, [selectedDetectionRunId, detectionClassFilter, detectionMinConfidenceFilter]) + + useEffect(() => { + loadSegmentationResults().catch(() => null) + }, [selectedSegmentationRunId, segmentationClassFilter, segmentationMinConfidenceFilter]) + + const createProject = async (event: FormEvent) => { + event.preventDefault() + if (!projectForm.name.trim()) { + setErrorMessage('Project name is required') + return + } + try { + await projectsApi.create({ + name: projectForm.name.trim(), + description: projectForm.description?.trim() || undefined, + region: projectForm.region?.trim() || 'Kempen', + }) + setProjectForm((previous) => ({ ...previous, name: '', description: '' })) + await loadProjects() + } catch (error) { + setErrorMessage(error instanceof Error ? error.message : 'Failed to create project') + } + } + + const loadDemoWorkflow = async () => { + setLoadingDemoWorkflow(true) + setDemoWorkflowMessage(null) + setErrorMessage(null) + try { + const result = await demoApi.seedWorkflow() + setSelectedProjectId(result.project_id) + setSelectedDatasetId(result.candidate_dataset_id) + setQaCandidateDatasetId(result.candidate_dataset_id) + setQaReferenceDatasetId(result.reference_dataset_id) + setQaAreaId(result.area_id) + setDetectionReferenceDatasetId(result.reference_dataset_id) + setSegmentationReferenceDatasetId(result.reference_dataset_id) + setDemoWorkflowMessage(result.message) + await loadProjects() + await Promise.all([ + loadProjectData(result.project_id), + loadDetectionRuns(result.project_id), + loadSegmentationRuns(result.project_id), + loadQualityChecks(result.project_id), + loadExports(result.project_id), + ]) + } catch (error) { + setErrorMessage(formatError(error, 'Failed to load demo workflow')) + } finally { + setLoadingDemoWorkflow(false) + } + } + + const createArea = async (event: FormEvent) => { + event.preventDefault() + if (!selectedProjectId) { + setErrorMessage('Select a project first') + return + } + let geometry: AreaCreate['geometry'] + try { + geometry = JSON.parse(areaForm.geometry) as AreaCreate['geometry'] + } catch { + setErrorMessage('Invalid GeoJSON geometry JSON') + return + } + try { + await areasApi.create(selectedProjectId, { + name: areaForm.name, + crs: areaForm.crs, + geometry, + }) + await loadProjectData(selectedProjectId) + setAreaForm((previous) => ({ ...previous, name: '' })) + } catch (error) { + setErrorMessage(error instanceof Error ? error.message : 'Failed to create area') + } + } + + const uploadDataset = async (event: FormEvent) => { + event.preventDefault() + if (!selectedProjectId || !datasetForm.file) { + setErrorMessage('Select project and upload a file') + return + } + if (!datasetForm.datasetRole) { + setErrorMessage('Select dataset role') + return + } + if (datasetForm.datasetRole === 'reference' && datasetForm.datasetType !== 'vector' && datasetForm.datasetType !== 'geojson') { + setErrorMessage('Reference role requires vector dataset upload') + return + } + if (datasetForm.sourceMetadataJson) { + try { + const parsedSourceMetadata = JSON.parse(datasetForm.sourceMetadataJson) + if (parsedSourceMetadata === null || typeof parsedSourceMetadata !== 'object') { + setErrorMessage('Source metadata must be a JSON object') + return + } + } catch { + setErrorMessage('Source metadata must be valid JSON') + return + } + } + if (datasetForm.provenanceMetadataJson) { + try { + const parsedProvenanceMetadata = JSON.parse(datasetForm.provenanceMetadataJson) + if (parsedProvenanceMetadata === null || typeof parsedProvenanceMetadata !== 'object') { + setErrorMessage('Provenance metadata must be a JSON object') + return + } + } catch { + setErrorMessage('Provenance metadata must be valid JSON') + return + } + } + try { + await datasetsApi.upload(selectedProjectId, { + file: datasetForm.file, + datasetType: datasetForm.datasetType, + source: datasetForm.source, + datasetRole: datasetForm.datasetRole, + sourceName: datasetForm.sourceName || undefined, + referenceLayerName: datasetForm.referenceLayerName || undefined, + sourceMetadataJson: datasetForm.sourceMetadataJson || undefined, + provenanceMetadataJson: datasetForm.provenanceMetadataJson || undefined, + areaId: datasetForm.areaId || undefined, + }) + setDatasetForm((previous) => ({ ...previous, file: null })) + await loadProjectData(selectedProjectId) + } catch (error) { + setErrorMessage(error instanceof Error ? error.message : 'Failed to upload dataset') + } + } + + const runVectorClip = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + if (!selectedClipAreaId) { + setDatasetDetailError('Select an area for clipping') + return + } + setDatasetDetailError(null) + try { + await datasetsApi.vectorClip(selectedProjectId, selectedDatasetId, { + area_id: selectedClipAreaId, + output_name: `${selectedDataset?.name.replace(/\.geojson$/, '')}-clipped`, + }) + await Promise.all([loadDatasetJobs(selectedProjectId, selectedDatasetId), loadProjectData(selectedProjectId)]) + const refreshed = datasets.find((dataset) => dataset.id === selectedDatasetId) + if (refreshed && selectedProjectId) { + loadDatasetDetails(selectedProjectId, refreshed).catch(() => null) + } + } catch (error) { + setDatasetDetailError(formatError(error, 'Vector clip failed')) + } + } + + const runVectorBuffer = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + setDatasetDetailError(null) + try { + await datasetsApi.vectorBuffer(selectedProjectId, selectedDatasetId, { + distance_m: 25, + dissolve: false, + }) + await Promise.all([loadDatasetJobs(selectedProjectId, selectedDatasetId), loadProjectData(selectedProjectId)]) + const refreshed = datasets.find((dataset) => dataset.id === selectedDatasetId) + if (refreshed && selectedProjectId) { + loadDatasetDetails(selectedProjectId, refreshed).catch(() => null) + } + } catch (error) { + setDatasetDetailError(formatError(error, 'Vector buffer failed')) + } + } + + const runVectorIntersect = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + const targetId = selectedIntersectTargetId || availableVectorTargets[0]?.id + if (!targetId) { + setDatasetDetailError('Select an intersect target dataset') + return + } + setDatasetDetailError(null) + try { + await datasetsApi.vectorIntersect(selectedProjectId, selectedDatasetId, { + other_dataset_id: targetId, + }) + await Promise.all([loadDatasetJobs(selectedProjectId, selectedDatasetId), loadProjectData(selectedProjectId)]) + const refreshed = datasets.find((dataset) => dataset.id === selectedDatasetId) + if (refreshed && selectedProjectId) { + loadDatasetDetails(selectedProjectId, refreshed).catch(() => null) + } + } catch (error) { + setDatasetDetailError(formatError(error, 'Vector intersect failed')) + } + } + + const runRasterInspect = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + try { + const response = await datasetsApi.inspectRaster(selectedProjectId, selectedDatasetId) + setSelectedRasterMetadata(toRasterMetadata(response.metadata)) + setDatasetDetailError(null) + } catch (error) { + setDatasetDetailError(formatError(error, 'Raster inspect unavailable')) + } + } + + const runRasterPreview = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + try { + const response = await datasetsApi.rasterPreview(selectedProjectId, selectedDatasetId) + setRasterPreview(response) + if (response.metadata) { + setSelectedRasterMetadata(toRasterMetadata(response.metadata)) + } + setDatasetDetailError(null) + } catch (error) { + setDatasetDetailError(formatError(error, 'Raster preview unavailable')) + setRasterPreview(null) + } + } + + const runRasterStats = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + try { + const response = await datasetsApi.rasterStats(selectedProjectId, selectedDatasetId) + setSelectedRasterStats(response) + setDatasetDetailError(null) + } catch (error) { + setDatasetDetailError(formatError(error, 'Raster stats unavailable')) + setSelectedRasterStats(null) + } + } + + const runRasterReproject = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + const targetCrs = rasterReprojectCrs.trim() + if (!targetCrs) { + setDatasetDetailError('Target CRS is required for raster reproject') + return + } + try { + await datasetsApi.rasterReproject(selectedProjectId, selectedDatasetId, { + target_crs: targetCrs, + resampling: rasterReprojectResampling, + }) + await Promise.all([loadDatasetJobs(selectedProjectId, selectedDatasetId), loadProjectData(selectedProjectId)]) + const refreshed = datasets.find((dataset) => dataset.id === selectedDatasetId) + if (refreshed && selectedProjectId) { + loadDatasetDetails(selectedProjectId, refreshed).catch(() => null) + } + setDatasetDetailError(null) + } catch (error) { + setDatasetDetailError(formatError(error, 'Raster reproject failed')) + } + } + + const runRasterClip = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + if (!selectedClipAreaId) { + setDatasetDetailError('Select an area for raster clip') + return + } + try { + await datasetsApi.rasterClip(selectedProjectId, selectedDatasetId, { + area_id: selectedClipAreaId, + output_name: `raster-clipped-${selectedDatasetId}`, + }) + await Promise.all([loadDatasetJobs(selectedProjectId, selectedDatasetId), loadProjectData(selectedProjectId)]) + const refreshed = datasets.find((dataset) => dataset.id === selectedDatasetId) + if (refreshed && selectedProjectId) { + loadDatasetDetails(selectedProjectId, refreshed).catch(() => null) + } + setDatasetDetailError(null) + } catch (error) { + setDatasetDetailError(formatError(error, 'Raster clip failed')) + } + } + + const runRasterTile = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + try { + await datasetsApi.rasterTile(selectedProjectId, selectedDatasetId, { + tile_size: rasterTileSize, + overlap: rasterTileOverlap, + output_name: rasterTileOutputName || undefined, + }) + await Promise.all([loadDatasetJobs(selectedProjectId, selectedDatasetId), loadProjectData(selectedProjectId)]) + const refreshed = datasets.find((dataset) => dataset.id === selectedDatasetId) + if (refreshed && selectedProjectId) { + loadDatasetDetails(selectedProjectId, refreshed).catch(() => null) + } + setDatasetDetailError(null) + } catch (error) { + setDatasetDetailError(formatError(error, 'Raster tile failed')) + } + } + + const runRasterNdvi = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + try { + await datasetsApi.rasterNdvi(selectedProjectId, selectedDatasetId, { + nir_band: ndviNirBand, + red_band: ndviRedBand, + }) + await Promise.all([loadDatasetJobs(selectedProjectId, selectedDatasetId), loadProjectData(selectedProjectId)]) + const refreshed = datasets.find((dataset) => dataset.id === selectedDatasetId) + if (refreshed && selectedProjectId) { + loadDatasetDetails(selectedProjectId, refreshed).catch(() => null) + } + setDatasetDetailError(null) + } catch (error) { + setDatasetDetailError(formatError(error, 'Raster NDVI failed')) + } + } + + const runRasterNdwi = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + try { + await datasetsApi.rasterNdwi(selectedProjectId, selectedDatasetId, { + green_band: ndwiGreenBand, + nir_band: ndwiNirBand, + }) + await Promise.all([loadDatasetJobs(selectedProjectId, selectedDatasetId), loadProjectData(selectedProjectId)]) + const refreshed = datasets.find((dataset) => dataset.id === selectedDatasetId) + if (refreshed && selectedProjectId) { + loadDatasetDetails(selectedProjectId, refreshed).catch(() => null) + } + setDatasetDetailError(null) + } catch (error) { + setDatasetDetailError(formatError(error, 'Raster NDWI failed')) + } + } + + const runRasterNdbi = async () => { + if (!selectedProjectId || !selectedDatasetId) { + return + } + try { + await datasetsApi.rasterNdbi(selectedProjectId, selectedDatasetId, { + swir_band: ndbiSwirBand, + nir_band: ndbiNirBand, + }) + await Promise.all([loadDatasetJobs(selectedProjectId, selectedDatasetId), loadProjectData(selectedProjectId)]) + const refreshed = datasets.find((dataset) => dataset.id === selectedDatasetId) + if (refreshed && selectedProjectId) { + loadDatasetDetails(selectedProjectId, refreshed).catch(() => null) + } + setDatasetDetailError(null) + } catch (error) { + setDatasetDetailError(formatError(error, 'Raster NDBI failed')) + } + } + + const runQaComparison = async () => { + if (!selectedProjectId) { + setQaError('Select a project first') + return + } + if (!qaCandidateDatasetId) { + setQaError('Select candidate dataset') + return + } + if (!qaReferenceDatasetId) { + setQaError('Select reference dataset') + return + } + if (qaCandidateDatasetId === qaReferenceDatasetId) { + setQaError('Candidate and reference datasets must be different') + return + } + if (!Number.isFinite(qaIouThreshold) || qaIouThreshold < 0 || qaIouThreshold > 1) { + setQaError('IoU threshold must be between 0 and 1') + return + } + setQaError(null) + setQaResult(null) + setQaRunning(true) + try { + const request: QaComparisonRequest = { + candidate_dataset_id: qaCandidateDatasetId, + reference_dataset_id: qaReferenceDatasetId, + iou_threshold: qaIouThreshold, + area_id: qaAreaId || null, + } + const job: JobRead = await qaApi.runQa(request) + if (job.status === 'failed') { + setQaError(job.error_message || 'QA comparison failed') + return + } + const payload = job.result_json + if (!payload || typeof payload !== 'object') { + setQaError('QA result was not available') + return + } + const parsed = payload as unknown as QaComparisonResult + if (!parsed || typeof parsed.status !== 'string') { + setQaError('QA result format was unexpected') + return + } + setQaResult(parsed) + await loadQualityChecks(selectedProjectId) + if (job.output_dataset_id) { + await loadProjectData(selectedProjectId) + } + } catch (error) { + setQaError(error instanceof Error ? error.message : 'QA comparison failed') + } finally { + setQaRunning(false) + } + } + + const runDetection = async () => { + if (!selectedProjectId) { + setDetectionRunError('Select a project first') + return + } + const datasetId = selectedDetectionDatasetId || rasterDatasets[0]?.id + if (!datasetId) { + setDetectionRunError('Select a raster dataset') + return + } + setDetectionRunError(null) + setDetectionRunResult(null) + setRunningDetection(true) + try { + const result = await detectionApi.run({ + project_id: selectedProjectId, + dataset_id: datasetId, + model_id: selectedDetectionModelId, + confidence_threshold: detectionConfidenceThreshold, + tile_manifest_path: detectionTileManifestPath.trim() || null, + parameters_json: {}, + }) + setDetectionRunResult(result) + setSelectedDetectionRunId(result.analysis_run_id) + await loadDetectionRuns(selectedProjectId) + await loadDetectionResults(result.analysis_run_id) + await loadProjectData(selectedProjectId) + } catch (error) { + setDetectionRunError(formatError(error, 'Detection run failed')) + } finally { + setRunningDetection(false) + } + } + + const runDetectionQa = async () => { + if (!selectedDetectionRunId) { + setDetectionQaError('Select a detection run') + return + } + if (!detectionReferenceDatasetId) { + setDetectionQaError('Select a reference dataset') + return + } + setDetectionQaError(null) + setDetectionQaResult(null) + setRunningDetectionQa(true) + try { + const result = await detectionApi.compareWithReference(selectedDetectionRunId, { + reference_dataset_id: detectionReferenceDatasetId, + iou_threshold: qaIouThreshold, + class_name: detectionClassFilter || null, + min_confidence: detectionMinConfidenceFilter > 0 ? detectionMinConfidenceFilter : null, + }) + setDetectionQaResult(result) + await loadQualityChecks(selectedProjectId) + } catch (error) { + setDetectionQaError(formatError(error, 'Detection QA failed')) + } finally { + setRunningDetectionQa(false) + } + } + + const runSegmentation = async () => { + if (!selectedProjectId) { + setSegmentationRunError('Select a project first') + return + } + const datasetId = selectedSegmentationDatasetId || rasterDatasets[0]?.id + if (!datasetId) { + setSegmentationRunError('Select a raster dataset') + return + } + if (!selectedSegmentationModel?.configured) { + setSegmentationRunError('Selected segmentation model is not configured') + return + } + setSegmentationRunError(null) + setSegmentationRunResult(null) + setRunningSegmentation(true) + try { + const parameters = + selectedSegmentationModelId === 'fixture-segmenter' + ? { fixture_mode: true, fixture_segmentations: [] } + : {} + const result = await segmentationApi.run({ + project_id: selectedProjectId, + dataset_id: datasetId, + model_id: selectedSegmentationModelId, + confidence_threshold: segmentationConfidenceThreshold, + parameters_json: parameters, + }) + setSegmentationRunResult(result) + setSelectedSegmentationRunId(result.analysis_run_id) + await loadSegmentationRuns(selectedProjectId) + await loadSegmentationResults(result.analysis_run_id) + await loadProjectData(selectedProjectId) + } catch (error) { + setSegmentationRunError(formatError(error, 'Segmentation run failed')) + } finally { + setRunningSegmentation(false) + } + } + + const runSegmentationQa = async () => { + if (!selectedSegmentationRunId) { + setSegmentationQaError('Select a segmentation run') + return + } + if (!segmentationReferenceDatasetId) { + setSegmentationQaError('Select a reference dataset') + return + } + setSegmentationQaError(null) + setSegmentationQaResult(null) + setRunningSegmentationQa(true) + try { + const result = await segmentationApi.compareWithReference(selectedSegmentationRunId, { + reference_dataset_id: segmentationReferenceDatasetId, + iou_threshold: qaIouThreshold, + class_name: segmentationClassFilter || null, + min_confidence: segmentationMinConfidenceFilter > 0 ? segmentationMinConfidenceFilter : null, + }) + setSegmentationQaResult(result) + await loadQualityChecks(selectedProjectId) + } catch (error) { + setSegmentationQaError(formatError(error, 'Segmentation QA failed')) + } finally { + setRunningSegmentationQa(false) + } + } + + const pickDerivedDataset = async (datasetId: string) => { + if (!selectedProjectId) { + return + } + const target = datasets.find((item) => item.id === datasetId) + if (target) { + await loadDatasetDetails(selectedProjectId, target) + } + } + + const refreshMetadata = async (datasetId: string) => { + if (!selectedProjectId) { + return + } + const refreshed = await datasetsApi.refreshMetadata(selectedProjectId, datasetId) + await loadProjectData(selectedProjectId) + if (selectedDataset?.id === datasetId) { + setSelectedDataset(refreshed) + if (isVectorDatasetType(refreshed.dataset_type)) { + loadDatasetDetails(selectedProjectId, refreshed).catch(() => null) + } + } + } + + return ( +
+
+

GeoIntel Kempen Sprint 9

+

Sprint 9: raster/vector workbench with detection and segmentation foundations.

+
+ + {errorMessage ?

{errorMessage}

: null} + +
+ + + + + + + loadDetectionRuns()} + onSelectRun={setSelectedDetectionRunId} + onSetClassFilter={setDetectionClassFilter} + onSetMinConfidenceFilter={setDetectionMinConfidenceFilter} + onLoadResults={() => loadDetectionResults()} + onSelectReferenceDataset={setDetectionReferenceDatasetId} + onRunQa={runDetectionQa} + /> + + loadSegmentationRuns()} + onSelectRun={setSelectedSegmentationRunId} + onSetClassFilter={setSegmentationClassFilter} + onSetMinConfidenceFilter={setSegmentationMinConfidenceFilter} + onLoadResults={() => loadSegmentationResults()} + onSelectReferenceDataset={setSegmentationReferenceDatasetId} + onRunQa={runSegmentationQa} + /> + +
+

QA/QC Results

+ + {qualityChecksError ?

{qualityChecksError}

: null} + {qualityChecks.length === 0 ?

No persisted QA/QC results yet

: null} +
    + {qualityChecks.map((check) => ( +
  • + {check.check_type} +
    status: {check.status}
    +
    score: {check.score ?? 'n/a'}
    +
    candidate: {check.candidate_dataset_id ?? 'n/a'}
    +
    reference: {check.reference_dataset_id}
    +
    quality check: {check.id}
    +
      + {check.metrics.map((metric) => ( +
    • + {metric.metric_key}: {metric.metric_value ?? 'n/a'} +
      • + ))} +
    +
    • + ))} +
+
+ + loadExports()} + onExportDataset={exportSelectedDatasetGeoJson} + onExportDetectionRun={exportSelectedDetectionRunGeoJson} + onExportSegmentationRun={exportSelectedSegmentationRunGeoJson} + onExportProjectMetadata={exportProjectMetadata} + onExportProjectReport={exportProjectReport} + onPreviewContent={previewExportContent} + onDownload={downloadExportArtifact} + /> + {exportPreview ? ( +
+

Export Preview

+ 
{JSON.stringify(exportPreview, null, 2)}
+
+ ) : null} + +
+

Datasets

+
+ + setDatasetForm((previous) => ({ ...previous, source: event.target.value }))} + placeholder="user_upload" + /> + + setDatasetForm((previous) => ({ ...previous, file: event.target.files?.[0] ?? null }))} + /> + +
+ + {loadingDatasets ?

Loading datasets...

: null} + {datasets.length === 0 ?

No datasets yet

: null} +
    + {datasets.map((dataset) => ( +
  • + {dataset.name} +
    type: {dataset.dataset_type}
    +
    status: {dataset.status}
    +
    readiness: {dataset.status === 'ready' ? 'ready' : dataset.status === 'failed' ? 'failed' : 'pending'}
    +
    size: {formatBytes(dataset.size_bytes)}
    +
    features: {dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 'n/a'}
    +
    bbox: {formatBounds(dataset.bounds_json ?? dataset.vector_summary?.bounds_json)}
    + + +
    • + ))} +
+
+
+ +
+

Dataset details

+ {selectedDatasetId ?

Selected dataset: {selectedDatasetId}

:

No dataset selected

} + {selectedDataset ? ( +
+

+ {selectedDataset.name} +

+

Type: {selectedDataset.dataset_type}

+

Status: {selectedDataset.status}

+

Original file: {selectedDataset.original_filename ?? 'n/a'}

+

Stored file: {selectedDataset.stored_filename ?? 'n/a'}

+

Content type: {selectedDataset.content_type ?? 'n/a'}

+

File size: {formatBytes(selectedDataset.size_bytes)}

+

SHA256: {selectedDataset.checksum_sha256 ?? 'n/a'}

+

Feature count: {selectedDatasetSummary?.feature_count ?? selectedDataset.feature_count ?? 'n/a'}

+

BBox: {formatBounds(selectedDatasetSummary?.bounds_json ?? selectedDataset.bounds_json)}

+ {selectedDataset.dataset_type === 'raster' ? ( +
+

Raster driver: {selectedRasterMetadata?.driver ?? 'n/a'}

+

Raster size: {selectedRasterMetadata ? `${selectedRasterMetadata.width} x ${selectedRasterMetadata.height}` : 'n/a'}

+

Raster checksum: {selectedRasterMetadata?.checksum_sha256 ?? 'n/a'}

+

+ Profile: CRS {selectedRasterMetadata?.crs ?? 'n/a'} | bands {selectedRasterMetadata?.band_count ?? 'n/a'} | dtype { + (selectedRasterMetadata?.dtype as string[] | undefined)?.join(', ') ?? 'n/a'} +

+

Bounds: {formatRasterBounds(selectedRasterMetadata?.bounds)}

+

Resolution: {selectedRasterMetadata?.resolution ? selectedRasterMetadata.resolution.join(', ') : 'n/a'}

+ {rasterUnavailableMessage ?

Raster unavailable: {rasterUnavailableMessage}

: null} +

Raster operations

+

Available operations: inspect, stats, reproject, preview, clip by selected area, tile generation.

+

Preview: {rasterPreview?.preview.path ?? 'not generated'}

+

Preview size: {rasterPreview?.preview.width ?? 'n/a'} x {rasterPreview?.preview.height ?? 'n/a'}

+ + + + {selectedRasterStats ? ( +
+

Band statistics

+

Generated: {selectedRasterStats.generated_at ?? 'n/a'}

+
    + {selectedRasterStats.bands.map((band) => ( +
  • + Band {band.band_index}: min {band.min ?? 'n/a'}, max {band.max ?? 'n/a'}, mean {band.mean ?? 'n/a'}, std {band.std ?? 'n/a'}, + valid {band.valid_pixel_count}, nodata ratio {(band.nodata_ratio * 100).toFixed(2)}%, dtype {band.dtype ?? 'n/a'} +
    • + ))} +
+
+ ) : null} +
+ + + +
+
+ + + {areas.length === 0 ?

Create an area before raster clipping.

: null} +
+
+ + + + + {!isRasterTileInputValid ? ( +

+ Tile size must be {'>'} 0 and overlap must be {'>='} 0 and smaller than tile size. +

+ ) : null} +
+

Spectral indices

+
+

Use available band indexes from the raster file (1-based).

+
+

NDVI

+ + + +
+
+

NDWI

+ + + +
+
+

NDBI

+ + + +
+
+
+ ) : null} + + {isVectorDatasetType(selectedDataset.dataset_type) ? ( +
+

Vector operations

+
+ + +
+ +
+ + +
+
+ ) : null} + +

Jobs

+ {jobs.length === 0 ?

No jobs yet.

: null} +
    + {jobs.map((job) => ( +
  • +
    + {job.job_type} · {job.status} +
    + {job.result_json ? ( + 
    {JSON.stringify(job.result_json, null, 2)}
    + ) : null} + {job.error_message ?
    error: {job.error_message}
    : null} + {job.result_json?.output_dataset_id ? ( + + ) : null} +
    • + ))} +
+
+ ) : null} + {loadingDatasetDetails ?

Loading dataset details...

: null} + {datasetDetailError ?

Dataset detail error: {datasetDetailError}

: null} +
+ +
+

Map workspace

+

{selectedDatasetId ? `Showing dataset ${selectedDatasetId}` : 'No vector dataset selected'}

+ +
+
+ ) +} + +export default App diff --git a/frontend/src/app/.gitkeep b/frontend/src/app/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/frontend/src/components/.gitkeep b/frontend/src/components/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/frontend/src/components/GeoMap.tsx b/frontend/src/components/GeoMap.tsx new file mode 100644 index 00000000..8f997b1b --- /dev/null +++ b/frontend/src/components/GeoMap.tsx @@ -0,0 +1,116 @@ +import { useEffect, useRef } from 'react' +import maplibregl from 'maplibre-gl' +import 'maplibre-gl/dist/maplibre-gl.css' + +interface GeoMapProps { + data: GeoJSON.FeatureCollection | null +} + +function collectCoordinates(featureCollection: GeoJSON.FeatureCollection): maplibregl.LngLatBoundsLike | null { + const coordinates: [number, number][] = [] + const walk = (coords: unknown) => { + if (!Array.isArray(coords)) { + return + } + if (coords.length === 2 && typeof coords[0] === 'number' && typeof coords[1] === 'number') { + coordinates.push([coords[0], coords[1]]) + return + } + for (const item of coords) { + walk(item) + } + } + + for (const feature of featureCollection.features) { + const geometry = feature.geometry as any + if (geometry && geometry.coordinates) { + walk(geometry.coordinates) + } + } + + if (coordinates.length === 0) { + return null + } + + const xs = coordinates.map((point) => point[0]) + const ys = coordinates.map((point) => point[1]) + return [ + [Math.min(...xs), Math.min(...ys)], + [Math.max(...xs), Math.max(...ys)], + ] +} + +function GeoMap({ data }: GeoMapProps): JSX.Element { + const containerRef = useRef(null) + const mapRef = useRef(null) + + useEffect(() => { + if (!containerRef.current || mapRef.current) { + return + } + + const map = new maplibregl.Map({ + container: containerRef.current, + style: import.meta.env.VITE_MAP_STYLE_URL || 'https://demotiles.maplibre.org/style.json', + center: [5.3, 51.3], + zoom: 9, + }) + map.addControl(new maplibregl.NavigationControl(), 'top-right') + mapRef.current = map + + return () => { + map.remove() + mapRef.current = null + } + }, []) + + useEffect(() => { + const map = mapRef.current + if (!map) { + return + } + + if (map.getSource('dataset')) { + if (data) { + ;(map.getSource('dataset') as maplibregl.GeoJSONSource).setData(data) + } else { + if (map.getLayer('dataset-fill')) { + map.removeLayer('dataset-fill') + } + if (map.getLayer('dataset-line')) { + map.removeLayer('dataset-line') + } + map.removeSource('dataset') + return + } + } else if (data) { + map.addSource('dataset', { type: 'geojson', data }) + map.addLayer({ + id: 'dataset-fill', + type: 'fill', + source: 'dataset', + paint: { 'fill-color': '#f97316', 'fill-opacity': 0.4 }, + }) + map.addLayer({ + id: 'dataset-line', + type: 'line', + source: 'dataset', + paint: { 'line-color': '#ea580c', 'line-width': 2 }, + }) + } + + if (data) { + const collection = data + if (collection.type === 'FeatureCollection' && collection.features.length > 0) { + const bounds = collectCoordinates(collection) + if (bounds) { + map.fitBounds(bounds, { padding: 40 }) + } + } + } + }, [data]) + + return
+} + +export default GeoMap diff --git a/frontend/src/components/detection/DetectionLab.tsx b/frontend/src/components/detection/DetectionLab.tsx new file mode 100644 index 00000000..5f324806 --- /dev/null +++ b/frontend/src/components/detection/DetectionLab.tsx @@ -0,0 +1,241 @@ +import type { + DatasetCreateResponse, + DetectionModelCapability, + DetectionQaResult, + DetectionRead, + DetectionRunRead, + DetectionRunResponse, +} from '../../types' + +interface DetectionLabProps { + detectionModels: DetectionModelCapability[] + loadingDetectionModels: boolean + detectionModelError: string | null + selectedDetectionDatasetId: string + selectedDetectionModelId: string + detectionTileManifestPath: string + detectionConfidenceThreshold: number + runningDetection: boolean + detectionRunResult: DetectionRunResponse | null + detectionRunError: string | null + detectionRuns: DetectionRunRead[] + selectedDetectionRunId: string + detectionItems: DetectionRead[] + detectionClassFilter: string + detectionMinConfidenceFilter: number + loadingDetectionResults: boolean + detectionReferenceDatasetId: string + detectionQaResult: DetectionQaResult | null + detectionQaError: string | null + runningDetectionQa: boolean + selectedProjectId: string | null + rasterDatasets: DatasetCreateResponse[] + referenceDatasets: DatasetCreateResponse[] + onLoadModels: () => void + onSelectDataset: (datasetId: string) => void + onSelectModel: (modelId: string) => void + onSetConfidenceThreshold: (value: number) => void + onSetTileManifestPath: (value: string) => void + onRunDetection: () => void + onLoadRuns: () => void + onSelectRun: (runId: string) => void + onSetClassFilter: (value: string) => void + onSetMinConfidenceFilter: (value: number) => void + onLoadResults: () => void + onSelectReferenceDataset: (datasetId: string) => void + onRunQa: () => void +} + +export function DetectionLab({ + detectionModels, + loadingDetectionModels, + detectionModelError, + selectedDetectionDatasetId, + selectedDetectionModelId, + detectionTileManifestPath, + detectionConfidenceThreshold, + runningDetection, + detectionRunResult, + detectionRunError, + detectionRuns, + selectedDetectionRunId, + detectionItems, + detectionClassFilter, + detectionMinConfidenceFilter, + loadingDetectionResults, + detectionReferenceDatasetId, + detectionQaResult, + detectionQaError, + runningDetectionQa, + selectedProjectId, + rasterDatasets, + referenceDatasets, + onLoadModels, + onSelectDataset, + onSelectModel, + onSetConfidenceThreshold, + onSetTileManifestPath, + onRunDetection, + onLoadRuns, + onSelectRun, + onSetClassFilter, + onSetMinConfidenceFilter, + onLoadResults, + onSelectReferenceDataset, + onRunQa, +}: DetectionLabProps): JSX.Element { + return ( +
+

Detection Lab

+ + {loadingDetectionModels ?

Loading detection models...

: null} + {detectionModelError ?

{detectionModelError}

: null} + {detectionModels.length === 0 && !loadingDetectionModels ?

No detection models reported by backend

: null} +
    + {detectionModels.map((model) => ( +
  • + {model.display_name} +
    model: {model.model_id}
    +
    framework: {model.framework}
    +
    task: {model.task_type}
    +
    status: {model.status}
    +
    configured: {model.configured ? 'yes' : 'no'}
    +
    classes: {model.supported_classes.join(', ')}
    +
    limitation: {model.limitation_message}
    +
    • + ))} +
+
+ + + onSetConfidenceThreshold(Number(event.target.value))} + /> + {selectedDetectionModelId === 'yolo-configured' ? ( + onSetTileManifestPath(event.target.value)} + /> + ) : null} + +
+ {detectionRunError ?

{detectionRunError}

: null} + {detectionRunResult ? ( +
+

Status: {detectionRunResult.status}

+

Message: {detectionRunResult.message}

+

Analysis run: {detectionRunResult.analysis_run_id}

+

Job: {detectionRunResult.job_id}

+

Detections: {detectionRunResult.detection_count}

+ {detectionRunResult.error_code ?

Code: {detectionRunResult.error_code}

: null} +
+ ) : null} +
+

Detection results

+ + + onSetClassFilter(event.target.value)} + /> + onSetMinConfidenceFilter(Number(event.target.value))} + /> + + {loadingDetectionResults ?

Loading detection results...

: null} +

Detections loaded: {detectionItems.length}

+ {detectionItems.length > 0 ? ( + + + + + + + + + + + {detectionItems.map((detection) => ( + + + + + + + ))} + +
ClassConfidenceModelSource tile
{detection.class_name}{detection.confidence.toFixed(2)}{detection.model_name}{detection.source_tile_path || 'n/a'}
+ ) : null} +
+
+

Detection QA

+ + + {detectionQaError ?

{detectionQaError}

: null} + {detectionQaResult ? ( +
+

Status: {detectionQaResult.status}

+

Quality check: {detectionQaResult.quality_check_id}

+

Precision: {detectionQaResult.precision?.toFixed(3) ?? 'n/a'}

+

Recall: {detectionQaResult.recall?.toFixed(3) ?? 'n/a'}

+

F1: {detectionQaResult.f1_score?.toFixed(3) ?? 'n/a'}

+

Mean IoU: {detectionQaResult.mean_iou?.toFixed(3) ?? 'n/a'}

+

False positives: {detectionQaResult.false_positives}

+

False negatives: {detectionQaResult.false_negatives}

+
+ ) : null} +
+
+ ) +} diff --git a/frontend/src/components/exports/ExportCenter.tsx b/frontend/src/components/exports/ExportCenter.tsx new file mode 100644 index 00000000..247c5fb8 --- /dev/null +++ b/frontend/src/components/exports/ExportCenter.tsx @@ -0,0 +1,94 @@ +import type { DatasetCreateResponse, ExportCreateResponse, ExportRead } from '../../types' + +interface ExportCenterProps { + selectedProjectId: string | null + selectedDataset: DatasetCreateResponse | null + selectedDetectionRunId: string + selectedSegmentationRunId: string + exports: ExportRead[] + latestExport: ExportCreateResponse | null + exportError: string | null + loadingExports: boolean + exporting: boolean + onRefresh: () => void + onExportDataset: () => void + onExportDetectionRun: () => void + onExportSegmentationRun: () => void + onExportProjectMetadata: () => void + onExportProjectReport: () => void + onPreviewContent: (exportId: string) => void + onDownload: (exportId: string) => void +} + +function isVectorDatasetType(datasetType: string): boolean { + return datasetType === 'vector' || datasetType === 'geojson' +} + +export function ExportCenter({ + selectedProjectId, + selectedDataset, + selectedDetectionRunId, + selectedSegmentationRunId, + exports, + latestExport, + exportError, + loadingExports, + exporting, + onRefresh, + onExportDataset, + onExportDetectionRun, + onExportSegmentationRun, + onExportProjectMetadata, + onExportProjectReport, + onPreviewContent, + onDownload, +}: ExportCenterProps): JSX.Element { + const canExportDataset = Boolean(selectedDataset && isVectorDatasetType(selectedDataset.dataset_type)) + + return ( +
+

Export Center

+ + + + + + + {exportError ?

{exportError}

: null} + {latestExport ? ( +

+ Latest export: {latestExport.export_type} {'->'} {latestExport.path} +

+ ) : null} + {exports.length === 0 ?

No exports registered yet.

: null} +
    + {exports.map((item) => ( +
  • + {item.export_type} +
    status: {item.status}
    +
    path: {item.storage_path}
    +
    export id: {item.id}
    + + +
    • + ))} +
+
+ ) +} diff --git a/frontend/src/components/project/AreaPanel.tsx b/frontend/src/components/project/AreaPanel.tsx new file mode 100644 index 00000000..842889de --- /dev/null +++ b/frontend/src/components/project/AreaPanel.tsx @@ -0,0 +1,66 @@ +import type { FormEvent } from 'react' +import type { AreaRead, ProjectRead } from '../../types' + +interface AreaFormState { + name: string + geometry: string + crs: string +} + +interface AreaPanelProps { + areas: AreaRead[] + selectedProject: ProjectRead | null + selectedProjectId: string | null + loadingAreas: boolean + areaForm: AreaFormState + onCreateArea: (event: FormEvent) => void + onUpdateAreaForm: (areaForm: AreaFormState) => void +} + +export function AreaPanel({ + areas, + selectedProject, + selectedProjectId, + loadingAreas, + areaForm, + onCreateArea, + onUpdateAreaForm, +}: AreaPanelProps): JSX.Element { + return ( +
+

Area manager

+

{selectedProject ? `Selected project: ${selectedProject.name}` : 'Select a project first'}

+ +
+ onUpdateAreaForm({ ...areaForm, name: event.target.value })} + placeholder="AOI name" + /> + onUpdateAreaForm({ ...areaForm, crs: event.target.value })} + placeholder="EPSG:4326" + /> +