diff --git a/.gitignore b/.gitignore index 55c15e50..65a2790d 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ build/ /storage/models/* /storage/operator-data/* /storage/operator-evidence/* +/storage/release-evidence/* /storage/previews/* /storage/training/* /storage/ultralytics/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 96bfa158..0d9a20f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ # Changelog +## Autonomous Belgium and North Sea RC program (2026-07-17) + +- Expanded the release-candidate geography from Mol/Kempen to all of Belgium + and the Belgian North Sea while retaining the existing areas as golden + regression references. +- Added an executable autonomous RC-0 through RC-11 roadmap and explicit + national/maritime scope freeze. +- Folded fresh-install, upgrade, rollback and runtime proof into RC-5 and + RC-11 instead of creating a separate RC-12 phase. +- Added a read-only release-evidence manifest command with Git, migration, + dependency, configuration checksum and optional live endpoint evidence. +- Replaced the obsolete pre-build status with the current implemented + foundation and active release blockers. +- Added atomic PostgreSQL release backup, read-only checksum verification and + isolated generated-database restore-smoke tooling. +- Split health into process liveness and fail-closed readiness covering + PostgreSQL, PostGIS, Alembic head and writable storage. +- Made system capabilities report real PostGIS and configured local YOLO + state, added request IDs and exception logging, reduced SQL engine logging, + and terminalized impossible orphaned work after all-in-one restarts. + ## Sprint 240 Operational forest, agriculture, nature and soil themes (2026-07-17) - Extended the governed Landgebruik Vlaanderen 2025 raster registry with diff --git a/backend/README.md b/backend/README.md index 789f1b7f..13ee511e 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,6 +1,19 @@ # GeoIntel Backend (Sprint 3 foundation layer) -FastAPI backend for GeoIntel Kempen Foundation Sprints. +FastAPI backend for the GeoIntel Belgium and Belgian North Sea workbench. + +Runtime probes: + +- `GET /health/live`: process liveness, always independent from PostgreSQL. +- `GET /health/ready`: fail-closed database/PostGIS/migration/storage + readiness used by Docker. +- `GET /health`: compatibility alias for readiness. +- `GET /api/v1/system/capabilities`: runtime-derived PostGIS, GIS dependency, + configured YOLO and provider state. + +The all-in-one production runtime enables interrupted job/analysis-run +reconciliation at startup. Local tests and development leave it disabled +unless `GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP=true`. The map-first explorer uses the existing persisted vector selection endpoint. Its bounded GeoJSON preview reports `feature_count`, while `total_feature_count` reports the exact PostGIS intersection count before the 1,000-feature response cap. When an active `area_id` is supplied, vector, temporal, derived-dataset and export paths all use `bbox ∩ Area`. A full-work-area bbox resolves to the exact persisted Area geometry; a boundary-crossing rectangle is clipped to the official boundary. diff --git a/backend/app/api/routes/health.py b/backend/app/api/routes/health.py index 5d4914ae..093f96bd 100644 --- a/backend/app/api/routes/health.py +++ b/backend/app/api/routes/health.py @@ -1,12 +1,23 @@ from __future__ import annotations from importlib import import_module -from sqlalchemy import text -from fastapi import APIRouter +from pathlib import Path +from tempfile import NamedTemporaryFile -from app.schemas.health import HealthResponse, SystemCapabilities -from app.providers.registry import list_provider_capabilities +from alembic.config import Config +from alembic.script import ScriptDirectory +from fastapi import APIRouter, Response, status +from sqlalchemy import text + +from app.core.config import get_settings from app.db.session import get_engine +from app.providers.registry import list_provider_capabilities +from app.schemas.health import ( + HealthResponse, + SystemCapabilities, + SystemCapabilitiesEnvelope, +) +from app.services.model_registry_service import ModelRegistryService router = APIRouter() @@ -19,27 +30,136 @@ def _dependency_enabled(module: str) -> bool: return False -@router.get("/health") -def readiness() -> HealthResponse: - db_status = "ok" +def _expected_migration_heads() -> list[str]: + backend_root = Path(__file__).resolve().parents[3] + config = Config(str(backend_root / "alembic.ini")) + config.set_main_option("script_location", str(backend_root / "alembic")) + return list(ScriptDirectory.from_config(config).get_heads()) + + +def _database_checks() -> dict[str, str]: + checks = { + "database": "degraded", + "postgis": "degraded", + "migration": "degraded", + } try: with get_engine().connect() as connection: connection.execute(text("SELECT 1")) + checks["database"] = "ok" + postgis_version = connection.execute( + text("SELECT PostGIS_Version()") + ).scalar_one() + checks["postgis"] = f"ok:{postgis_version}" + database_head = connection.execute( + text("SELECT version_num FROM alembic_version") + ).scalar_one() + expected_heads = _expected_migration_heads() + if len(expected_heads) == 1 and database_head == expected_heads[0]: + checks["migration"] = f"ok:{database_head}" + else: + checks["migration"] = ( + f"degraded:database={database_head};" + f"expected={','.join(expected_heads) or 'none'}" + ) except Exception: - db_status = "degraded" - return HealthResponse(status="ok", service="geointel-backend", version="0.1.0", database=db_status) + return checks + return checks -@router.get("/api/v1/system/capabilities") -def capabilities() -> dict: +def _storage_check(storage_root: str) -> str: + root = Path(storage_root).expanduser() + try: + root.mkdir(parents=True, exist_ok=True) + with NamedTemporaryFile( + prefix=".geointel-readiness-", + dir=root, + delete=True, + ) as handle: + handle.write(b"ok") + handle.flush() + return "ok" + except OSError: + return "degraded" + + +def _readiness_payload() -> HealthResponse: + settings = get_settings() + checks = _database_checks() + checks["storage"] = _storage_check(settings.storage_root) + ready = all( + value == "ok" or value.startswith("ok:") + for value in checks.values() + ) + return HealthResponse( + status="ok" if ready else "degraded", + service="geointel-backend", + version=settings.app_version, + build_sha=settings.build_sha, + build_time=settings.build_time, + database=checks["database"], + postgis=checks["postgis"], + migration=checks["migration"], + storage=checks["storage"], + checks=checks, + ) + + +@router.get("/health/live", response_model=HealthResponse) +def liveness() -> HealthResponse: + settings = get_settings() + return HealthResponse( + status="ok", + service="geointel-backend", + version=settings.app_version, + build_sha=settings.build_sha, + build_time=settings.build_time, + ) + + +def _readiness_response(response: Response) -> HealthResponse: + payload = _readiness_payload() + if payload.status != "ok": + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return payload + + +@router.get("/health", response_model=HealthResponse) +def readiness(response: Response) -> HealthResponse: + return _readiness_response(response) + + +@router.get("/health/ready", response_model=HealthResponse) +def readiness_explicit(response: Response) -> HealthResponse: + return _readiness_response(response) + + +@router.get( + "/api/v1/system/capabilities", + response_model=SystemCapabilitiesEnvelope, +) +def capabilities() -> SystemCapabilitiesEnvelope: + settings = get_settings() providers = [item.to_dict() for item in list_provider_capabilities()] - return {"data": SystemCapabilities( - postgis=True, - rasterio=_dependency_enabled("rasterio"), - geopandas=_dependency_enabled("geopandas"), - yolo=False, - sam=False, - grb="bounded", - sentinel="planned", - providers=providers, - ).model_dump()} + configured_yolo = ModelRegistryService.get_model_capability( + settings.yolo_model_id, + settings=settings, + ) + yolo_configured = bool(configured_yolo and configured_yolo.configured) + yolo_status = configured_yolo.status if configured_yolo else "not_configured" + postgis_ready = _database_checks()["postgis"].startswith("ok:") + return SystemCapabilitiesEnvelope( + data=SystemCapabilities( + postgis=postgis_ready, + rasterio=_dependency_enabled("rasterio"), + geopandas=_dependency_enabled("geopandas"), + yolo=yolo_configured, + yolo_status=yolo_status, + sam=False, + grb="bounded", + sentinel="planned", + version=settings.app_version, + build_sha=settings.build_sha, + providers=providers, + ) + ) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index c9f8da17..b5c1d6e6 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -12,6 +12,8 @@ class Settings(BaseSettings): app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV") app_version: str = Field(default="0.1.0") + build_sha: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_SHA") + build_time: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_TIME") api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX") database_url: str = Field( default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1", @@ -231,6 +233,11 @@ class Settings(BaseSettings): thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB") redis_url: str | None = Field(default=None, validation_alias="REDIS_URL") log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL") + sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL") + reconcile_interrupted_runs_on_startup: bool = Field( + default=False, + validation_alias="GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP", + ) database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS") yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED") yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR") diff --git a/backend/app/core/logging.py b/backend/app/core/logging.py index dcf74961..6c8a4ccb 100644 --- a/backend/app/core/logging.py +++ b/backend/app/core/logging.py @@ -2,11 +2,13 @@ import logging import sys -def configure_logging(level: str = "INFO") -> None: +def configure_logging(level: str = "INFO", sql_level: str = "WARNING") -> None: logging.basicConfig( level=level, format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", stream=sys.stdout, + force=True, ) - for name in ["uvicorn", "uvicorn.error", "uvicorn.access", "sqlalchemy.engine"]: + for name in ["uvicorn", "uvicorn.error", "uvicorn.access"]: logging.getLogger(name).setLevel(level) + logging.getLogger("sqlalchemy.engine").setLevel(sql_level) diff --git a/backend/app/main.py b/backend/app/main.py index 7a2cdc99..d5902ecb 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,5 +1,9 @@ from __future__ import annotations +import logging +import uuid +from contextlib import asynccontextmanager + from fastapi import FastAPI, HTTPException, Request from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware @@ -9,6 +13,11 @@ from app.api.routes import analysis, areas, assistant, datasets, demo, detection from app.core.config import get_settings from app.core.errors import AppError from app.core.logging import configure_logging +from app.db.session import SessionLocal +from app.services.runtime_reconciliation_service import RuntimeReconciliationService + + +logger = logging.getLogger("geointel") def _to_error_payload( @@ -27,13 +36,33 @@ def _to_error_payload( def create_app() -> FastAPI: settings = get_settings() - configure_logging(settings.log_level) + configure_logging(settings.log_level, settings.sql_log_level) + + @asynccontextmanager + async def lifespan(_: FastAPI): + if settings.reconcile_interrupted_runs_on_startup: + db = SessionLocal() + try: + result = RuntimeReconciliationService.reconcile(db) + logger.info( + "Runtime reconciliation completed: jobs=%s analysis_runs=%s", + result.interrupted_jobs, + result.interrupted_analysis_runs, + ) + except Exception: + db.rollback() + logger.exception("Runtime reconciliation failed") + raise + finally: + db.close() + yield app = FastAPI( - title="GeoIntel Kempen", + title="GeoIntel", version=settings.app_version, docs_url="/docs", redoc_url="/redoc", + lifespan=lifespan, ) app.add_middleware( @@ -60,6 +89,14 @@ def create_app() -> FastAPI: app.include_router(temporal.router, prefix=settings.api_prefix) app.include_router(assistant.router, prefix=settings.api_prefix) + @app.middleware("http") + async def request_identity(request: Request, call_next): + request_id = request.headers.get("x-request-id") or str(uuid.uuid4()) + request.state.request_id = request_id + response = await call_next(request) + response.headers["x-request-id"] = request_id + return response + @app.exception_handler(AppError) async def app_error(request: Request, exc: AppError): # noqa: ARG001 return JSONResponse( @@ -68,7 +105,7 @@ def create_app() -> FastAPI: exc.code, exc.message, exc.details, - request_id=request.headers.get("x-request-id"), + request_id=request.state.request_id, ), ) @@ -88,7 +125,7 @@ def create_app() -> FastAPI: code, message, details, - request_id=request.headers.get("x-request-id"), + request_id=request.state.request_id, ), ) @@ -100,19 +137,25 @@ def create_app() -> FastAPI: "VALIDATION_ERROR", "Validation failed", exc.errors(), - request_id=request.headers.get("x-request-id"), + request_id=request.state.request_id, ), ) @app.exception_handler(Exception) - async def unexpected_error(request: Request, exc: Exception): # noqa: ARG001 + async def unexpected_error(request: Request, exc: Exception): + logger.exception( + "Unhandled request error request_id=%s method=%s path=%s", + request.state.request_id, + request.method, + request.url.path, + ) return JSONResponse( status_code=500, content=_to_error_payload( "INTERNAL_ERROR", "Unexpected server error", {"type": exc.__class__.__name__}, - request_id=request.headers.get("x-request-id"), + request_id=request.state.request_id, ), ) diff --git a/backend/app/schemas/health.py b/backend/app/schemas/health.py index 67924441..0996a142 100644 --- a/backend/app/schemas/health.py +++ b/backend/app/schemas/health.py @@ -23,7 +23,13 @@ class HealthResponse(BaseModel): status: str service: str version: str + build_sha: str | None = None + build_time: str | None = None database: str | None = None + postgis: str | None = None + migration: str | None = None + storage: str | None = None + checks: dict[str, str] = Field(default_factory=dict) class SystemCapabilities(BaseModel): @@ -31,7 +37,14 @@ class SystemCapabilities(BaseModel): rasterio: bool geopandas: bool yolo: bool | str + yolo_status: str sam: bool | str grb: str sentinel: str + version: str + build_sha: str | None = None providers: list[ProviderCapability] = Field(default_factory=list) + + +class SystemCapabilitiesEnvelope(BaseModel): + data: SystemCapabilities diff --git a/backend/app/services/runtime_reconciliation_service.py b/backend/app/services/runtime_reconciliation_service.py new file mode 100644 index 00000000..8658ca1d --- /dev/null +++ b/backend/app/services/runtime_reconciliation_service.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone + +from sqlalchemy.orm import Session + +from app.models import AnalysisRun, Job + + +@dataclass(frozen=True) +class ReconciliationResult: + interrupted_jobs: int + interrupted_analysis_runs: int + + +class RuntimeReconciliationService: + ERROR_MESSAGE = ( + "PROCESS_INTERRUPTED: the GeoIntel process restarted before this work " + "reached a terminal state" + ) + + @staticmethod + def reconcile( + db: Session, + *, + finished_at: datetime | None = None, + ) -> ReconciliationResult: + resolved_finished_at = finished_at or datetime.now(timezone.utc) + interrupted_jobs = ( + db.query(Job) + .filter(Job.status == "running") + .update( + { + Job.status: "failed", + Job.finished_at: resolved_finished_at, + Job.error_message: RuntimeReconciliationService.ERROR_MESSAGE, + }, + synchronize_session=False, + ) + ) + interrupted_analysis_runs = ( + db.query(AnalysisRun) + .filter(AnalysisRun.status == "running") + .update( + { + AnalysisRun.status: "failed", + AnalysisRun.finished_at: resolved_finished_at, + AnalysisRun.error_message: RuntimeReconciliationService.ERROR_MESSAGE, + }, + synchronize_session=False, + ) + ) + db.commit() + return ReconciliationResult( + interrupted_jobs=interrupted_jobs, + interrupted_analysis_runs=interrupted_analysis_runs, + ) diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py index b9932ca0..e6b44e99 100644 --- a/backend/tests/test_docker_runtime_config.py +++ b/backend/tests/test_docker_runtime_config.py @@ -204,9 +204,9 @@ def test_compose_mounts_demo_fixtures_for_backend_runtime() -> None: def test_compose_has_backend_and_frontend_healthchecks() -> None: compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") - assert "http://127.0.0.1:8000/health" in compose + assert "http://127.0.0.1:8000/health/ready" in compose assert "urllib.request.urlopen" in compose - assert "http://127.0.0.1/health" in compose + assert "http://127.0.0.1/health/ready" in compose assert "wget -q -O -" in compose assert "start_period: 30s" in compose assert "start_period: 10s" in compose diff --git a/backend/tests/test_health.py b/backend/tests/test_health.py index d690b31c..adc95cef 100644 --- a/backend/tests/test_health.py +++ b/backend/tests/test_health.py @@ -1,34 +1,100 @@ from __future__ import annotations +from types import SimpleNamespace + from fastapi.testclient import TestClient from app.api.routes import health from app.main import app -def test_health_endpoint_returns_status_payload() -> None: - client = TestClient(app) - response = client.get("/health") +READY_DATABASE = { + "database": "ok", + "postgis": "ok:3.4 USE_GEOS=1 USE_PROJ=1", + "migration": "ok:202607160001", +} + + +def test_liveness_is_independent_from_database(monkeypatch) -> None: + monkeypatch.setattr( + health, + "_database_checks", + lambda: (_ for _ in ()).throw(AssertionError("must not query DB")), + ) + + response = TestClient(app).get("/health/live") assert response.status_code == 200 + assert response.json()["status"] == "ok" + +def test_readiness_returns_ok_only_when_all_checks_pass(monkeypatch) -> None: + monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy()) + monkeypatch.setattr(health, "_storage_check", lambda _: "ok") + + response = TestClient(app).get("/health/ready") + + assert response.status_code == 200 payload = response.json() - assert payload["status"] in {"ok", "degraded"} + assert payload["status"] == "ok" assert payload["service"] == "geointel-backend" - assert payload["version"] == "0.1.0" + assert payload["database"] == "ok" + assert payload["postgis"].startswith("ok:") + assert payload["migration"] == "ok:202607160001" + assert payload["storage"] == "ok" -def test_system_capabilities_reports_gis_dependency_flags(monkeypatch) -> None: +def test_compatibility_health_is_fail_closed(monkeypatch) -> None: + monkeypatch.setattr( + health, + "_database_checks", + lambda: { + "database": "degraded", + "postgis": "degraded", + "migration": "degraded", + }, + ) + monkeypatch.setattr(health, "_storage_check", lambda _: "ok") + + response = TestClient(app).get("/health") + + assert response.status_code == 503 + assert response.json()["status"] == "degraded" + + +def test_system_capabilities_report_runtime_state(monkeypatch) -> None: monkeypatch.setattr( health, "_dependency_enabled", lambda module_name: module_name in {"rasterio", "geopandas"}, ) + monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy()) + monkeypatch.setattr( + health.ModelRegistryService, + "get_model_capability", + lambda *args, **kwargs: SimpleNamespace( + configured=True, + status="configured", + ), + ) - client = TestClient(app) - response = client.get("/api/v1/system/capabilities") + response = TestClient(app).get("/api/v1/system/capabilities") assert response.status_code == 200 - payload = response.json() - assert payload["data"]["rasterio"] is True - assert payload["data"]["geopandas"] is True + payload = response.json()["data"] + assert payload["postgis"] is True + assert payload["rasterio"] is True + assert payload["geopandas"] is True + assert payload["yolo"] is True + assert payload["yolo_status"] == "configured" + assert payload["version"] + + +def test_requests_receive_a_correlation_id() -> None: + client = TestClient(app) + + generated = client.get("/health/live") + retained = client.get("/health/live", headers={"x-request-id": "test-request"}) + + assert generated.headers["x-request-id"] + assert retained.headers["x-request-id"] == "test-request" diff --git a/backend/tests/test_rc_backup_restore_scripts.py b/backend/tests/test_rc_backup_restore_scripts.py new file mode 100644 index 00000000..e0e6549c --- /dev/null +++ b/backend/tests/test_rc_backup_restore_scripts.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" + + +def read(name: str) -> str: + return (SCRIPTS / name).read_text(encoding="utf-8") + + +def test_backup_is_atomic_read_only_and_checksum_bound() -> None: + script = read("backup_release_state.sh") + + assert "pg_dump" in script + assert "-Fc" in script + assert "--no-owner" in script + assert "CHECKSUMS.sha256" in script + assert "database-password" not in script.lower() + assert "mv \"$PARTIAL\" \"$FINAL\"" in script + assert "rm -rf -- \"$PARTIAL\"" in script + assert "DROP DATABASE" not in script + assert "pg_restore --clean" not in script + + +def test_backup_verification_is_read_only() -> None: + script = read("verify_release_backup.sh") + + assert "sha256sum -c CHECKSUMS.sha256" in script + assert "pg_restore --list" in script + assert "createdb" not in script + assert "dropdb" not in script + assert "pg_restore --clean" not in script + + +def test_restore_smoke_is_forced_to_generated_isolated_database() -> None: + script = read("restore_release_backup_smoke.sh") + + assert "--confirm-isolated-restore" in script + assert "geointel_restore_verify_" in script + assert 'if [ "$TARGET_DB" = "$DB_NAME" ]' in script + assert "createdb" in script + assert "dropdb --if-exists" in script + assert "pg_restore \\\n --clean" not in script + assert '"production_database_untouched": True' in script + + +def test_release_safety_scripts_have_valid_bash_syntax() -> None: + for name in ( + "backup_release_state.sh", + "verify_release_backup.sh", + "restore_release_backup_smoke.sh", + ): + result = subprocess.run( + ["bash", "-n", f"scripts/{name}"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, f"{name}: {result.stderr}" + + +def test_readiness_gate_checks_release_safety_scripts() -> None: + readiness = read("run_readiness_check.sh") + + for name in ( + "backup_release_state.sh", + "verify_release_backup.sh", + "restore_release_backup_smoke.sh", + ): + assert f"bash -n scripts/{name}" in readiness diff --git a/backend/tests/test_rc_release_evidence.py b/backend/tests/test_rc_release_evidence.py new file mode 100644 index 00000000..28bc67c9 --- /dev/null +++ b/backend/tests/test_rc_release_evidence.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "capture_release_evidence.py" + + +def load_script(): + spec = importlib.util.spec_from_file_location("capture_release_evidence", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_release_evidence_manifest_is_secret_free_and_read_only(tmp_path: Path) -> None: + module = load_script() + args = module.parse_args( + [ + "--output", + str(tmp_path / "evidence.json"), + "--release-id", + "test-rc", + ] + ) + + manifest = module.build_manifest(args) + + assert manifest["schema_version"] == 1 + assert manifest["release_id"] == "test-rc" + assert manifest["read_only"] is True + assert manifest["scope"] == "Belgium and the Belgian North Sea" + assert "DATABASE_URL" not in json.dumps(manifest).replace( + '"DATABASE_URL": false', + "", + ).replace( + '"DATABASE_URL": true', + "", + ) + assert manifest["storage"] == {"requested": False} + assert manifest["live"] == {"requested": False} + + +def test_release_evidence_cli_writes_single_head_manifest(tmp_path: Path) -> None: + output = tmp_path / "baseline.json" + + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--output", + str(output), + "--release-id", + "test-cli", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + timeout=60, + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["git"]["commit"] + assert payload["migrations"]["single_head"] is True + assert payload["files"]["docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md"]["sha256"] + assert payload["files"]["docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md"]["sha256"] + + +def test_readiness_gate_compiles_release_evidence_command() -> None: + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text( + encoding="utf-8" + ) + + assert "py_compile scripts/capture_release_evidence.py" in readiness diff --git a/backend/tests/test_runtime_reconciliation_service.py b/backend/tests/test_runtime_reconciliation_service.py new file mode 100644 index 00000000..031ff75c --- /dev/null +++ b/backend/tests/test_runtime_reconciliation_service.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from unittest.mock import MagicMock + +from app.models import AnalysisRun, Job +from app.services.runtime_reconciliation_service import RuntimeReconciliationService + + +def test_reconciliation_terminalizes_only_running_work() -> None: + db = MagicMock() + jobs = MagicMock() + runs = MagicMock() + jobs.filter.return_value.update.return_value = 5 + runs.filter.return_value.update.return_value = 2 + db.query.side_effect = [jobs, runs] + finished_at = datetime(2026, 7, 17, 20, 0, tzinfo=timezone.utc) + + result = RuntimeReconciliationService.reconcile( + db, + finished_at=finished_at, + ) + + assert result.interrupted_jobs == 5 + assert result.interrupted_analysis_runs == 2 + jobs.filter.assert_called_once() + runs.filter.assert_called_once() + job_values = jobs.filter.return_value.update.call_args.args[0] + run_values = runs.filter.return_value.update.call_args.args[0] + assert job_values[Job.status] == "failed" + assert job_values[Job.finished_at] == finished_at + assert "PROCESS_INTERRUPTED" in job_values[Job.error_message] + assert run_values[AnalysisRun.status] == "failed" + assert run_values[AnalysisRun.finished_at] == finished_at + assert "PROCESS_INTERRUPTED" in run_values[AnalysisRun.error_message] + db.commit.assert_called_once_with() + + +def test_startup_reconciliation_is_enabled_only_in_all_in_one_runtime() -> None: + start_script = ( + __import__("pathlib").Path(__file__).resolve().parents[2] + / "deploy" + / "unraid" + / "all-in-one-start.sh" + ).read_text(encoding="utf-8") + + assert ( + 'GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true' + in start_script + ) diff --git a/backend/tests/test_sprint31_unraid_template.py b/backend/tests/test_sprint31_unraid_template.py index 296d7d5f..cc05d58f 100644 --- a/backend/tests/test_sprint31_unraid_template.py +++ b/backend/tests/test_sprint31_unraid_template.py @@ -99,7 +99,7 @@ def test_unraid_all_in_one_runtime_starts_embedded_postgis_backend_and_nginx() - assert "migrate_compose_volume_if_needed" in dockerman_script assert "docker rm -f geointel" in dockerman_script assert "proxy_pass http://127.0.0.1:8000/api/" in nginx_config - assert "proxy_pass http://127.0.0.1:8000/health" in nginx_config + assert "proxy_pass http://127.0.0.1:8000/health/ready" in nginx_config assert "location = /geointel-icon.png" in nginx_config assert "frontend/node_modules" in dockerignore assert "storage" in dockerignore diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 399577bd..e4bd079d 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -153,6 +153,6 @@ VOLUME ["/var/lib/postgresql/data", "/app/storage"] EXPOSE 80 HEALTHCHECK --interval=30s --timeout=5s --retries=10 --start-period=60s \ - CMD curl -fsS http://127.0.0.1/health >/dev/null || exit 1 + CMD curl -fsS http://127.0.0.1/health/ready >/dev/null || exit 1 CMD ["/usr/local/bin/geointel-all-in-one-start"] diff --git a/deploy/unraid/README.md b/deploy/unraid/README.md index 574a9fde..410e6f6f 100644 --- a/deploy/unraid/README.md +++ b/deploy/unraid/README.md @@ -134,10 +134,12 @@ If multiple model files are present, add `--model-file /mnt/user/appdata/geointe The helper does not download weights, does not load a model and does not run inference; it only updates the env file for the mounted local model. -Validate: +Validate liveness, dependency readiness and the canonical API: ```bash -curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/health" +curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/health/live" +curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/health/ready" +curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/api/v1/system/capabilities" curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/api/v1/projects" curl -I "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/geointel-icon.svg" curl -I "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/geointel-icon.png" @@ -204,6 +206,12 @@ OLLAMA_MAX_OUTPUT_TOKENS=1200 OLLAMA_CONTEXT_TOKENS=16384 ``` +`/health/live` proves only that FastAPI is serving. `/health/ready` returns +HTTP 503 when PostgreSQL, PostGIS, the migration head or persistent storage is +not ready, and is the container healthcheck. The all-in-one startup also marks +work left in `running` by a previous process as failed with +`PROCESS_INTERRUPTED`; synchronous work cannot survive a container restart. + The model dropdown comes from Ollama `/api/tags`, so changing the installed models requires no frontend rebuild. Verify after deployment with: diff --git a/deploy/unraid/all-in-one-start.sh b/deploy/unraid/all-in-one-start.sh index 3ca5918a..c067c2f9 100644 --- a/deploy/unraid/all-in-one-start.sh +++ b/deploy/unraid/all-in-one-start.sh @@ -11,6 +11,7 @@ export CORS_ORIGINS="${GEOINTEL_CORS_ORIGINS:-${CORS_ORIGINS:-http://localhost:1 export MAX_UPLOAD_MB="${GEOINTEL_MAX_UPLOAD_MB:-${MAX_UPLOAD_MB:-500}}" export YOLO_MODELS_DIR="${YOLO_MODELS_DIR:-/app/models}" export YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-$STORAGE_ROOT/ultralytics}" +export GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP="${GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true}" mkdir -p "$PGDATA" "$STORAGE_ROOT" "$YOLO_CONFIG_DIR" /run/nginx /var/log/nginx chown -R postgres:postgres "$PGDATA" @@ -66,7 +67,7 @@ import urllib.request last_error = None for attempt in range(1, 61): try: - urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=3).read() + urllib.request.urlopen("http://127.0.0.1:8000/health/ready", timeout=3).read() print(f"Backend is ready after attempt {attempt}.") break except Exception as exc: diff --git a/deploy/unraid/nginx-all-in-one.conf b/deploy/unraid/nginx-all-in-one.conf index 44994a91..9675d651 100644 --- a/deploy/unraid/nginx-all-in-one.conf +++ b/deploy/unraid/nginx-all-in-one.conf @@ -46,6 +46,18 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } + location = /health/live { + proxy_pass http://127.0.0.1:8000/health/live; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location = /health/ready { + proxy_pass http://127.0.0.1:8000/health/ready; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + location / { try_files $uri $uri/ /index.html; } diff --git a/docker-compose.yml b/docker-compose.yml index 052dcbf1..c6c4935d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -127,7 +127,7 @@ services: db: condition: service_healthy healthcheck: - test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5).read()\""] + test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/ready', timeout=5).read()\""] interval: 10s timeout: 5s retries: 12 @@ -142,7 +142,7 @@ services: backend: condition: service_healthy healthcheck: - test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1/health | grep -q '\"status\"'"] + test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1/health/ready | grep -q '\"status\":\"ok\"'"] interval: 10s timeout: 5s retries: 12 diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 9e88b0cb..1881f845 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -41,48 +41,68 @@ Any valid GeoJSON geometry object. V1 primarily expects `Polygon` and `MultiPoly ## Health -### GET `/health` +### GET `/health/live` -Returns service status. +Returns process liveness only. It never queries PostgreSQL. ```json { "status": "ok", "service": "geointel-backend", - "version": "0.1.0" + "version": "0.1.0", + "build_sha": null, + "build_time": null +} +``` + +### GET `/health` + +Backward-compatible alias for dependency readiness. + +### GET `/health/ready` + +Returns dependency readiness. Both readiness routes return HTTP 503 when the +database, PostGIS, single migration head or writable storage check is +degraded. Docker uses `/health/ready`. + +```json +{ + "status": "ok", + "service": "geointel-backend", + "version": "0.1.0", + "database": "ok", + "postgis": "ok:3.x", + "migration": "ok:202607160001", + "storage": "ok", + "checks": { + "database": "ok", + "postgis": "ok:3.x", + "migration": "ok:202607160001", + "storage": "ok" + } } ``` ### GET `/api/v1/system/capabilities` -Returns enabled feature flags and tool availability. +Returns enabled feature flags and tool availability in the canonical data +envelope. PostGIS and configured YOLO state are derived at runtime. ```json { - "postgis": true, - "rasterio": true, - "geopandas": true, - "yolo": false, - "sam": false, - "grb": "bounded", - "sentinel": "planned", - "providers": [ - { - "provider_name": "grb", - "display_name": "GRB", - "authority_level": "authoritative", - "supported_layers": ["buildings", "roads", "water", "parcels"], - "supported_geometry_types": ["Polygon", "MultiPolygon", "LineString", "MultiLineString"], - "supported_query_modes": ["bbox", "persisted_area"], - "fetch_signature": "POST /api/v1/projects/{project_id}/datasets/grb/acquire", - "configured": true, - "status": "configured", - "limitation_message": "Alleen expliciet begrensde selecties tot 20 km per zijde worden opgehaald.", - "attribution": "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen", - "license_note": "Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen.", - "not_configured_reason": null - } - ] + "data": { + "postgis": true, + "rasterio": true, + "geopandas": true, + "yolo": true, + "yolo_status": "configured", + "sam": false, + "grb": "bounded", + "sentinel": "planned", + "version": "0.1.0", + "build_sha": null, + "providers": [] + } } ``` diff --git a/docs/BUILD_STATUS.md b/docs/BUILD_STATUS.md index d501264d..f134e4e1 100644 --- a/docs/BUILD_STATUS.md +++ b/docs/BUILD_STATUS.md @@ -1,38 +1,62 @@ # GeoIntel Build Status -Current preparation milestone: M7 Implementation Control Layer. +Updated: 2026-07-17 -## Done +## Current state -- Product blueprint. -- Data specifications. -- Architecture specifications. -- API/database/service documentation. -- Codex build plans and prompts. -- Operational readiness docs. -- Autonomy pack. -- M7 build control, regression traps and self-review layer. +GeoIntel is an implemented map-first GeoAI workbench running as an all-in-one +Unraid container with embedded PostGIS, FastAPI, React/MapLibre, local +Ollama integration and optional local YOLO/PyTorch inference. -## Ready for Codex +The active release program is +`docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md`. -Codex can begin with repository verification and backend foundation using the locked build sequence. +## Product scope -## Must Preserve +- Target: all Belgian land and the separately labelled Belgian territorial + sea, EEZ and continental shelf. +- Existing deep regression references: Mol and the Kempen transport region. +- Required new golden areas: Wallonia, Brussels, a cross-region area, coast + and Belgian North Sea. +- Architecture: federated authoritative providers with one coverage matrix; + no assumption that GRB, PICC, UrbIS and NGI are semantically identical. -- GeoIntel is a GeoAI Workbench for the Kempen. -- GRB-first reference strategy. -- FastAPI + React + PostGIS. -- API-driven frontend. -- CRS-aware geospatial processing. -- Fixture mode must be clearly labeled. +## Implemented foundation -## Known Limitations Before Code Build +- Projects, Areas, Datasets, versions, PostGIS vector features and raster + artifacts. +- Map selection, semantic metrics, historical comparison and exports. +- Governed Flemish sources for buildings, context, orthophotos, terrain, + flooding, soil, nature, agriculture, population and policy rasters. +- Detection persistence, local configured YOLO/PyTorch path, QA/QC and review + evidence. +- Segmentation persistence foundation without fake production inference. +- Local Ollama assistant grounded in persisted evidence. +- Single-container Unraid deployment on port 1202. -- Real GRB WFS integration still needs implementation. -- Real YOLO/SAM inference should follow fixture boundary first. -- Sentinel and LiDAR remain post-foundation roadmap items. -- No production authentication in V1. +## Release blockers -## Next Recommended Codex Pass +- Current production backup and isolated restore proof. +- Fail-closed readiness and truthful runtime capability reporting. +- Stale job/run reconciliation and correlated exception logging. +- Temporal compatibility guard for detection QA. +- Production secret/configuration parity, immutable images and rollback. +- Full CI, reproducible dependencies and supply-chain evidence. +- Critical response-model typing and real frontend/browser E2E coverage. +- National and maritime scope/providers/golden areas. -Run `prompts/codex/PASS_00_REPO_AUDIT.md`, then implement backend foundation according to `docs/12-build-control/BUILD_SEQUENCE_LOCK.md`. +## Latest evidence + +Run: + +```bash +python scripts/capture_release_evidence.py \ + --output storage/release-evidence/rc-current/baseline.json \ + --release-id rc-belgium-north-sea \ + --live-base-url http://192.168.10.150:1202 +``` + +The evidence manifest is runtime output and is deliberately ignored by Git. +The first captured baseline reported one Alembic head (`202607160001`) and +reachable live health/capability routes. Health truthfulness is an open RC-2 +blocker until readiness becomes fail-closed. diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 52219833..8b0b4344 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,42 @@ +## Autonomous RC program for Belgium and the Belgian North Sea (2026-07-17) + +- Froze the RC geography as all Belgian land plus the separately labelled + territorial sea, EEZ and continental shelf. +- Retained Mol and the Kempen as validated regression areas instead of the + final product boundary. +- Added `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md`. +- Added `docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md` with autonomous stop rules, + exact phase gates, evidence requirements and RC-0 through RC-11. +- Removed the need for a separate RC-12 phase. Fresh-install, upgrade, + rollback and runtime evidence remain mandatory inside RC-5 and RC-11. +- Verified the official source families for NGI/IGN, Statbel, SPW Wallonia, + UrbIS Brussels, federal marine planning, RBINS/BMDC and MDK before freezing + the coverage architecture. +- Started RC-0 implementation; the release-evidence command is the first code + gate. +- Implemented and tested `scripts/capture_release_evidence.py`. The first + local/live baseline recorded commit `9513f86`, dirty-state evidence, one + Alembic head `202607160001`, dependency/configuration checksums and reachable + Tower health/capability routes without storing secret values. +- Replaced the obsolete pre-implementation `docs/BUILD_STATUS.md` with the + current RC state and moved runtime release evidence outside Git. +- RC-0 is complete. RC-1 backup/restore safety is active. +- Added atomic custom-format PostgreSQL backup, read-only verification and + generated-database restore-smoke scripts. Their focused safety/syntax tests + pass; Tower backup and isolated restore execution remain the RC-1 live gate. +- Implemented RC-2 truthful runtime state: independent liveness, fail-closed + DB/PostGIS/Alembic/storage readiness, runtime-derived YOLO capability, + request correlation IDs, exception trace logging, SQL logging at WARNING + and all-in-one startup reconciliation for orphaned `running` jobs/runs. +- Docker, Nginx, API contracts and health documentation now use + `/health/ready`; `/health` remains a compatibility readiness alias. +- Validation passed: targeted Ruff, backend compile, 963 backend tests, + frontend typecheck/build, 122-route contract audit, one Alembic head, + complete offline migration SQL and the full readiness gate. +- Local Docker CLI is unavailable on the Windows Codex host. Docker config, + image build, live health truthfulness, backup and restore are therefore + scheduled on the Docker-enabled Tower after push. + ## Sprint 229 - Governed ALZ definitive release promotion (2026-07-17) Implemented: diff --git a/docs/HEALTHCHECK_CONTRACTS.md b/docs/HEALTHCHECK_CONTRACTS.md index a9baa5d6..da33574d 100644 --- a/docs/HEALTHCHECK_CONTRACTS.md +++ b/docs/HEALTHCHECK_CONTRACTS.md @@ -5,8 +5,15 @@ Elke runtime-component moet een eenvoudige en machineleesbare healthcheck hebben ## Backend -### Endpoint -`GET /health` +### Liveness + +`GET /health/live` returns HTTP 200 while the FastAPI process can serve a +request. It has no database or storage dependency. + +### Readiness + +`GET /health/ready` is the Docker healthcheck. `GET /health` is the +backward-compatible alias. ### Response 200 ```json @@ -15,8 +22,15 @@ Elke runtime-component moet een eenvoudige en machineleesbare healthcheck hebben "service": "geointel-backend", "version": "0.1.0", "database": "ok", - "redis": "ok", - "storage": "ok" + "postgis": "ok:3.x", + "migration": "ok:202607160001", + "storage": "ok", + "checks": { + "database": "ok", + "postgis": "ok:3.x", + "migration": "ok:202607160001", + "storage": "ok" + } } ``` @@ -25,10 +39,10 @@ Elke runtime-component moet een eenvoudige en machineleesbare healthcheck hebben { "status": "degraded", "service": "geointel-backend", - "database": "unavailable", - "redis": "ok", - "storage": "ok", - "errors": ["database connection failed"] + "database": "degraded", + "postgis": "degraded", + "migration": "degraded", + "storage": "ok" } ``` @@ -51,7 +65,8 @@ Codex mag kiezen tussen een internal endpoint of CLI-command, maar het contract Frontend moet een `/status` of settings/statuspanel hebben dat toont: backend bereikbaar, API version, auth status, feature flags en laatste healthchecktijd. ## Storage -Storage healthcheck controleert of upload, originals, derived en exports writable zijn. +De readinesscheck maakt en verwijdert een tijdelijk bestand in `STORAGE_ROOT`. ## Database -Database healthcheck controleert connectie, PostGIS extension en migrationstatus. +De readinesscheck controleert `SELECT 1`, `PostGIS_Version()` en dat de +databaseversie exact overeenkomt met de enige Alembic-head in de image. diff --git a/docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md b/docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md new file mode 100644 index 00000000..3cd80e78 --- /dev/null +++ b/docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md @@ -0,0 +1,471 @@ +# Autonomous RC Roadmap: Belgium and the Belgian North Sea + +## Mission + +Turn the current GeoIntel implementation into a reproducible, demonstrable and +release-candidate-quality map-first workbench for all of Belgium and the +Belgian North Sea. + +This file is an executable runbook for Codex. It is both the implementation +order and the release evidence index. Work continues autonomously between +phases. A separate RC-12 phase is deliberately omitted. + +## Sources of truth + +Read in this order before changing behavior: + +1. `AGENTS.md` +2. `docs/CODEX_BOOTSTRAP_PROMPT.md` +3. `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md` +4. `docs/governance/GEOINTEL_CONSTITUTION.md` +5. `docs/governance/ARCHITECTURE_INVARIANTS.md` +6. `docs/governance/FORBIDDEN_DECISIONS.md` +7. `docs/API_CONTRACTS.md` +8. `docs/DATABASE_IMPLEMENTATION_PLAN.md` +9. `docs/DATA_SPECIFICATION.md` +10. `docs/DATA_SOURCES.md` +11. `docs/STORAGE_ARCHITECTURE.md` +12. `docs/DEFINITION_OF_DONE.md` + +## Autonomous execution contract + +Codex shall: + +- execute phases in order and continue without requesting routine approval; +- use existing service, dataset, vector-feature, raster and job boundaries; +- keep every provider bounded, fail-closed and provenance-complete; +- add or update tests with every behavior change; +- update `docs/CODEX_EXECUTION_LOG.md`, `docs/TODO.md` and `CHANGELOG.md` + after each completed phase; +- run the phase gate and record exact output under + `storage/release-evidence//`; +- commit and push only after the relevant local gate is green; +- deploy only a commit that passed the local gate; +- run browser and live API acceptance after deployment; +- retain Mol/Kempen as regression references while adding Wallonia, Brussels, + cross-region, coast and North Sea golden areas. + +Codex stops only when: + +- an operation would destructively mutate or delete unbacked production data; +- an external licence, credential or authority decision cannot be derived from + public source metadata; +- an official endpoint fails security validation and no authoritative + alternative exists; +- the same mandatory gate fails three times for the same external reason; or +- continuing would require fabricating data, metrics, provenance or model + output. + +For a stop, write the blocker, evidence and exact resume command to the +execution log. Continue with independent work when possible. + +## Release priorities + +### P0 - release blockers + +- truthful health/readiness and capability reporting; +- current backup plus isolated restore proof; +- stale job/run reconciliation and useful logs; +- temporally correct detection QA; +- production secrets, upload limits, configuration parity and rollback; +- full CI and reproducible dependencies; +- fresh install and upgrade proof. + +### P1 - national product completeness + +- national and maritime scope model; +- governed source coverage matrix; +- Wallonia, Brussels and North Sea adapters; +- nationwide bounded selection and mixed-zone result handling; +- historical compatibility matrix; +- golden areas outside Flanders. + +### P2 - quality and maintainability + +- typed response models on critical routes; +- frontend unit and browser E2E coverage; +- loading/accessibility/performance hardening; +- retention and operator documentation; +- remaining bounded modularization where protected by tests. + +## Global gates + +Run after every behavior-changing phase: + +```bash +python -m compileall backend/app +cd backend && python -m pytest +cd frontend && npm run typecheck +cd frontend && npm run build +bash scripts/run_readiness_check.sh +cd backend && python -m alembic heads +cd backend && python -m alembic upgrade head --sql +bash -n scripts/live_migration_smoke.sh +docker compose config +``` + +When Tower is reachable: + +```powershell +.\scripts\deploy_tower.ps1 +``` + +Then verify: + +```bash +curl -fsS http://192.168.10.150:1202/health/live +curl -fsS http://192.168.10.150:1202/health/ready +curl -fsS http://192.168.10.150:1202/api/v1/system/capabilities +``` + +The in-app browser acceptance must cover desktop, widescreen and a narrow +viewport. It must inspect console errors, failed requests and the complete +map-select-analyse-export flow. + +## Official source baseline + +The roadmap starts from these verified official families. Machine endpoints, +editions and licences must still pass source-specific probes before activation. + +| Scope | Candidate | Role | +| --- | --- | --- | +| Belgium | NGI/IGN AdminVector and Top10Vector | authoritative boundary and common topographic baseline | +| Belgium | Statbel statistical sectors/population | authoritative statistical geometry and time series | +| Flanders | existing Digitaal Vlaanderen/VMM/DOV/INBO/ALZ adapters | detailed authoritative regional layers | +| Wallonia | SPW Géoportail, PICC, orthophoto, MNT and thematic services | detailed authoritative regional layers | +| Brussels | Paradigm UrbIS and regional environmental geodata | detailed authoritative urban layers | +| North Sea | FPS Marine Environment marine spatial plan | legal/use zones | +| North Sea | RBINS/BMDC and Marine Atlas | marine environment and observations | +| North Sea | MDK/Flemish Hydrography | bathymetry and nautical source candidate | +| North Sea | EMODnet | contextual fallback only when explicitly labelled | + +## Phase status + +| Phase | State | Purpose | +| --- | --- | --- | +| RC-0 | complete | freeze scope and produce release evidence baseline | +| RC-1 | in progress | backup, restore and data safety | +| RC-2 | verification pending | health, capabilities and stale-runtime correctness | +| RC-3 | pending | temporal detection/QA correctness and observability | +| RC-4 | pending | national/maritime scope and provider coverage contracts | +| RC-5 | pending | deployment, secrets, configuration, fresh install and rollback | +| RC-6 | pending | complete CI, dependency and supply-chain gates | +| RC-7 | pending | critical API envelope typing and contract validation | +| RC-8 | pending | frontend and browser E2E release journeys | +| RC-9 | pending | loading, accessibility and performance hardening | +| RC-10 | pending | retention, cleanup and national data operations | +| RC-11 | pending | final package, upgrade proof, release tag and handoff | + +## RC-0 - Scope freeze and evidence baseline + +**State: complete.** The read-only baseline command and focused tests pass. +Runtime evidence is stored outside Git under `storage/release-evidence/`. + +### Work + +- Freeze Belgium plus Belgian North Sea scope. +- Record current commit, migration head, dependency versions, container image, + database/storage sizes and readiness results. +- Create a release-evidence manifest command. +- Replace stale product-status wording that still treats Kempen as the product + boundary. +- Add a coverage matrix schema with `operational`, `partial`, + `not_configured`, `unsupported`. + +### Likely files + +- `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md` +- `docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md` +- `docs/BUILD_STATUS.md` +- `docs/DATA_SOURCES.md` +- `scripts/capture_release_evidence.py` +- `backend/tests/test_rc_release_evidence.py` + +### Evidence and exit + +- scope documents contain Belgium, all three regions and maritime zones; +- evidence command is read-only and deterministic; +- manifest contains commit, dirty state, Alembic head and gate status; +- compile and focused tests pass. + +## RC-1 - Backup, restore and data safety + +### Work + +- Add a safe production backup command for PostgreSQL custom-format dumps. +- Capture schema head, PostGIS version, table counts, model checksums and a + storage checksum inventory without duplicating all large artifacts. +- Reject default/empty production database passwords. +- Add backup verification with `pg_restore --list` and checksums. +- Add an isolated restore drill using a temporary database/container; never + restore over production. +- Document retention, encryption boundary and cleanup. + +### Likely files + +- `scripts/backup_release_state.sh` +- `scripts/verify_release_backup.sh` +- `scripts/restore_release_backup_smoke.sh` +- `deploy/unraid/README.md` +- `docs/ROLLBACK_AND_RECOVERY.md` +- `backend/tests/test_rc_backup_restore_scripts.py` + +### Exit + +- current Tower database has a dated, checksum-verified backup; +- isolated restore reaches the recorded Alembic head; +- representative project, dataset, geometry and QA counts reconcile; +- no production application data was mutated by the drill. + +## RC-2 - Truthful runtime state + +**State: implementation complete, live verification pending.** Local compile, +963 backend tests, frontend typecheck/build and the full readiness gate pass. + +### Work + +- Split process liveness from dependency readiness. +- Make readiness return HTTP 503 when DB, PostGIS, migration head or writable + storage is unavailable. +- Derive version/build identity and AI capability from runtime configuration. +- Make Docker use the readiness endpoint. +- Reconcile orphaned `running` jobs and analysis runs on process startup. +- Add request IDs and exception logging without exposing secrets or full SQL. + +### Likely files + +- `backend/app/api/routes/health.py` +- `backend/app/schemas/health.py` +- `backend/app/core/config.py` +- `backend/app/core/logging.py` +- `backend/app/main.py` +- `backend/app/services/runtime_reconciliation_service.py` +- Docker/nginx files and health tests + +### Exit + +- liveness remains 200 while the process runs; +- readiness is fail-closed; +- live capabilities match the active local YOLO state; +- restart converts impossible orphaned work to a terminal failed state with + `PROCESS_INTERRUPTED`; +- Docker reports unhealthy for a broken database. + +## RC-3 - Temporal QA correctness and observability + +### Work + +- Stop Detection Lab from silently selecting the first raster. +- Require explicit raster/model choice. +- Prevent current reference QA against historical imagery unless a compatible + reference edition/coverage is selected. +- Persist source observation period and QA compatibility decision. +- Add structured request/job/run logs and operator diagnostics. +- Add a stale-runtime report and safe reconciliation command. + +### Exit + +- no detection run starts with an implicit old raster; +- incompatible temporal QA fails with a clear code; +- compatible current orthophoto/GRB path remains green; +- logs correlate request, job, analysis run and quality check. + +## RC-4 - Belgium and North Sea coverage foundation + +### Work + +1. Persist NGI authoritative Belgium land and administrative scopes. +2. Persist territorial sea, EEZ and continental shelf as distinct scope + layers. +3. Implement the coverage matrix and source authority resolver. +4. Add bounded provider contracts for: + - NGI/Statbel national baseline; + - SPW/PICC/orthophoto/relief for Wallonia; + - UrbIS/environmental context for Brussels; + - federal marine zones and BMDC/MDK marine sources. +5. Normalize the public theme vocabulary without discarding provider-native + semantics. +6. Add Wallonia, Brussels, cross-region, coast and North Sea golden areas. +7. Make mixed-zone selection split into bounded provider requests and merge + only semantically compatible metrics. + +### Required rules + +- all persistence flows through DatasetService and geospatial services; +- no provider writes directly to feature tables; +- no country-wide startup import; +- no TLS bypass; +- no implicit vertical datum conversion; +- unsupported metrics remain unavailable. + +### Likely files + +- provider registry/base and new regional provider modules; +- geographic scope and source coverage services; +- dataset/source schemas; +- map source portfolio and scope UI; +- source, API, data and storage documentation; +- source-specific tests and fixtures. + +### Exit + +- a user can draw within every golden area and receive an honest coverage + matrix; +- at least the common national baseline is operational on land; +- regional detailed layers are operational or explicitly partial; +- maritime legal zones are operational; +- bathymetry remains gated until a source passes strict acquisition evidence. + +## RC-5 - Production deployment and rollback + +### Work + +- align backend, nginx and proxy upload/time limits; +- make every runtime setting intentionally configured, internal or documented; +- reject production default secrets; +- add image labels with version, commit and build time; +- deploy immutable tags in addition to `latest`; +- preserve previous image for one-command rollback; +- prove fresh install against empty volumes; +- prove upgrade from a copied current database/storage state; +- prove rollback without schema or data loss. + +### Exit + +- Unraid edit fields cover all operator-owned settings; +- fresh install, upgrade and rollback commands are documented and executed; +- deployed UI exposes build identity; +- previous image remains addressable; +- no mutable-only release is accepted. + +## RC-6 - CI and supply-chain gates + +### Work + +- make CI run backend compile/tests, frontend typecheck/build, readiness, + Alembic single-head/offline SQL and Docker config/build; +- add Python and npm dependency vulnerability checks; +- add container scan and SBOM; +- introduce reproducible Python resolution/lock output; +- preserve optional AI dependency separation; +- publish gate artefacts. + +### Exit + +- a broken compile, test, build, migration or contract blocks CI; +- dependency and container findings have severity policy; +- a clean checkout can reproduce dependency versions and image metadata. + +## RC-7 - Critical API contract hardening + +### Work + +- replace `response_model=dict` first on health, projects, areas, datasets, + jobs, analysis runs, QA, exports and map-critical endpoints; +- retain canonical `{data: ...}` success and canonical error envelopes; +- add OpenAPI response validation tests; +- document intentionally streaming/download responses separately. + +### Exit + +- all critical map-first routes have concrete response models; +- actual payloads pass FastAPI validation; +- API contract audit and frontend typecheck pass. + +## RC-8 - Release journey automation + +### Work + +- add frontend unit tests for selection, coverage, loading and temporal guards; +- add browser E2E for: + - open Belgium map; + - select each golden area; + - load a theme; + - inspect metric/provenance; + - compare compatible history; + - run a bounded configured detection path; + - export; + - ask Ollama with persisted context; +- include no-data, partial coverage, provider failure and unsupported metric + journeys. + +### Exit + +- E2E runs against a real backend/PostGIS fixture; +- no workflow depends only on source-text assertion tests; +- console and failed-request audits are clean. + +## RC-9 - UX, accessibility and performance + +### Work + +- replace initial false `missing` source states with explicit loading; +- provide accessible names, labels, keyboard focus and status announcements; +- enforce desktop, widescreen and narrow layouts; +- record API and map-selection budgets for golden areas; +- virtualize or page large panels; +- split large modules only where tests protect behavior. + +### Exit + +- no core control lacks an accessible name; +- loading, empty, partial, error and ready states are distinct; +- map selection remains usable at target viewports; +- no release journey exceeds its documented budget without a visible warning. + +## RC-10 - Data operations and retention + +### Work + +- define lifecycle for raw, normalized, derived, export, AI and evidence + artifacts; +- add dry-run-first cleanup for stale technical projects, failed work and + superseded caches; +- retain immutable official source editions and release evidence; +- report disk pressure before acquisition; +- schedule nothing implicitly; provide explicit operator commands; +- add source freshness reports for national/regional/maritime families. + +### Exit + +- cleanup cannot remove release evidence or current authoritative editions; +- every destructive action requires explicit confirmation and a current + backup; +- storage growth and retained provenance are reportable. + +## RC-11 - Final release package + +### Work + +- rerun fresh-install and upgrade proof using the release image; +- rerun backup/isolated restore and rollback proof; +- execute every golden area and core browser journey; +- generate coverage, security, dependency, migration, performance and known + limitation reports; +- update operator/end-user documentation; +- remove stale status documents from active navigation without deleting + historical evidence; +- assign semantic version, create signed/checksummed release manifest, tag, + push and deploy the immutable image. + +### Final acceptance + +- all P0 findings are closed; +- every mandatory global gate is green; +- backup and isolated restore are proven; +- one Alembic head applies to empty and upgraded PostGIS; +- Belgium and maritime golden areas pass; +- no unsupported metric or unavailable source appears as successful; +- live browser acceptance passes at port 1202; +- remaining P1/P2 limitations are explicit, bounded and non-deceptive. + +## Next command + +After creating this roadmap, start RC-0 immediately: + +```bash +python scripts/capture_release_evidence.py --output storage/release-evidence/rc-current/baseline.json +``` + +If the command does not exist yet, implementing it and its tests is the first +code task. diff --git a/docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md b/docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md new file mode 100644 index 00000000..cf059e1d --- /dev/null +++ b/docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md @@ -0,0 +1,148 @@ +# RC Scope Freeze: Belgium and the Belgian North Sea + +## Status and precedence + +This document freezes the geographic and release scope for the GeoIntel +release-candidate program. It supersedes the geographic limitation in +`docs/V1_SCOPE_FREEZE.md`. The existing V1 domain, persistence, provenance, +GIS-correctness and no-fake-data rules remain in force. + +Mol and the Kempen remain golden regression areas. They are no longer the +product boundary. + +## Product identity + +GeoIntel RC is a map-first geospatial analysis workbench for: + +- the complete land territory of Belgium; +- the Belgian territorial sea; +- the Belgian exclusive economic zone and continental shelf, labelled + according to their legal meaning; +- bounded user-drawn areas that may cross municipal, provincial, regional or + land/sea boundaries. + +The primary flow remains: + +1. open the map; +2. choose one or more understandable themes; +3. draw or select an area; +4. load only the authoritative or explicitly contextual data available for + that area; +5. calculate source-appropriate metrics; +6. inspect provenance, limitations and time; +7. compare compatible historical editions; +8. export the result or ask the local assistant about persisted evidence. + +## Geographic federation + +Belgian coverage is a federation of governed adapters. GeoIntel must never +pretend that regional datasets have identical semantics merely because they +can be displayed together. + +| Zone | Baseline authority | Detailed authoritative families | +| --- | --- | --- | +| Belgium-wide | NGI/IGN, Statbel, federal geo.be | AdminVector, Top10Vector/CartoWeb, statistical sectors and population | +| Flanders | Digitaal Vlaanderen and competent Flemish agencies | GRB, orthophotos, DHMV, VMM/VHA, DOV, INBO, ALZ | +| Wallonia | Service public de Wallonie | PICC, SPW orthophotos, relief, hydrography, land cover and thematic catalogues | +| Brussels | Brussels Capital Region/Paradigm and competent administrations | UrbIS, BruGIS and Brussels open geodata | +| Belgian North Sea | federal Marine Environment, RBINS/BMDC, MDK and legally competent publishers | maritime plan and zones, marine environment, hydrography and validated bathymetry | + +NGI Top10Vector is the preferred common topographic baseline candidate when +its machine access, licence, edition and bounded retrieval contract have been +validated. Regional large-scale products remain preferred where their +authority and semantic detail are higher. + +## Required RC coverage + +The RC must provide: + +- a persisted authoritative Belgium land boundary; +- persisted and legally labelled maritime scope boundaries; +- area selection anywhere inside the combined Belgium/maritime scope; +- a coverage matrix that reports every theme as `operational`, `partial`, + `not_configured` or `unsupported` for the selected zone; +- source, authority, observation time, publication time, CRS, licence, + attribution, coverage and limitation metadata for every result; +- bounded acquisition and analysis; no browser-side source fetch and no + unbounded country-wide request to a public service; +- compatible historical comparison only where editions, units, coverage and + methods are comparable; +- a stable national theme vocabulary while preserving provider-native fields + and semantics. + +The minimum common theme vocabulary is: + +- administrative context; +- buildings and built environment; +- roads and mobility context; +- surface water and hydrography; +- land cover and land use; +- vegetation, forest and nature; +- population and statistical context; +- elevation and terrain; +- orthophoto/aerial imagery; +- flood or climate context where authoritative data exists; +- maritime planning, marine environment and bathymetry for the Belgian North + Sea where authoritative data exists. + +Not every theme must have the same resolution or historical depth in every +zone. Absence or partial coverage must be visible and must not be replaced by +fabricated values. + +## Golden release areas + +National support is validated through bounded golden areas, not one monolithic +import: + +- Mol municipality: existing deep land and AI regression reference; +- Kempen transport region: existing cross-municipality and partition test; +- one Walloon urban/rural boundary-crossing area; +- one Brussels urban area; +- one cross-region or language-boundary area; +- one coastal land/sea area; +- one Belgian North Sea area containing at least two legal/use zones. + +Exact golden area identifiers and checksums must be stored in release +evidence. A source is not nationally operational merely because the Mol test +passes. + +## Maritime and bathymetry rules + +- Territorial sea, EEZ and continental shelf are separate legal scopes. +- TAW, LAT, mDNG, NAP and product-specific vertical references remain + separate dimensions. +- No value is transformed between vertical datums without an authoritative, + tested transformation and an uncertainty statement. +- Bathymetry is bed elevation or depth relative to a stated reference plane. + It is not water volume. +- Water volume requires compatible bed elevation, water-surface elevation, + time, coverage and uncertainty contracts. +- WMTS/WMS imagery may provide visual context but is not an analytical raster + unless pixel values and georeferencing are governed and validated. + +## Explicit exclusions from this RC + +- automatic whole-country mirroring of every source; +- bypassing TLS, licence or source-integrity failures; +- claiming semantic parity between GRB, PICC, UrbIS and NGI products; +- hidden external downloads at application startup; +- browser-direct provider access; +- production multi-user authentication; +- real-time surveillance or operational maritime navigation; +- unsupported water-volume estimates; +- a separate RC-12 soak phase. + +Fresh-install, upgrade, rollback and runtime-stability evidence are mandatory +but are incorporated into RC-5 and RC-11. + +## Scope change control + +During the RC program, a scope change is allowed only when it: + +1. fixes a release blocker; +2. adds missing coverage required by this document; +3. corrects source authority, provenance, CRS, time or metric semantics; or +4. improves verification without adding unrelated product capability. + +Every accepted change must update the autonomous roadmap, execution log and +relevant source/API documentation. diff --git a/docs/ROLLBACK_AND_RECOVERY.md b/docs/ROLLBACK_AND_RECOVERY.md index 060737bf..4456f27e 100644 --- a/docs/ROLLBACK_AND_RECOVERY.md +++ b/docs/ROLLBACK_AND_RECOVERY.md @@ -11,3 +11,24 @@ Bestanden worden niet stilzwijgend verwijderd. Gebruik soft-delete waar mogelijk ## AI/model rollback Elke analysis run bewaart model id, version, parameters, confidence threshold en pipeline version. +# RC backup and isolated restore gate + +Before any release migration, credential rotation or image promotion: + +1. run `scripts/backup_release_state.sh` against the running all-in-one + container; +2. run `scripts/verify_release_backup.sh` read-only; +3. run `scripts/restore_release_backup_smoke.sh` with + `--confirm-isolated-restore`; +4. retain the backup directory and the separate restore-smoke JSON with the + release evidence. + +The restore smoke may only create databases whose name starts with +`geointel_restore_verify_`. It refuses the production database name, does not +use `pg_restore --clean` and drops the temporary database unless an operator +explicitly asks to retain it. + +Storage and model files are inventoried rather than copied into the database +dump. Release backups must therefore be paired with the persistent storage +volume backup policy. Use `--inventory-mode sha256` for final release +evidence. diff --git a/docs/TODO.md b/docs/TODO.md index 151dc450..8986db82 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1,5 +1,35 @@ # GeoIntel TODO +## Actief RC-programma: Belgie en de Belgische Noordzee + +`docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md` is vanaf 2026-07-17 het enige actieve +releasebord. Mol en de Kempen blijven regressiereferenties, maar de +productscope is nu heel Belgie plus de juridisch correct gelabelde Belgische +maritieme zones. + +- [x] RC-0: Belgische en maritieme scopefreeze vastleggen. +- [x] RC-0: autonome roadmap zonder afzonderlijke RC-12 vastleggen. +- [x] RC-0: deterministisch release-evidence manifest implementeren en draaien. +- [ ] RC-1: actuele productiebackup, verificatie en geisoleerde restore bewijzen. +- [ ] RC-2: liveness/readiness/capabilities fail-closed en waarheidsgetrouw maken. +- [ ] RC-2: verweesde jobs en analysis runs na een procesherstart verzoenen. +- [ ] RC-3: expliciete rasterkeuze en temporeel compatibele detectie-QA afdwingen. +- [ ] RC-4: nationale basisdekking, Wallonie, Brussel en Belgische Noordzee via + beheerde providers en golden areas operationaliseren. +- [ ] RC-5: secrets/configuratie/uploadlimieten/immutable deploy en rollback + bewijzen. +- [ ] RC-6: volledige CI, dependency-audit, containerscan en SBOM toevoegen. +- [ ] RC-7: kritieke API-routes concrete responsemodellen geven. +- [ ] RC-8: echte frontend- en browser-E2E-releaseflows toevoegen. +- [ ] RC-9: loading, toegankelijkheid, widescreen/mobile en performance afronden. +- [ ] RC-10: dataretentie, diskdruk en veilige cleanup operationaliseren. +- [ ] RC-11: fresh install, upgrade, rollback, releasepakket, tag en live + acceptatie afronden. + +Een aparte RC-12/soakfase wordt niet uitgevoerd. De relevante fresh-install-, +upgrade-, rollback- en runtimebewijzen zijn verplicht opgenomen in RC-5 en +RC-11. + ## Actuele V1-productstatus Dit is het enige actuele afwerkingsbord. De lange sprint- en diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 02996a8a..c2238d7c 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -36,6 +36,18 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } + location = /health/live { + proxy_pass http://backend:8000/health/live; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location = /health/ready { + proxy_pass http://backend:8000/health/ready; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + location / { try_files $uri $uri/ /index.html; } diff --git a/scripts/README.md b/scripts/README.md index b61d75d6..d62a569d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1938,3 +1938,41 @@ docker exec geointel python /app/scripts/probe_mdk_bathymetry.py This performs only `GetCapabilities`, keeps strict TLS verification enabled and returns exit code `2` for an honest non-ready source. +# Release backup and restore proof + +The release-candidate safety path is host-operated against the running +all-in-one container: + +```bash +bash scripts/backup_release_state.sh \ + --container geointel \ + --output-root /mnt/user/appdata/geointel/backups \ + --storage-path /mnt/user/appdata/geointel/storage \ + --models-path /mnt/user/appdata/geointel/models \ + --inventory-mode sha256 +``` + +The backup is written atomically and contains a PostgreSQL custom-format dump, +archive listing, Alembic/PostGIS metadata, critical table counts, optional +storage/model inventories and SHA-256 checksums. An empty or known-default +database password leaves the release gate failed. For an emergency backup +before rotating that password, add `--allow-insecure-password`; the manifest +still records the insecure state. + +Verify without changing any database: + +```bash +bash scripts/verify_release_backup.sh \ + --backup-dir /mnt/user/appdata/geointel/backups/ +``` + +Prove restoration only in a generated temporary database: + +```bash +bash scripts/restore_release_backup_smoke.sh \ + --backup-dir /mnt/user/appdata/geointel/backups/ \ + --confirm-isolated-restore +``` + +The restore smoke rejects the production database name, compares PostGIS, +Alembic and retained table counts, and removes its temporary database. diff --git a/scripts/audit_api_contracts.py b/scripts/audit_api_contracts.py index 266c3b5a..4dc65e60 100644 --- a/scripts/audit_api_contracts.py +++ b/scripts/audit_api_contracts.py @@ -11,6 +11,8 @@ DOCS = ROOT / "docs" / "API_CONTRACTS.md" # docs/API_CONTRACTS.md ALLOWED_NON_ENVELOPE_ENDPOINTS = { ("GET", "/health"), + ("GET", "/health/live"), + ("GET", "/health/ready"), ("GET", "/api/v1/exports/{export_id}/download"), ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/image"), } diff --git a/scripts/backup_release_state.sh b/scripts/backup_release_state.sh new file mode 100644 index 00000000..c536c66e --- /dev/null +++ b/scripts/backup_release_state.sh @@ -0,0 +1,217 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONTAINER="geointel" +OUTPUT_ROOT="backups" +RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)" +STORAGE_PATH="" +MODELS_PATH="" +INVENTORY_MODE="metadata" +ALLOW_INSECURE_PASSWORD="false" + +usage() { + cat <<'EOF' +Usage: bash scripts/backup_release_state.sh [options] + +Creates an atomic, read-only release backup from the running all-in-one +GeoIntel container. It never deletes or restores application data. + +Options: + --container NAME Docker container (default: geointel) + --output-root PATH Host backup root (default: backups) + --release-id ID Safe backup directory name + --storage-path PATH Optional host storage path to inventory + --models-path PATH Optional host model path to inventory + --inventory-mode metadata|sha256 Hash all inventoried files only with sha256 + --allow-insecure-password Complete emergency backup despite an + empty/default production DB password +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --container) CONTAINER="$2"; shift 2 ;; + --output-root) OUTPUT_ROOT="$2"; shift 2 ;; + --release-id) RELEASE_ID="$2"; shift 2 ;; + --storage-path) STORAGE_PATH="$2"; shift 2 ;; + --models-path) MODELS_PATH="$2"; shift 2 ;; + --inventory-mode) INVENTORY_MODE="$2"; shift 2 ;; + --allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;; + --help|-h) usage; exit 0 ;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +if ! [[ "$RELEASE_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]]; then + echo "Unsafe release id: $RELEASE_ID" >&2 + exit 2 +fi +if [ "$INVENTORY_MODE" != "metadata" ] && [ "$INVENTORY_MODE" != "sha256" ]; then + echo "--inventory-mode must be metadata or sha256" >&2 + exit 2 +fi +for required in docker python3 sha256sum git; do + if ! command -v "$required" >/dev/null 2>&1; then + echo "Missing required command: $required" >&2 + exit 2 + fi +done +if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then + echo "Container '$CONTAINER' is not running." >&2 + exit 3 +fi + +OUTPUT_ROOT="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$OUTPUT_ROOT")" +mkdir -p "$OUTPUT_ROOT" +PARTIAL="$OUTPUT_ROOT/.${RELEASE_ID}.partial" +FINAL="$OUTPUT_ROOT/$RELEASE_ID" +if [ -e "$PARTIAL" ] || [ -e "$FINAL" ]; then + echo "Backup target already exists: $FINAL" >&2 + exit 3 +fi +mkdir -p "$PARTIAL" + +cleanup_partial() { + if [ -d "$PARTIAL" ]; then + rm -rf -- "$PARTIAL" + fi +} +trap cleanup_partial EXIT + +DB_NAME="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"' )" +DB_USER="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"' )" +if [ -z "$DB_NAME" ] || [ -z "$DB_USER" ]; then + echo "Could not resolve database identity from the container." >&2 + exit 3 +fi + +PASSWORD_SECURE="true" +if ! docker exec "$CONTAINER" sh -c ' + password="${POSTGRES_PASSWORD:-${GEOINTEL_POSTGRES_PASSWORD:-}}" + test -n "$password" && + test "$password" != "geointel" && + test "$password" != "postgres" && + test "$password" != "password" +'; then + PASSWORD_SECURE="false" +fi + +echo "Creating PostgreSQL custom-format dump..." +docker exec "$CONTAINER" pg_dump \ + -U "$DB_USER" \ + -d "$DB_NAME" \ + -Fc \ + --no-owner \ + --no-privileges > "$PARTIAL/database.dump" +test -s "$PARTIAL/database.dump" + +docker exec -i "$CONTAINER" pg_restore --list \ + < "$PARTIAL/database.dump" \ + > "$PARTIAL/database.list" +test -s "$PARTIAL/database.list" + +IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")" +IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")" +GIT_COMMIT="$(git rev-parse HEAD)" +GIT_DIRTY="false" +if [ -n "$(git status --porcelain=v1)" ]; then + GIT_DIRTY="true" +fi + +docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \ + -c "SELECT 'alembic_head', version_num FROM alembic_version + UNION ALL SELECT 'postgis_version', postgis_version() + UNION ALL SELECT 'database_size_bytes', pg_database_size(current_database())::text + ORDER BY 1;" > "$PARTIAL/database-metadata.tsv" + +: > "$PARTIAL/table-counts.tsv" +for table in projects areas datasets dataset_versions vector_features jobs analysis_runs detections segmentations quality_checks metrics exports; do + count="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -At \ + -c "SELECT count(*) FROM public.${table};")" + printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv" +done + +inventory_path() { + local source_path="$1" + local output_path="$2" + if [ -z "$source_path" ]; then + printf 'not_requested\n' > "$output_path" + return + fi + python3 - "$source_path" "$output_path" "$INVENTORY_MODE" <<'PY' +import hashlib +import os +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]).expanduser().resolve() +output = pathlib.Path(sys.argv[2]) +mode = sys.argv[3] +if not root.is_dir(): + raise SystemExit(f"Inventory root is not a directory: {root}") + +def digest(path: pathlib.Path) -> str: + value = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + +with output.open("w", encoding="utf-8", newline="\n") as handle: + handle.write("relative_path\tsize_bytes\tmtime_ns\tsha256\n") + for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()): + if path.is_symlink() or not path.is_file(): + continue + stat = path.stat() + checksum = digest(path) if mode == "sha256" else "" + relative = path.relative_to(root).as_posix() + if "\t" in relative or "\n" in relative: + raise SystemExit(f"Unsupported inventory path: {relative!r}") + handle.write(f"{relative}\t{stat.st_size}\t{stat.st_mtime_ns}\t{checksum}\n") +PY +} + +inventory_path "$STORAGE_PATH" "$PARTIAL/storage-manifest.tsv" +inventory_path "$MODELS_PATH" "$PARTIAL/models-manifest.tsv" + +python3 - "$PARTIAL/manifest.json" < CHECKSUMS.sha256 +) + +mv "$PARTIAL" "$FINAL" +trap - EXIT +echo "Release backup created: $FINAL" + +if [ "$PASSWORD_SECURE" != "true" ] && [ "$ALLOW_INSECURE_PASSWORD" != "true" ]; then + echo "SECURITY GATE FAILED: production database password is empty or a known default." >&2 + echo "The emergency backup is valid, but the release remains blocked." >&2 + exit 4 +fi diff --git a/scripts/capture_release_evidence.py b/scripts/capture_release_evidence.py new file mode 100644 index 00000000..44d81371 --- /dev/null +++ b/scripts/capture_release_evidence.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Capture a read-only, secret-free GeoIntel release evidence manifest.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import platform +import subprocess +import sys +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Sequence + + +ROOT = Path(__file__).resolve().parents[1] +BACKEND = ROOT / "backend" +DEFAULT_HASHED_FILES = ( + "AGENTS.md", + "backend/pyproject.toml", + "backend/alembic.ini", + "frontend/package.json", + "frontend/package-lock.json", + "docker-compose.yml", + "docker-compose.unraid.yml", + "deploy/unraid/Dockerfile.all-in-one", + "docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md", + "docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md", +) +DEPENDENCIES = ( + "alembic", + "fastapi", + "geoalchemy2", + "geopandas", + "numpy", + "pydantic", + "pyproj", + "rasterio", + "shapely", + "sqlalchemy", + "torch", + "ultralytics", + "uvicorn", +) + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def run_command( + command: Sequence[str], + *, + cwd: Path = ROOT, + timeout_seconds: int = 30, +) -> dict[str, Any]: + try: + result = subprocess.run( + list(command), + cwd=cwd, + capture_output=True, + text=True, + check=False, + timeout=timeout_seconds, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + return { + "ok": False, + "exit_code": None, + "stdout": "", + "stderr": str(exc), + } + return { + "ok": result.returncode == 0, + "exit_code": result.returncode, + "stdout": result.stdout.strip(), + "stderr": result.stderr.strip(), + } + + +def git_evidence() -> dict[str, Any]: + commit = run_command(("git", "rev-parse", "HEAD")) + branch = run_command(("git", "branch", "--show-current")) + status = run_command(("git", "status", "--porcelain=v1")) + remote = run_command(("git", "remote", "get-url", "origin")) + dirty_paths = [ + line[3:].strip() + for line in status["stdout"].splitlines() + if len(line) >= 4 + ] + return { + "commit": commit["stdout"] or None, + "branch": branch["stdout"] or None, + "origin": remote["stdout"] or None, + "dirty": bool(dirty_paths), + "dirty_paths": dirty_paths, + "commands_ok": all(item["ok"] for item in (commit, branch, status)), + } + + +def migration_evidence() -> dict[str, Any]: + heads = run_command((sys.executable, "-m", "alembic", "heads"), cwd=BACKEND) + head_lines = [ + line.strip() + for line in heads["stdout"].splitlines() + if line.strip() + ] + return { + "command_ok": heads["ok"], + "heads": head_lines, + "single_head": heads["ok"] and len(head_lines) == 1, + "error": heads["stderr"] or None, + } + + +def dependency_evidence() -> dict[str, str | None]: + versions: dict[str, str | None] = {} + for name in DEPENDENCIES: + try: + versions[name] = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + versions[name] = None + return versions + + +def file_evidence() -> dict[str, dict[str, Any]]: + evidence: dict[str, dict[str, Any]] = {} + for relative_path in DEFAULT_HASHED_FILES: + path = ROOT / relative_path + evidence[relative_path] = { + "exists": path.is_file(), + "size_bytes": path.stat().st_size if path.is_file() else None, + "sha256": sha256_file(path) if path.is_file() else None, + } + return evidence + + +def storage_evidence(storage_root: Path | None) -> dict[str, Any]: + if storage_root is None: + return {"requested": False} + resolved = storage_root.expanduser().resolve() + if not resolved.is_dir(): + return { + "requested": True, + "root": str(resolved), + "available": False, + } + total_size = 0 + file_count = 0 + errors: list[str] = [] + for path in resolved.rglob("*"): + if path.is_symlink() or not path.is_file(): + continue + try: + total_size += path.stat().st_size + file_count += 1 + except OSError as exc: + errors.append(f"{path}: {exc}") + if len(errors) >= 20: + break + return { + "requested": True, + "root": str(resolved), + "available": True, + "file_count": file_count, + "size_bytes": total_size, + "scan_complete": not errors, + "errors": errors, + } + + +def fetch_json(url: str, timeout_seconds: int) -> dict[str, Any]: + request = urllib.request.Request( + url, + headers={"Accept": "application/json", "User-Agent": "GeoIntel-RC-Evidence/1.0"}, + ) + try: + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: + body = response.read(2 * 1024 * 1024 + 1) + if len(body) > 2 * 1024 * 1024: + raise ValueError("response exceeds 2 MiB evidence limit") + return { + "ok": 200 <= response.status < 300, + "status_code": response.status, + "payload": json.loads(body.decode("utf-8")), + "error": None, + } + except (urllib.error.URLError, ValueError, json.JSONDecodeError) as exc: + return { + "ok": False, + "status_code": getattr(exc, "code", None), + "payload": None, + "error": str(exc), + } + + +def live_evidence(base_url: str | None, timeout_seconds: int) -> dict[str, Any]: + if not base_url: + return {"requested": False} + normalized = base_url.rstrip("/") + return { + "requested": True, + "base_url": normalized, + "health": fetch_json(f"{normalized}/health", timeout_seconds), + "capabilities": fetch_json( + f"{normalized}/api/v1/system/capabilities", + timeout_seconds, + ), + } + + +def build_manifest(args: argparse.Namespace) -> dict[str, Any]: + storage_root = Path(args.storage_root) if args.storage_root else None + return { + "schema_version": 1, + "release_id": args.release_id, + "captured_at": utc_now(), + "read_only": True, + "scope": "Belgium and the Belgian North Sea", + "host": { + "platform": platform.platform(), + "python": platform.python_version(), + "hostname": platform.node(), + }, + "git": git_evidence(), + "migrations": migration_evidence(), + "dependencies": dependency_evidence(), + "files": file_evidence(), + "storage": storage_evidence(storage_root), + "live": live_evidence(args.live_base_url, args.timeout_seconds), + "environment_presence": { + "DATABASE_URL": bool(os.environ.get("DATABASE_URL")), + "STORAGE_ROOT": bool(os.environ.get("STORAGE_ROOT")), + "YOLO_ENABLED": bool(os.environ.get("YOLO_ENABLED")), + "YOLO_MODEL_PATH": bool(os.environ.get("YOLO_MODEL_PATH")), + "OLLAMA_ENABLED": bool(os.environ.get("OLLAMA_ENABLED")), + }, + } + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--release-id", default="rc-current") + parser.add_argument("--live-base-url") + parser.add_argument("--storage-root") + parser.add_argument("--timeout-seconds", type=int, default=10) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + if args.timeout_seconds < 1 or args.timeout_seconds > 120: + raise SystemExit("--timeout-seconds must be between 1 and 120") + manifest = build_manifest(args) + output = args.output.expanduser().resolve() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(f"Wrote read-only release evidence to {output}") + if not manifest["git"]["commands_ok"]: + return 2 + if not manifest["migrations"]["single_head"]: + return 3 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/restore_release_backup_smoke.sh b/scripts/restore_release_backup_smoke.sh new file mode 100644 index 00000000..15fe7ee4 --- /dev/null +++ b/scripts/restore_release_backup_smoke.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONTAINER="geointel" +BACKUP_DIR="" +CONFIRM="false" +KEEP_DATABASE="false" +OUTPUT="" + +usage() { + cat <<'EOF' +Usage: bash scripts/restore_release_backup_smoke.sh --backup-dir PATH \ + --confirm-isolated-restore [options] + +Restores into a generated geointel_restore_verify_* database only, verifies +PostGIS, Alembic and retained table counts, then drops that temporary database. +It refuses the production database name and never uses pg_restore --clean. + +Options: + --container NAME + --output PATH Write result JSON outside the immutable backup directory + --keep-database Keep the isolated verification database for inspection +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --backup-dir) BACKUP_DIR="$2"; shift 2 ;; + --container) CONTAINER="$2"; shift 2 ;; + --confirm-isolated-restore) CONFIRM="true"; shift ;; + --keep-database) KEEP_DATABASE="true"; shift ;; + --output) OUTPUT="$2"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +if [ "$CONFIRM" != "true" ] || [ -z "$BACKUP_DIR" ]; then + echo "Explicit --confirm-isolated-restore and --backup-dir are required." >&2 + exit 2 +fi +for required in docker python3; do + command -v "$required" >/dev/null 2>&1 || { + echo "Missing required command: $required" >&2 + exit 2 + } +done +BACKUP_DIR="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$BACKUP_DIR")" +bash "$(dirname "$0")/verify_release_backup.sh" --backup-dir "$BACKUP_DIR" --container "$CONTAINER" + +DB_NAME="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"' )" +DB_USER="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"' )" +TARGET_DB="geointel_restore_verify_$(date -u +%Y%m%d%H%M%S)_$$" +if [ "$TARGET_DB" = "$DB_NAME" ] || ! [[ "$TARGET_DB" =~ ^geointel_restore_verify_[0-9_]+$ ]]; then + echo "Unsafe restore target: $TARGET_DB" >&2 + exit 3 +fi +if docker exec "$CONTAINER" psql -X -U "$DB_USER" -d postgres -Atqc \ + "SELECT 1 FROM pg_database WHERE datname = '$TARGET_DB';" | grep -q 1; then + echo "Generated restore database already exists." >&2 + exit 3 +fi + +cleanup_database() { + if [ "$KEEP_DATABASE" != "true" ]; then + docker exec "$CONTAINER" dropdb --if-exists -U "$DB_USER" "$TARGET_DB" >/dev/null 2>&1 || true + fi +} +trap cleanup_database EXIT + +docker exec "$CONTAINER" createdb -U "$DB_USER" "$TARGET_DB" +docker exec -i "$CONTAINER" pg_restore \ + --exit-on-error \ + --no-owner \ + --no-privileges \ + -U "$DB_USER" \ + -d "$TARGET_DB" < "$BACKUP_DIR/database.dump" + +POSTGIS_VERSION="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$TARGET_DB" -Atqc \ + "SELECT postgis_version();")" +ALEMBIC_HEAD="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$TARGET_DB" -Atqc \ + "SELECT version_num FROM alembic_version;")" +EXPECTED_HEAD="$(awk -F $'\t' '$1 == "alembic_head" { print $2 }' "$BACKUP_DIR/database-metadata.tsv")" +if [ "$ALEMBIC_HEAD" != "$EXPECTED_HEAD" ]; then + echo "Restored Alembic head '$ALEMBIC_HEAD' differs from '$EXPECTED_HEAD'." >&2 + exit 4 +fi + +while IFS=$'\t' read -r table expected; do + [[ "$table" =~ ^[a-z_]+$ ]] || { + echo "Unsafe table name in retained counts: $table" >&2 + exit 4 + } + actual="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$TARGET_DB" -Atqc \ + "SELECT count(*) FROM public.${table};")" + if [ "$actual" != "$expected" ]; then + echo "Restored count mismatch for $table: expected $expected, got $actual." >&2 + exit 4 + fi +done < "$BACKUP_DIR/table-counts.tsv" + +if [ -z "$OUTPUT" ]; then + OUTPUT="${BACKUP_DIR%/}-restore-smoke.json" +fi +python3 - "$OUTPUT" <&2; usage >&2; exit 2 ;; + esac +done + +if [ -z "$BACKUP_DIR" ]; then + usage >&2 + exit 2 +fi +for required in docker python3 sha256sum; do + command -v "$required" >/dev/null 2>&1 || { + echo "Missing required command: $required" >&2 + exit 2 + } +done +BACKUP_DIR="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$BACKUP_DIR")" +for required_file in manifest.json database.dump database.list database-metadata.tsv table-counts.tsv CHECKSUMS.sha256; do + test -s "$BACKUP_DIR/$required_file" || { + echo "Missing or empty backup artifact: $required_file" >&2 + exit 3 + } +done + +( + cd "$BACKUP_DIR" + sha256sum -c CHECKSUMS.sha256 +) + +python3 - "$BACKUP_DIR/manifest.json" <<'PY' +import json +import pathlib +import sys + +payload = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +required = { + "schema_version", + "release_id", + "created_at", + "read_only_source", + "database_name", + "database_user", + "image_id", + "git_commit", +} +missing = sorted(required - payload.keys()) +if missing: + raise SystemExit(f"Backup manifest misses: {', '.join(missing)}") +if payload["schema_version"] != 1 or payload["read_only_source"] is not True: + raise SystemExit("Unsupported or unsafe backup manifest") +PY + +if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then + echo "Container '$CONTAINER' is required to run pg_restore --list." >&2 + exit 3 +fi +TMP_LIST="$(mktemp)" +trap 'rm -f -- "$TMP_LIST"' EXIT +docker exec -i "$CONTAINER" pg_restore --list \ + < "$BACKUP_DIR/database.dump" \ + > "$TMP_LIST" +cmp -s "$TMP_LIST" "$BACKUP_DIR/database.list" || { + echo "The current pg_restore listing differs from the retained listing." >&2 + exit 3 +} +grep -q "TABLE DATA" "$TMP_LIST" || { + echo "Archive contains no table data entries." >&2 + exit 3 +} + +echo "Release backup verified read-only: $BACKUP_DIR"