Start autonomous Belgium and North Sea RC

backend/README.md

@@ -1,6 +1,19 @@
 # GeoIntel Backend (Sprint 3 foundation layer)
 
-FastAPI backend for GeoIntel Kempen Foundation Sprints.
+FastAPI backend for the GeoIntel Belgium and Belgian North Sea workbench.
+
+Runtime probes:
+
+- `GET /health/live`: process liveness, always independent from PostgreSQL.
+- `GET /health/ready`: fail-closed database/PostGIS/migration/storage
+  readiness used by Docker.
+- `GET /health`: compatibility alias for readiness.
+- `GET /api/v1/system/capabilities`: runtime-derived PostGIS, GIS dependency,
+  configured YOLO and provider state.
+
+The all-in-one production runtime enables interrupted job/analysis-run
+reconciliation at startup. Local tests and development leave it disabled
+unless `GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP=true`.
 
 The map-first explorer uses the existing persisted vector selection endpoint. Its bounded GeoJSON preview reports `feature_count`, while `total_feature_count` reports the exact PostGIS intersection count before the 1,000-feature response cap. When an active `area_id` is supplied, vector, temporal, derived-dataset and export paths all use `bbox ∩ Area`. A full-work-area bbox resolves to the exact persisted Area geometry; a boundary-crossing rectangle is clipped to the official boundary.
 

backend/app/api/routes/health.py

@@ -1,12 +1,23 @@
 from __future__ import annotations
 
 from importlib import import_module
-from sqlalchemy import text
-from fastapi import APIRouter
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 
-from app.schemas.health import HealthResponse, SystemCapabilities
-from app.providers.registry import list_provider_capabilities
+from alembic.config import Config
+from alembic.script import ScriptDirectory
+from fastapi import APIRouter, Response, status
+from sqlalchemy import text
+
+from app.core.config import get_settings
 from app.db.session import get_engine
+from app.providers.registry import list_provider_capabilities
+from app.schemas.health import (
+    HealthResponse,
+    SystemCapabilities,
+    SystemCapabilitiesEnvelope,
+)
+from app.services.model_registry_service import ModelRegistryService
 
 router = APIRouter()
 
@@ -19,27 +30,136 @@ def _dependency_enabled(module: str) -> bool:
         return False
 
 
-@router.get("/health")
-def readiness() -> HealthResponse:
-    db_status = "ok"
+def _expected_migration_heads() -> list[str]:
+    backend_root = Path(__file__).resolve().parents[3]
+    config = Config(str(backend_root / "alembic.ini"))
+    config.set_main_option("script_location", str(backend_root / "alembic"))
+    return list(ScriptDirectory.from_config(config).get_heads())
+
+
+def _database_checks() -> dict[str, str]:
+    checks = {
+        "database": "degraded",
+        "postgis": "degraded",
+        "migration": "degraded",
+    }
     try:
         with get_engine().connect() as connection:
             connection.execute(text("SELECT 1"))
+            checks["database"] = "ok"
+            postgis_version = connection.execute(
+                text("SELECT PostGIS_Version()")
+            ).scalar_one()
+            checks["postgis"] = f"ok:{postgis_version}"
+            database_head = connection.execute(
+                text("SELECT version_num FROM alembic_version")
+            ).scalar_one()
+            expected_heads = _expected_migration_heads()
+            if len(expected_heads) == 1 and database_head == expected_heads[0]:
+                checks["migration"] = f"ok:{database_head}"
+            else:
+                checks["migration"] = (
+                    f"degraded:database={database_head};"
+                    f"expected={','.join(expected_heads) or 'none'}"
+                )
     except Exception:
-        db_status = "degraded"
-    return HealthResponse(status="ok", service="geointel-backend", version="0.1.0", database=db_status)
+        return checks
+    return checks
 
 
-@router.get("/api/v1/system/capabilities")
-def capabilities() -> dict:
+def _storage_check(storage_root: str) -> str:
+    root = Path(storage_root).expanduser()
+    try:
+        root.mkdir(parents=True, exist_ok=True)
+        with NamedTemporaryFile(
+            prefix=".geointel-readiness-",
+            dir=root,
+            delete=True,
+        ) as handle:
+            handle.write(b"ok")
+            handle.flush()
+        return "ok"
+    except OSError:
+        return "degraded"
+
+
+def _readiness_payload() -> HealthResponse:
+    settings = get_settings()
+    checks = _database_checks()
+    checks["storage"] = _storage_check(settings.storage_root)
+    ready = all(
+        value == "ok" or value.startswith("ok:")
+        for value in checks.values()
+    )
+    return HealthResponse(
+        status="ok" if ready else "degraded",
+        service="geointel-backend",
+        version=settings.app_version,
+        build_sha=settings.build_sha,
+        build_time=settings.build_time,
+        database=checks["database"],
+        postgis=checks["postgis"],
+        migration=checks["migration"],
+        storage=checks["storage"],
+        checks=checks,
+    )
+
+
+@router.get("/health/live", response_model=HealthResponse)
+def liveness() -> HealthResponse:
+    settings = get_settings()
+    return HealthResponse(
+        status="ok",
+        service="geointel-backend",
+        version=settings.app_version,
+        build_sha=settings.build_sha,
+        build_time=settings.build_time,
+    )
+
+
+def _readiness_response(response: Response) -> HealthResponse:
+    payload = _readiness_payload()
+    if payload.status != "ok":
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+    return payload
+
+
+@router.get("/health", response_model=HealthResponse)
+def readiness(response: Response) -> HealthResponse:
+    return _readiness_response(response)
+
+
+@router.get("/health/ready", response_model=HealthResponse)
+def readiness_explicit(response: Response) -> HealthResponse:
+    return _readiness_response(response)
+
+
+@router.get(
+    "/api/v1/system/capabilities",
+    response_model=SystemCapabilitiesEnvelope,
+)
+def capabilities() -> SystemCapabilitiesEnvelope:
+    settings = get_settings()
     providers = [item.to_dict() for item in list_provider_capabilities()]
-    return {"data": SystemCapabilities(
-        postgis=True,
-        rasterio=_dependency_enabled("rasterio"),
-        geopandas=_dependency_enabled("geopandas"),
-        yolo=False,
-        sam=False,
-        grb="bounded",
-        sentinel="planned",
-        providers=providers,
-    ).model_dump()}
+    configured_yolo = ModelRegistryService.get_model_capability(
+        settings.yolo_model_id,
+        settings=settings,
+    )
+    yolo_configured = bool(configured_yolo and configured_yolo.configured)
+    yolo_status = configured_yolo.status if configured_yolo else "not_configured"
+    postgis_ready = _database_checks()["postgis"].startswith("ok:")
+    return SystemCapabilitiesEnvelope(
+        data=SystemCapabilities(
+            postgis=postgis_ready,
+            rasterio=_dependency_enabled("rasterio"),
+            geopandas=_dependency_enabled("geopandas"),
+            yolo=yolo_configured,
+            yolo_status=yolo_status,
+            sam=False,
+            grb="bounded",
+            sentinel="planned",
+            version=settings.app_version,
+            build_sha=settings.build_sha,
+            providers=providers,
+        )
+    )

backend/app/core/config.py

@@ -12,6 +12,8 @@ class Settings(BaseSettings):
 
     app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV")
     app_version: str = Field(default="0.1.0")
+    build_sha: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_SHA")
+    build_time: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_TIME")
     api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX")
     database_url: str = Field(
         default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1",
@@ -231,6 +233,11 @@ class Settings(BaseSettings):
     thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
     redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
     log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
+    sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL")
+    reconcile_interrupted_runs_on_startup: bool = Field(
+        default=False,
+        validation_alias="GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP",
+    )
     database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
     yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED")
     yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR")

backend/app/core/logging.py

@@ -2,11 +2,13 @@ import logging
 import sys
 
 
-def configure_logging(level: str = "INFO") -> None:
+def configure_logging(level: str = "INFO", sql_level: str = "WARNING") -> None:
     logging.basicConfig(
         level=level,
         format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
         stream=sys.stdout,
+        force=True,
     )
-    for name in ["uvicorn", "uvicorn.error", "uvicorn.access", "sqlalchemy.engine"]:
+    for name in ["uvicorn", "uvicorn.error", "uvicorn.access"]:
         logging.getLogger(name).setLevel(level)
+    logging.getLogger("sqlalchemy.engine").setLevel(sql_level)

backend/app/main.py

@@ -1,5 +1,9 @@
 from __future__ import annotations
 
+import logging
+import uuid
+from contextlib import asynccontextmanager
+
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
@@ -9,6 +13,11 @@ from app.api.routes import analysis, areas, assistant, datasets, demo, detection
 from app.core.config import get_settings
 from app.core.errors import AppError
 from app.core.logging import configure_logging
+from app.db.session import SessionLocal
+from app.services.runtime_reconciliation_service import RuntimeReconciliationService
+
+
+logger = logging.getLogger("geointel")
 
 
 def _to_error_payload(
@@ -27,13 +36,33 @@ def _to_error_payload(
 
 def create_app() -> FastAPI:
     settings = get_settings()
-    configure_logging(settings.log_level)
+    configure_logging(settings.log_level, settings.sql_log_level)
+
+    @asynccontextmanager
+    async def lifespan(_: FastAPI):
+        if settings.reconcile_interrupted_runs_on_startup:
+            db = SessionLocal()
+            try:
+                result = RuntimeReconciliationService.reconcile(db)
+                logger.info(
+                    "Runtime reconciliation completed: jobs=%s analysis_runs=%s",
+                    result.interrupted_jobs,
+                    result.interrupted_analysis_runs,
+                )
+            except Exception:
+                db.rollback()
+                logger.exception("Runtime reconciliation failed")
+                raise
+            finally:
+                db.close()
+        yield
 
     app = FastAPI(
-        title="GeoIntel Kempen",
+        title="GeoIntel",
         version=settings.app_version,
         docs_url="/docs",
         redoc_url="/redoc",
+        lifespan=lifespan,
     )
 
     app.add_middleware(
@@ -60,6 +89,14 @@ def create_app() -> FastAPI:
     app.include_router(temporal.router, prefix=settings.api_prefix)
     app.include_router(assistant.router, prefix=settings.api_prefix)
 
+    @app.middleware("http")
+    async def request_identity(request: Request, call_next):
+        request_id = request.headers.get("x-request-id") or str(uuid.uuid4())
+        request.state.request_id = request_id
+        response = await call_next(request)
+        response.headers["x-request-id"] = request_id
+        return response
+
     @app.exception_handler(AppError)
     async def app_error(request: Request, exc: AppError):  # noqa: ARG001
         return JSONResponse(
@@ -68,7 +105,7 @@ def create_app() -> FastAPI:
                 exc.code,
                 exc.message,
                 exc.details,
-                request_id=request.headers.get("x-request-id"),
+                request_id=request.state.request_id,
             ),
         )
 
@@ -88,7 +125,7 @@ def create_app() -> FastAPI:
                 code,
                 message,
                 details,
-                request_id=request.headers.get("x-request-id"),
+                request_id=request.state.request_id,
             ),
         )
 
@@ -100,19 +137,25 @@ def create_app() -> FastAPI:
                 "VALIDATION_ERROR",
                 "Validation failed",
                 exc.errors(),
-                request_id=request.headers.get("x-request-id"),
+                request_id=request.state.request_id,
             ),
         )
 
     @app.exception_handler(Exception)
-    async def unexpected_error(request: Request, exc: Exception):  # noqa: ARG001
+    async def unexpected_error(request: Request, exc: Exception):
+        logger.exception(
+            "Unhandled request error request_id=%s method=%s path=%s",
+            request.state.request_id,
+            request.method,
+            request.url.path,
+        )
         return JSONResponse(
             status_code=500,
             content=_to_error_payload(
                 "INTERNAL_ERROR",
                 "Unexpected server error",
                 {"type": exc.__class__.__name__},
-                request_id=request.headers.get("x-request-id"),
+                request_id=request.state.request_id,
             ),
         )
 

backend/app/schemas/health.py

@@ -23,7 +23,13 @@ class HealthResponse(BaseModel):
     status: str
     service: str
     version: str
+    build_sha: str | None = None
+    build_time: str | None = None
     database: str | None = None
+    postgis: str | None = None
+    migration: str | None = None
+    storage: str | None = None
+    checks: dict[str, str] = Field(default_factory=dict)
 
 
 class SystemCapabilities(BaseModel):
@@ -31,7 +37,14 @@ class SystemCapabilities(BaseModel):
     rasterio: bool
     geopandas: bool
     yolo: bool | str
+    yolo_status: str
     sam: bool | str
     grb: str
     sentinel: str
+    version: str
+    build_sha: str | None = None
     providers: list[ProviderCapability] = Field(default_factory=list)
+
+
+class SystemCapabilitiesEnvelope(BaseModel):
+    data: SystemCapabilities

backend/app/services/runtime_reconciliation_service.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+from sqlalchemy.orm import Session
+
+from app.models import AnalysisRun, Job
+
+
+@dataclass(frozen=True)
+class ReconciliationResult:
+    interrupted_jobs: int
+    interrupted_analysis_runs: int
+
+
+class RuntimeReconciliationService:
+    ERROR_MESSAGE = (
+        "PROCESS_INTERRUPTED: the GeoIntel process restarted before this work "
+        "reached a terminal state"
+    )
+
+    @staticmethod
+    def reconcile(
+        db: Session,
+        *,
+        finished_at: datetime | None = None,
+    ) -> ReconciliationResult:
+        resolved_finished_at = finished_at or datetime.now(timezone.utc)
+        interrupted_jobs = (
+            db.query(Job)
+            .filter(Job.status == "running")
+            .update(
+                {
+                    Job.status: "failed",
+                    Job.finished_at: resolved_finished_at,
+                    Job.error_message: RuntimeReconciliationService.ERROR_MESSAGE,
+                },
+                synchronize_session=False,
+            )
+        )
+        interrupted_analysis_runs = (
+            db.query(AnalysisRun)
+            .filter(AnalysisRun.status == "running")
+            .update(
+                {
+                    AnalysisRun.status: "failed",
+                    AnalysisRun.finished_at: resolved_finished_at,
+                    AnalysisRun.error_message: RuntimeReconciliationService.ERROR_MESSAGE,
+                },
+                synchronize_session=False,
+            )
+        )
+        db.commit()
+        return ReconciliationResult(
+            interrupted_jobs=interrupted_jobs,
+            interrupted_analysis_runs=interrupted_analysis_runs,
+        )

backend/tests/test_docker_runtime_config.py

@@ -204,9 +204,9 @@ def test_compose_mounts_demo_fixtures_for_backend_runtime() -> None:
 def test_compose_has_backend_and_frontend_healthchecks() -> None:
     compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
 
-    assert "http://127.0.0.1:8000/health" in compose
+    assert "http://127.0.0.1:8000/health/ready" in compose
     assert "urllib.request.urlopen" in compose
-    assert "http://127.0.0.1/health" in compose
+    assert "http://127.0.0.1/health/ready" in compose
     assert "wget -q -O -" in compose
     assert "start_period: 30s" in compose
     assert "start_period: 10s" in compose

backend/tests/test_health.py

@@ -1,34 +1,100 @@
 from __future__ import annotations
 
+from types import SimpleNamespace
+
 from fastapi.testclient import TestClient
 
 from app.api.routes import health
 from app.main import app
 
 
-def test_health_endpoint_returns_status_payload() -> None:
-    client = TestClient(app)
-    response = client.get("/health")
+READY_DATABASE = {
+    "database": "ok",
+    "postgis": "ok:3.4 USE_GEOS=1 USE_PROJ=1",
+    "migration": "ok:202607160001",
+}
+
+
+def test_liveness_is_independent_from_database(monkeypatch) -> None:
+    monkeypatch.setattr(
+        health,
+        "_database_checks",
+        lambda: (_ for _ in ()).throw(AssertionError("must not query DB")),
+    )
+
+    response = TestClient(app).get("/health/live")
 
     assert response.status_code == 200
+    assert response.json()["status"] == "ok"
 
+
+def test_readiness_returns_ok_only_when_all_checks_pass(monkeypatch) -> None:
+    monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy())
+    monkeypatch.setattr(health, "_storage_check", lambda _: "ok")
+
+    response = TestClient(app).get("/health/ready")
+
+    assert response.status_code == 200
     payload = response.json()
-    assert payload["status"] in {"ok", "degraded"}
+    assert payload["status"] == "ok"
     assert payload["service"] == "geointel-backend"
-    assert payload["version"] == "0.1.0"
+    assert payload["database"] == "ok"
+    assert payload["postgis"].startswith("ok:")
+    assert payload["migration"] == "ok:202607160001"
+    assert payload["storage"] == "ok"
 
 
-def test_system_capabilities_reports_gis_dependency_flags(monkeypatch) -> None:
+def test_compatibility_health_is_fail_closed(monkeypatch) -> None:
+    monkeypatch.setattr(
+        health,
+        "_database_checks",
+        lambda: {
+            "database": "degraded",
+            "postgis": "degraded",
+            "migration": "degraded",
+        },
+    )
+    monkeypatch.setattr(health, "_storage_check", lambda _: "ok")
+
+    response = TestClient(app).get("/health")
+
+    assert response.status_code == 503
+    assert response.json()["status"] == "degraded"
+
+
+def test_system_capabilities_report_runtime_state(monkeypatch) -> None:
     monkeypatch.setattr(
         health,
         "_dependency_enabled",
         lambda module_name: module_name in {"rasterio", "geopandas"},
     )
+    monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy())
+    monkeypatch.setattr(
+        health.ModelRegistryService,
+        "get_model_capability",
+        lambda *args, **kwargs: SimpleNamespace(
+            configured=True,
+            status="configured",
+        ),
+    )
 
-    client = TestClient(app)
-    response = client.get("/api/v1/system/capabilities")
+    response = TestClient(app).get("/api/v1/system/capabilities")
 
     assert response.status_code == 200
-    payload = response.json()
-    assert payload["data"]["rasterio"] is True
-    assert payload["data"]["geopandas"] is True
+    payload = response.json()["data"]
+    assert payload["postgis"] is True
+    assert payload["rasterio"] is True
+    assert payload["geopandas"] is True
+    assert payload["yolo"] is True
+    assert payload["yolo_status"] == "configured"
+    assert payload["version"]
+
+
+def test_requests_receive_a_correlation_id() -> None:
+    client = TestClient(app)
+
+    generated = client.get("/health/live")
+    retained = client.get("/health/live", headers={"x-request-id": "test-request"})
+
+    assert generated.headers["x-request-id"]
+    assert retained.headers["x-request-id"] == "test-request"

backend/tests/test_rc_backup_restore_scripts.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPTS = ROOT / "scripts"
+
+
+def read(name: str) -> str:
+    return (SCRIPTS / name).read_text(encoding="utf-8")
+
+
+def test_backup_is_atomic_read_only_and_checksum_bound() -> None:
+    script = read("backup_release_state.sh")
+
+    assert "pg_dump" in script
+    assert "-Fc" in script
+    assert "--no-owner" in script
+    assert "CHECKSUMS.sha256" in script
+    assert "database-password" not in script.lower()
+    assert "mv \"$PARTIAL\" \"$FINAL\"" in script
+    assert "rm -rf -- \"$PARTIAL\"" in script
+    assert "DROP DATABASE" not in script
+    assert "pg_restore --clean" not in script
+
+
+def test_backup_verification_is_read_only() -> None:
+    script = read("verify_release_backup.sh")
+
+    assert "sha256sum -c CHECKSUMS.sha256" in script
+    assert "pg_restore --list" in script
+    assert "createdb" not in script
+    assert "dropdb" not in script
+    assert "pg_restore --clean" not in script
+
+
+def test_restore_smoke_is_forced_to_generated_isolated_database() -> None:
+    script = read("restore_release_backup_smoke.sh")
+
+    assert "--confirm-isolated-restore" in script
+    assert "geointel_restore_verify_" in script
+    assert 'if [ "$TARGET_DB" = "$DB_NAME" ]' in script
+    assert "createdb" in script
+    assert "dropdb --if-exists" in script
+    assert "pg_restore \\\n  --clean" not in script
+    assert '"production_database_untouched": True' in script
+
+
+def test_release_safety_scripts_have_valid_bash_syntax() -> None:
+    for name in (
+        "backup_release_state.sh",
+        "verify_release_backup.sh",
+        "restore_release_backup_smoke.sh",
+    ):
+        result = subprocess.run(
+            ["bash", "-n", f"scripts/{name}"],
+            cwd=ROOT,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        assert result.returncode == 0, f"{name}: {result.stderr}"
+
+
+def test_readiness_gate_checks_release_safety_scripts() -> None:
+    readiness = read("run_readiness_check.sh")
+
+    for name in (
+        "backup_release_state.sh",
+        "verify_release_backup.sh",
+        "restore_release_backup_smoke.sh",
+    ):
+        assert f"bash -n scripts/{name}" in readiness

backend/tests/test_rc_release_evidence.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = ROOT / "scripts" / "capture_release_evidence.py"
+
+
+def load_script():
+    spec = importlib.util.spec_from_file_location("capture_release_evidence", SCRIPT)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_release_evidence_manifest_is_secret_free_and_read_only(tmp_path: Path) -> None:
+    module = load_script()
+    args = module.parse_args(
+        [
+            "--output",
+            str(tmp_path / "evidence.json"),
+            "--release-id",
+            "test-rc",
+        ]
+    )
+
+    manifest = module.build_manifest(args)
+
+    assert manifest["schema_version"] == 1
+    assert manifest["release_id"] == "test-rc"
+    assert manifest["read_only"] is True
+    assert manifest["scope"] == "Belgium and the Belgian North Sea"
+    assert "DATABASE_URL" not in json.dumps(manifest).replace(
+        '"DATABASE_URL": false',
+        "",
+    ).replace(
+        '"DATABASE_URL": true',
+        "",
+    )
+    assert manifest["storage"] == {"requested": False}
+    assert manifest["live"] == {"requested": False}
+
+
+def test_release_evidence_cli_writes_single_head_manifest(tmp_path: Path) -> None:
+    output = tmp_path / "baseline.json"
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(SCRIPT),
+            "--output",
+            str(output),
+            "--release-id",
+            "test-cli",
+        ],
+        cwd=ROOT,
+        capture_output=True,
+        text=True,
+        check=False,
+        timeout=60,
+    )
+
+    assert result.returncode == 0, result.stderr
+    payload = json.loads(output.read_text(encoding="utf-8"))
+    assert payload["git"]["commit"]
+    assert payload["migrations"]["single_head"] is True
+    assert payload["files"]["docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md"]["sha256"]
+    assert payload["files"]["docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md"]["sha256"]
+
+
+def test_readiness_gate_compiles_release_evidence_command() -> None:
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
+        encoding="utf-8"
+    )
+
+    assert "py_compile scripts/capture_release_evidence.py" in readiness

backend/tests/test_runtime_reconciliation_service.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from unittest.mock import MagicMock
+
+from app.models import AnalysisRun, Job
+from app.services.runtime_reconciliation_service import RuntimeReconciliationService
+
+
+def test_reconciliation_terminalizes_only_running_work() -> None:
+    db = MagicMock()
+    jobs = MagicMock()
+    runs = MagicMock()
+    jobs.filter.return_value.update.return_value = 5
+    runs.filter.return_value.update.return_value = 2
+    db.query.side_effect = [jobs, runs]
+    finished_at = datetime(2026, 7, 17, 20, 0, tzinfo=timezone.utc)
+
+    result = RuntimeReconciliationService.reconcile(
+        db,
+        finished_at=finished_at,
+    )
+
+    assert result.interrupted_jobs == 5
+    assert result.interrupted_analysis_runs == 2
+    jobs.filter.assert_called_once()
+    runs.filter.assert_called_once()
+    job_values = jobs.filter.return_value.update.call_args.args[0]
+    run_values = runs.filter.return_value.update.call_args.args[0]
+    assert job_values[Job.status] == "failed"
+    assert job_values[Job.finished_at] == finished_at
+    assert "PROCESS_INTERRUPTED" in job_values[Job.error_message]
+    assert run_values[AnalysisRun.status] == "failed"
+    assert run_values[AnalysisRun.finished_at] == finished_at
+    assert "PROCESS_INTERRUPTED" in run_values[AnalysisRun.error_message]
+    db.commit.assert_called_once_with()
+
+
+def test_startup_reconciliation_is_enabled_only_in_all_in_one_runtime() -> None:
+    start_script = (
+        __import__("pathlib").Path(__file__).resolve().parents[2]
+        / "deploy"
+        / "unraid"
+        / "all-in-one-start.sh"
+    ).read_text(encoding="utf-8")
+
+    assert (
+        'GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true'
+        in start_script
+    )

backend/tests/test_sprint31_unraid_template.py

@@ -99,7 +99,7 @@ def test_unraid_all_in_one_runtime_starts_embedded_postgis_backend_and_nginx() -
     assert "migrate_compose_volume_if_needed" in dockerman_script
     assert "docker rm -f geointel" in dockerman_script
     assert "proxy_pass http://127.0.0.1:8000/api/" in nginx_config
-    assert "proxy_pass http://127.0.0.1:8000/health" in nginx_config
+    assert "proxy_pass http://127.0.0.1:8000/health/ready" in nginx_config
     assert "location = /geointel-icon.png" in nginx_config
     assert "frontend/node_modules" in dockerignore
     assert "storage" in dockerignore