Start autonomous Belgium and North Sea RC
This commit is contained in:
@@ -28,6 +28,7 @@ build/
|
||||
/storage/models/*
|
||||
/storage/operator-data/*
|
||||
/storage/operator-evidence/*
|
||||
/storage/release-evidence/*
|
||||
/storage/previews/*
|
||||
/storage/training/*
|
||||
/storage/ultralytics/*
|
||||
|
||||
@@ -7,6 +7,27 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Autonomous Belgium and North Sea RC program (2026-07-17)
|
||||
|
||||
- Expanded the release-candidate geography from Mol/Kempen to all of Belgium
|
||||
and the Belgian North Sea while retaining the existing areas as golden
|
||||
regression references.
|
||||
- Added an executable autonomous RC-0 through RC-11 roadmap and explicit
|
||||
national/maritime scope freeze.
|
||||
- Folded fresh-install, upgrade, rollback and runtime proof into RC-5 and
|
||||
RC-11 instead of creating a separate RC-12 phase.
|
||||
- Added a read-only release-evidence manifest command with Git, migration,
|
||||
dependency, configuration checksum and optional live endpoint evidence.
|
||||
- Replaced the obsolete pre-build status with the current implemented
|
||||
foundation and active release blockers.
|
||||
- Added atomic PostgreSQL release backup, read-only checksum verification and
|
||||
isolated generated-database restore-smoke tooling.
|
||||
- Split health into process liveness and fail-closed readiness covering
|
||||
PostgreSQL, PostGIS, Alembic head and writable storage.
|
||||
- Made system capabilities report real PostGIS and configured local YOLO
|
||||
state, added request IDs and exception logging, reduced SQL engine logging,
|
||||
and terminalized impossible orphaned work after all-in-one restarts.
|
||||
|
||||
## Sprint 240 Operational forest, agriculture, nature and soil themes (2026-07-17)
|
||||
|
||||
- Extended the governed Landgebruik Vlaanderen 2025 raster registry with
|
||||
|
||||
+14
-1
@@ -1,6 +1,19 @@
|
||||
# GeoIntel Backend (Sprint 3 foundation layer)
|
||||
|
||||
FastAPI backend for GeoIntel Kempen Foundation Sprints.
|
||||
FastAPI backend for the GeoIntel Belgium and Belgian North Sea workbench.
|
||||
|
||||
Runtime probes:
|
||||
|
||||
- `GET /health/live`: process liveness, always independent from PostgreSQL.
|
||||
- `GET /health/ready`: fail-closed database/PostGIS/migration/storage
|
||||
readiness used by Docker.
|
||||
- `GET /health`: compatibility alias for readiness.
|
||||
- `GET /api/v1/system/capabilities`: runtime-derived PostGIS, GIS dependency,
|
||||
configured YOLO and provider state.
|
||||
|
||||
The all-in-one production runtime enables interrupted job/analysis-run
|
||||
reconciliation at startup. Local tests and development leave it disabled
|
||||
unless `GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP=true`.
|
||||
|
||||
The map-first explorer uses the existing persisted vector selection endpoint. Its bounded GeoJSON preview reports `feature_count`, while `total_feature_count` reports the exact PostGIS intersection count before the 1,000-feature response cap. When an active `area_id` is supplied, vector, temporal, derived-dataset and export paths all use `bbox ∩ Area`. A full-work-area bbox resolves to the exact persisted Area geometry; a boundary-crossing rectangle is clipped to the official boundary.
|
||||
|
||||
|
||||
@@ -1,12 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from importlib import import_module
|
||||
from sqlalchemy import text
|
||||
from fastapi import APIRouter
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
from app.schemas.health import HealthResponse, SystemCapabilities
|
||||
from app.providers.registry import list_provider_capabilities
|
||||
from alembic.config import Config
|
||||
from alembic.script import ScriptDirectory
|
||||
from fastapi import APIRouter, Response, status
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.session import get_engine
|
||||
from app.providers.registry import list_provider_capabilities
|
||||
from app.schemas.health import (
|
||||
HealthResponse,
|
||||
SystemCapabilities,
|
||||
SystemCapabilitiesEnvelope,
|
||||
)
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -19,27 +30,136 @@ def _dependency_enabled(module: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
def readiness() -> HealthResponse:
|
||||
db_status = "ok"
|
||||
def _expected_migration_heads() -> list[str]:
|
||||
backend_root = Path(__file__).resolve().parents[3]
|
||||
config = Config(str(backend_root / "alembic.ini"))
|
||||
config.set_main_option("script_location", str(backend_root / "alembic"))
|
||||
return list(ScriptDirectory.from_config(config).get_heads())
|
||||
|
||||
|
||||
def _database_checks() -> dict[str, str]:
|
||||
checks = {
|
||||
"database": "degraded",
|
||||
"postgis": "degraded",
|
||||
"migration": "degraded",
|
||||
}
|
||||
try:
|
||||
with get_engine().connect() as connection:
|
||||
connection.execute(text("SELECT 1"))
|
||||
checks["database"] = "ok"
|
||||
postgis_version = connection.execute(
|
||||
text("SELECT PostGIS_Version()")
|
||||
).scalar_one()
|
||||
checks["postgis"] = f"ok:{postgis_version}"
|
||||
database_head = connection.execute(
|
||||
text("SELECT version_num FROM alembic_version")
|
||||
).scalar_one()
|
||||
expected_heads = _expected_migration_heads()
|
||||
if len(expected_heads) == 1 and database_head == expected_heads[0]:
|
||||
checks["migration"] = f"ok:{database_head}"
|
||||
else:
|
||||
checks["migration"] = (
|
||||
f"degraded:database={database_head};"
|
||||
f"expected={','.join(expected_heads) or 'none'}"
|
||||
)
|
||||
except Exception:
|
||||
db_status = "degraded"
|
||||
return HealthResponse(status="ok", service="geointel-backend", version="0.1.0", database=db_status)
|
||||
return checks
|
||||
return checks
|
||||
|
||||
|
||||
@router.get("/api/v1/system/capabilities")
|
||||
def capabilities() -> dict:
|
||||
def _storage_check(storage_root: str) -> str:
|
||||
root = Path(storage_root).expanduser()
|
||||
try:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
with NamedTemporaryFile(
|
||||
prefix=".geointel-readiness-",
|
||||
dir=root,
|
||||
delete=True,
|
||||
) as handle:
|
||||
handle.write(b"ok")
|
||||
handle.flush()
|
||||
return "ok"
|
||||
except OSError:
|
||||
return "degraded"
|
||||
|
||||
|
||||
def _readiness_payload() -> HealthResponse:
|
||||
settings = get_settings()
|
||||
checks = _database_checks()
|
||||
checks["storage"] = _storage_check(settings.storage_root)
|
||||
ready = all(
|
||||
value == "ok" or value.startswith("ok:")
|
||||
for value in checks.values()
|
||||
)
|
||||
return HealthResponse(
|
||||
status="ok" if ready else "degraded",
|
||||
service="geointel-backend",
|
||||
version=settings.app_version,
|
||||
build_sha=settings.build_sha,
|
||||
build_time=settings.build_time,
|
||||
database=checks["database"],
|
||||
postgis=checks["postgis"],
|
||||
migration=checks["migration"],
|
||||
storage=checks["storage"],
|
||||
checks=checks,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/health/live", response_model=HealthResponse)
|
||||
def liveness() -> HealthResponse:
|
||||
settings = get_settings()
|
||||
return HealthResponse(
|
||||
status="ok",
|
||||
service="geointel-backend",
|
||||
version=settings.app_version,
|
||||
build_sha=settings.build_sha,
|
||||
build_time=settings.build_time,
|
||||
)
|
||||
|
||||
|
||||
def _readiness_response(response: Response) -> HealthResponse:
|
||||
payload = _readiness_payload()
|
||||
if payload.status != "ok":
|
||||
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
return payload
|
||||
|
||||
|
||||
@router.get("/health", response_model=HealthResponse)
|
||||
def readiness(response: Response) -> HealthResponse:
|
||||
return _readiness_response(response)
|
||||
|
||||
|
||||
@router.get("/health/ready", response_model=HealthResponse)
|
||||
def readiness_explicit(response: Response) -> HealthResponse:
|
||||
return _readiness_response(response)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/system/capabilities",
|
||||
response_model=SystemCapabilitiesEnvelope,
|
||||
)
|
||||
def capabilities() -> SystemCapabilitiesEnvelope:
|
||||
settings = get_settings()
|
||||
providers = [item.to_dict() for item in list_provider_capabilities()]
|
||||
return {"data": SystemCapabilities(
|
||||
postgis=True,
|
||||
rasterio=_dependency_enabled("rasterio"),
|
||||
geopandas=_dependency_enabled("geopandas"),
|
||||
yolo=False,
|
||||
sam=False,
|
||||
grb="bounded",
|
||||
sentinel="planned",
|
||||
providers=providers,
|
||||
).model_dump()}
|
||||
configured_yolo = ModelRegistryService.get_model_capability(
|
||||
settings.yolo_model_id,
|
||||
settings=settings,
|
||||
)
|
||||
yolo_configured = bool(configured_yolo and configured_yolo.configured)
|
||||
yolo_status = configured_yolo.status if configured_yolo else "not_configured"
|
||||
postgis_ready = _database_checks()["postgis"].startswith("ok:")
|
||||
return SystemCapabilitiesEnvelope(
|
||||
data=SystemCapabilities(
|
||||
postgis=postgis_ready,
|
||||
rasterio=_dependency_enabled("rasterio"),
|
||||
geopandas=_dependency_enabled("geopandas"),
|
||||
yolo=yolo_configured,
|
||||
yolo_status=yolo_status,
|
||||
sam=False,
|
||||
grb="bounded",
|
||||
sentinel="planned",
|
||||
version=settings.app_version,
|
||||
build_sha=settings.build_sha,
|
||||
providers=providers,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -12,6 +12,8 @@ class Settings(BaseSettings):
|
||||
|
||||
app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV")
|
||||
app_version: str = Field(default="0.1.0")
|
||||
build_sha: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_SHA")
|
||||
build_time: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_TIME")
|
||||
api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX")
|
||||
database_url: str = Field(
|
||||
default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1",
|
||||
@@ -231,6 +233,11 @@ class Settings(BaseSettings):
|
||||
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
|
||||
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
|
||||
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
|
||||
sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL")
|
||||
reconcile_interrupted_runs_on_startup: bool = Field(
|
||||
default=False,
|
||||
validation_alias="GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP",
|
||||
)
|
||||
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
|
||||
yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED")
|
||||
yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR")
|
||||
|
||||
@@ -2,11 +2,13 @@ import logging
|
||||
import sys
|
||||
|
||||
|
||||
def configure_logging(level: str = "INFO") -> None:
|
||||
def configure_logging(level: str = "INFO", sql_level: str = "WARNING") -> None:
|
||||
logging.basicConfig(
|
||||
level=level,
|
||||
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
||||
stream=sys.stdout,
|
||||
force=True,
|
||||
)
|
||||
for name in ["uvicorn", "uvicorn.error", "uvicorn.access", "sqlalchemy.engine"]:
|
||||
for name in ["uvicorn", "uvicorn.error", "uvicorn.access"]:
|
||||
logging.getLogger(name).setLevel(level)
|
||||
logging.getLogger("sqlalchemy.engine").setLevel(sql_level)
|
||||
|
||||
+50
-7
@@ -1,5 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
@@ -9,6 +13,11 @@ from app.api.routes import analysis, areas, assistant, datasets, demo, detection
|
||||
from app.core.config import get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.core.logging import configure_logging
|
||||
from app.db.session import SessionLocal
|
||||
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
|
||||
|
||||
|
||||
logger = logging.getLogger("geointel")
|
||||
|
||||
|
||||
def _to_error_payload(
|
||||
@@ -27,13 +36,33 @@ def _to_error_payload(
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
settings = get_settings()
|
||||
configure_logging(settings.log_level)
|
||||
configure_logging(settings.log_level, settings.sql_log_level)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
if settings.reconcile_interrupted_runs_on_startup:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
result = RuntimeReconciliationService.reconcile(db)
|
||||
logger.info(
|
||||
"Runtime reconciliation completed: jobs=%s analysis_runs=%s",
|
||||
result.interrupted_jobs,
|
||||
result.interrupted_analysis_runs,
|
||||
)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Runtime reconciliation failed")
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
yield
|
||||
|
||||
app = FastAPI(
|
||||
title="GeoIntel Kempen",
|
||||
title="GeoIntel",
|
||||
version=settings.app_version,
|
||||
docs_url="/docs",
|
||||
redoc_url="/redoc",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
@@ -60,6 +89,14 @@ def create_app() -> FastAPI:
|
||||
app.include_router(temporal.router, prefix=settings.api_prefix)
|
||||
app.include_router(assistant.router, prefix=settings.api_prefix)
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_identity(request: Request, call_next):
|
||||
request_id = request.headers.get("x-request-id") or str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
response = await call_next(request)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
|
||||
@app.exception_handler(AppError)
|
||||
async def app_error(request: Request, exc: AppError): # noqa: ARG001
|
||||
return JSONResponse(
|
||||
@@ -68,7 +105,7 @@ def create_app() -> FastAPI:
|
||||
exc.code,
|
||||
exc.message,
|
||||
exc.details,
|
||||
request_id=request.headers.get("x-request-id"),
|
||||
request_id=request.state.request_id,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -88,7 +125,7 @@ def create_app() -> FastAPI:
|
||||
code,
|
||||
message,
|
||||
details,
|
||||
request_id=request.headers.get("x-request-id"),
|
||||
request_id=request.state.request_id,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -100,19 +137,25 @@ def create_app() -> FastAPI:
|
||||
"VALIDATION_ERROR",
|
||||
"Validation failed",
|
||||
exc.errors(),
|
||||
request_id=request.headers.get("x-request-id"),
|
||||
request_id=request.state.request_id,
|
||||
),
|
||||
)
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error(request: Request, exc: Exception): # noqa: ARG001
|
||||
async def unexpected_error(request: Request, exc: Exception):
|
||||
logger.exception(
|
||||
"Unhandled request error request_id=%s method=%s path=%s",
|
||||
request.state.request_id,
|
||||
request.method,
|
||||
request.url.path,
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content=_to_error_payload(
|
||||
"INTERNAL_ERROR",
|
||||
"Unexpected server error",
|
||||
{"type": exc.__class__.__name__},
|
||||
request_id=request.headers.get("x-request-id"),
|
||||
request_id=request.state.request_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -23,7 +23,13 @@ class HealthResponse(BaseModel):
|
||||
status: str
|
||||
service: str
|
||||
version: str
|
||||
build_sha: str | None = None
|
||||
build_time: str | None = None
|
||||
database: str | None = None
|
||||
postgis: str | None = None
|
||||
migration: str | None = None
|
||||
storage: str | None = None
|
||||
checks: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SystemCapabilities(BaseModel):
|
||||
@@ -31,7 +37,14 @@ class SystemCapabilities(BaseModel):
|
||||
rasterio: bool
|
||||
geopandas: bool
|
||||
yolo: bool | str
|
||||
yolo_status: str
|
||||
sam: bool | str
|
||||
grb: str
|
||||
sentinel: str
|
||||
version: str
|
||||
build_sha: str | None = None
|
||||
providers: list[ProviderCapability] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SystemCapabilitiesEnvelope(BaseModel):
|
||||
data: SystemCapabilities
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import AnalysisRun, Job
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReconciliationResult:
|
||||
interrupted_jobs: int
|
||||
interrupted_analysis_runs: int
|
||||
|
||||
|
||||
class RuntimeReconciliationService:
|
||||
ERROR_MESSAGE = (
|
||||
"PROCESS_INTERRUPTED: the GeoIntel process restarted before this work "
|
||||
"reached a terminal state"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def reconcile(
|
||||
db: Session,
|
||||
*,
|
||||
finished_at: datetime | None = None,
|
||||
) -> ReconciliationResult:
|
||||
resolved_finished_at = finished_at or datetime.now(timezone.utc)
|
||||
interrupted_jobs = (
|
||||
db.query(Job)
|
||||
.filter(Job.status == "running")
|
||||
.update(
|
||||
{
|
||||
Job.status: "failed",
|
||||
Job.finished_at: resolved_finished_at,
|
||||
Job.error_message: RuntimeReconciliationService.ERROR_MESSAGE,
|
||||
},
|
||||
synchronize_session=False,
|
||||
)
|
||||
)
|
||||
interrupted_analysis_runs = (
|
||||
db.query(AnalysisRun)
|
||||
.filter(AnalysisRun.status == "running")
|
||||
.update(
|
||||
{
|
||||
AnalysisRun.status: "failed",
|
||||
AnalysisRun.finished_at: resolved_finished_at,
|
||||
AnalysisRun.error_message: RuntimeReconciliationService.ERROR_MESSAGE,
|
||||
},
|
||||
synchronize_session=False,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return ReconciliationResult(
|
||||
interrupted_jobs=interrupted_jobs,
|
||||
interrupted_analysis_runs=interrupted_analysis_runs,
|
||||
)
|
||||
@@ -204,9 +204,9 @@ def test_compose_mounts_demo_fixtures_for_backend_runtime() -> None:
|
||||
def test_compose_has_backend_and_frontend_healthchecks() -> None:
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert "http://127.0.0.1:8000/health" in compose
|
||||
assert "http://127.0.0.1:8000/health/ready" in compose
|
||||
assert "urllib.request.urlopen" in compose
|
||||
assert "http://127.0.0.1/health" in compose
|
||||
assert "http://127.0.0.1/health/ready" in compose
|
||||
assert "wget -q -O -" in compose
|
||||
assert "start_period: 30s" in compose
|
||||
assert "start_period: 10s" in compose
|
||||
|
||||
@@ -1,34 +1,100 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api.routes import health
|
||||
from app.main import app
|
||||
|
||||
|
||||
def test_health_endpoint_returns_status_payload() -> None:
|
||||
client = TestClient(app)
|
||||
response = client.get("/health")
|
||||
READY_DATABASE = {
|
||||
"database": "ok",
|
||||
"postgis": "ok:3.4 USE_GEOS=1 USE_PROJ=1",
|
||||
"migration": "ok:202607160001",
|
||||
}
|
||||
|
||||
|
||||
def test_liveness_is_independent_from_database(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
health,
|
||||
"_database_checks",
|
||||
lambda: (_ for _ in ()).throw(AssertionError("must not query DB")),
|
||||
)
|
||||
|
||||
response = TestClient(app).get("/health/live")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "ok"
|
||||
|
||||
|
||||
def test_readiness_returns_ok_only_when_all_checks_pass(monkeypatch) -> None:
|
||||
monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy())
|
||||
monkeypatch.setattr(health, "_storage_check", lambda _: "ok")
|
||||
|
||||
response = TestClient(app).get("/health/ready")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["status"] in {"ok", "degraded"}
|
||||
assert payload["status"] == "ok"
|
||||
assert payload["service"] == "geointel-backend"
|
||||
assert payload["version"] == "0.1.0"
|
||||
assert payload["database"] == "ok"
|
||||
assert payload["postgis"].startswith("ok:")
|
||||
assert payload["migration"] == "ok:202607160001"
|
||||
assert payload["storage"] == "ok"
|
||||
|
||||
|
||||
def test_system_capabilities_reports_gis_dependency_flags(monkeypatch) -> None:
|
||||
def test_compatibility_health_is_fail_closed(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
health,
|
||||
"_database_checks",
|
||||
lambda: {
|
||||
"database": "degraded",
|
||||
"postgis": "degraded",
|
||||
"migration": "degraded",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(health, "_storage_check", lambda _: "ok")
|
||||
|
||||
response = TestClient(app).get("/health")
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json()["status"] == "degraded"
|
||||
|
||||
|
||||
def test_system_capabilities_report_runtime_state(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
health,
|
||||
"_dependency_enabled",
|
||||
lambda module_name: module_name in {"rasterio", "geopandas"},
|
||||
)
|
||||
monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy())
|
||||
monkeypatch.setattr(
|
||||
health.ModelRegistryService,
|
||||
"get_model_capability",
|
||||
lambda *args, **kwargs: SimpleNamespace(
|
||||
configured=True,
|
||||
status="configured",
|
||||
),
|
||||
)
|
||||
|
||||
client = TestClient(app)
|
||||
response = client.get("/api/v1/system/capabilities")
|
||||
response = TestClient(app).get("/api/v1/system/capabilities")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["data"]["rasterio"] is True
|
||||
assert payload["data"]["geopandas"] is True
|
||||
payload = response.json()["data"]
|
||||
assert payload["postgis"] is True
|
||||
assert payload["rasterio"] is True
|
||||
assert payload["geopandas"] is True
|
||||
assert payload["yolo"] is True
|
||||
assert payload["yolo_status"] == "configured"
|
||||
assert payload["version"]
|
||||
|
||||
|
||||
def test_requests_receive_a_correlation_id() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
generated = client.get("/health/live")
|
||||
retained = client.get("/health/live", headers={"x-request-id": "test-request"})
|
||||
|
||||
assert generated.headers["x-request-id"]
|
||||
assert retained.headers["x-request-id"] == "test-request"
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
|
||||
|
||||
def read(name: str) -> str:
|
||||
return (SCRIPTS / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_backup_is_atomic_read_only_and_checksum_bound() -> None:
|
||||
script = read("backup_release_state.sh")
|
||||
|
||||
assert "pg_dump" in script
|
||||
assert "-Fc" in script
|
||||
assert "--no-owner" in script
|
||||
assert "CHECKSUMS.sha256" in script
|
||||
assert "database-password" not in script.lower()
|
||||
assert "mv \"$PARTIAL\" \"$FINAL\"" in script
|
||||
assert "rm -rf -- \"$PARTIAL\"" in script
|
||||
assert "DROP DATABASE" not in script
|
||||
assert "pg_restore --clean" not in script
|
||||
|
||||
|
||||
def test_backup_verification_is_read_only() -> None:
|
||||
script = read("verify_release_backup.sh")
|
||||
|
||||
assert "sha256sum -c CHECKSUMS.sha256" in script
|
||||
assert "pg_restore --list" in script
|
||||
assert "createdb" not in script
|
||||
assert "dropdb" not in script
|
||||
assert "pg_restore --clean" not in script
|
||||
|
||||
|
||||
def test_restore_smoke_is_forced_to_generated_isolated_database() -> None:
|
||||
script = read("restore_release_backup_smoke.sh")
|
||||
|
||||
assert "--confirm-isolated-restore" in script
|
||||
assert "geointel_restore_verify_" in script
|
||||
assert 'if [ "$TARGET_DB" = "$DB_NAME" ]' in script
|
||||
assert "createdb" in script
|
||||
assert "dropdb --if-exists" in script
|
||||
assert "pg_restore \\\n --clean" not in script
|
||||
assert '"production_database_untouched": True' in script
|
||||
|
||||
|
||||
def test_release_safety_scripts_have_valid_bash_syntax() -> None:
|
||||
for name in (
|
||||
"backup_release_state.sh",
|
||||
"verify_release_backup.sh",
|
||||
"restore_release_backup_smoke.sh",
|
||||
):
|
||||
result = subprocess.run(
|
||||
["bash", "-n", f"scripts/{name}"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, f"{name}: {result.stderr}"
|
||||
|
||||
|
||||
def test_readiness_gate_checks_release_safety_scripts() -> None:
|
||||
readiness = read("run_readiness_check.sh")
|
||||
|
||||
for name in (
|
||||
"backup_release_state.sh",
|
||||
"verify_release_backup.sh",
|
||||
"restore_release_backup_smoke.sh",
|
||||
):
|
||||
assert f"bash -n scripts/{name}" in readiness
|
||||
@@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "capture_release_evidence.py"
|
||||
|
||||
|
||||
def load_script():
|
||||
spec = importlib.util.spec_from_file_location("capture_release_evidence", SCRIPT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_release_evidence_manifest_is_secret_free_and_read_only(tmp_path: Path) -> None:
|
||||
module = load_script()
|
||||
args = module.parse_args(
|
||||
[
|
||||
"--output",
|
||||
str(tmp_path / "evidence.json"),
|
||||
"--release-id",
|
||||
"test-rc",
|
||||
]
|
||||
)
|
||||
|
||||
manifest = module.build_manifest(args)
|
||||
|
||||
assert manifest["schema_version"] == 1
|
||||
assert manifest["release_id"] == "test-rc"
|
||||
assert manifest["read_only"] is True
|
||||
assert manifest["scope"] == "Belgium and the Belgian North Sea"
|
||||
assert "DATABASE_URL" not in json.dumps(manifest).replace(
|
||||
'"DATABASE_URL": false',
|
||||
"",
|
||||
).replace(
|
||||
'"DATABASE_URL": true',
|
||||
"",
|
||||
)
|
||||
assert manifest["storage"] == {"requested": False}
|
||||
assert manifest["live"] == {"requested": False}
|
||||
|
||||
|
||||
def test_release_evidence_cli_writes_single_head_manifest(tmp_path: Path) -> None:
|
||||
output = tmp_path / "baseline.json"
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--output",
|
||||
str(output),
|
||||
"--release-id",
|
||||
"test-cli",
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(output.read_text(encoding="utf-8"))
|
||||
assert payload["git"]["commit"]
|
||||
assert payload["migrations"]["single_head"] is True
|
||||
assert payload["files"]["docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md"]["sha256"]
|
||||
assert payload["files"]["docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md"]["sha256"]
|
||||
|
||||
|
||||
def test_readiness_gate_compiles_release_evidence_command() -> None:
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "py_compile scripts/capture_release_evidence.py" in readiness
|
||||
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.models import AnalysisRun, Job
|
||||
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
|
||||
|
||||
|
||||
def test_reconciliation_terminalizes_only_running_work() -> None:
|
||||
db = MagicMock()
|
||||
jobs = MagicMock()
|
||||
runs = MagicMock()
|
||||
jobs.filter.return_value.update.return_value = 5
|
||||
runs.filter.return_value.update.return_value = 2
|
||||
db.query.side_effect = [jobs, runs]
|
||||
finished_at = datetime(2026, 7, 17, 20, 0, tzinfo=timezone.utc)
|
||||
|
||||
result = RuntimeReconciliationService.reconcile(
|
||||
db,
|
||||
finished_at=finished_at,
|
||||
)
|
||||
|
||||
assert result.interrupted_jobs == 5
|
||||
assert result.interrupted_analysis_runs == 2
|
||||
jobs.filter.assert_called_once()
|
||||
runs.filter.assert_called_once()
|
||||
job_values = jobs.filter.return_value.update.call_args.args[0]
|
||||
run_values = runs.filter.return_value.update.call_args.args[0]
|
||||
assert job_values[Job.status] == "failed"
|
||||
assert job_values[Job.finished_at] == finished_at
|
||||
assert "PROCESS_INTERRUPTED" in job_values[Job.error_message]
|
||||
assert run_values[AnalysisRun.status] == "failed"
|
||||
assert run_values[AnalysisRun.finished_at] == finished_at
|
||||
assert "PROCESS_INTERRUPTED" in run_values[AnalysisRun.error_message]
|
||||
db.commit.assert_called_once_with()
|
||||
|
||||
|
||||
def test_startup_reconciliation_is_enabled_only_in_all_in_one_runtime() -> None:
|
||||
start_script = (
|
||||
__import__("pathlib").Path(__file__).resolve().parents[2]
|
||||
/ "deploy"
|
||||
/ "unraid"
|
||||
/ "all-in-one-start.sh"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert (
|
||||
'GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true'
|
||||
in start_script
|
||||
)
|
||||
@@ -99,7 +99,7 @@ def test_unraid_all_in_one_runtime_starts_embedded_postgis_backend_and_nginx() -
|
||||
assert "migrate_compose_volume_if_needed" in dockerman_script
|
||||
assert "docker rm -f geointel" in dockerman_script
|
||||
assert "proxy_pass http://127.0.0.1:8000/api/" in nginx_config
|
||||
assert "proxy_pass http://127.0.0.1:8000/health" in nginx_config
|
||||
assert "proxy_pass http://127.0.0.1:8000/health/ready" in nginx_config
|
||||
assert "location = /geointel-icon.png" in nginx_config
|
||||
assert "frontend/node_modules" in dockerignore
|
||||
assert "storage" in dockerignore
|
||||
|
||||
@@ -153,6 +153,6 @@ VOLUME ["/var/lib/postgresql/data", "/app/storage"]
|
||||
EXPOSE 80
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --retries=10 --start-period=60s \
|
||||
CMD curl -fsS http://127.0.0.1/health >/dev/null || exit 1
|
||||
CMD curl -fsS http://127.0.0.1/health/ready >/dev/null || exit 1
|
||||
|
||||
CMD ["/usr/local/bin/geointel-all-in-one-start"]
|
||||
|
||||
+10
-2
@@ -134,10 +134,12 @@ If multiple model files are present, add `--model-file /mnt/user/appdata/geointe
|
||||
The helper does not download weights, does not load a model and does not run
|
||||
inference; it only updates the env file for the mounted local model.
|
||||
|
||||
Validate:
|
||||
Validate liveness, dependency readiness and the canonical API:
|
||||
|
||||
```bash
|
||||
curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/health"
|
||||
curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/health/live"
|
||||
curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/health/ready"
|
||||
curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/api/v1/system/capabilities"
|
||||
curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/api/v1/projects"
|
||||
curl -I "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/geointel-icon.svg"
|
||||
curl -I "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/geointel-icon.png"
|
||||
@@ -204,6 +206,12 @@ OLLAMA_MAX_OUTPUT_TOKENS=1200
|
||||
OLLAMA_CONTEXT_TOKENS=16384
|
||||
```
|
||||
|
||||
`/health/live` proves only that FastAPI is serving. `/health/ready` returns
|
||||
HTTP 503 when PostgreSQL, PostGIS, the migration head or persistent storage is
|
||||
not ready, and is the container healthcheck. The all-in-one startup also marks
|
||||
work left in `running` by a previous process as failed with
|
||||
`PROCESS_INTERRUPTED`; synchronous work cannot survive a container restart.
|
||||
|
||||
The model dropdown comes from Ollama `/api/tags`, so changing the installed
|
||||
models requires no frontend rebuild. Verify after deployment with:
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ export CORS_ORIGINS="${GEOINTEL_CORS_ORIGINS:-${CORS_ORIGINS:-http://localhost:1
|
||||
export MAX_UPLOAD_MB="${GEOINTEL_MAX_UPLOAD_MB:-${MAX_UPLOAD_MB:-500}}"
|
||||
export YOLO_MODELS_DIR="${YOLO_MODELS_DIR:-/app/models}"
|
||||
export YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-$STORAGE_ROOT/ultralytics}"
|
||||
export GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP="${GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true}"
|
||||
|
||||
mkdir -p "$PGDATA" "$STORAGE_ROOT" "$YOLO_CONFIG_DIR" /run/nginx /var/log/nginx
|
||||
chown -R postgres:postgres "$PGDATA"
|
||||
@@ -66,7 +67,7 @@ import urllib.request
|
||||
last_error = None
|
||||
for attempt in range(1, 61):
|
||||
try:
|
||||
urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=3).read()
|
||||
urllib.request.urlopen("http://127.0.0.1:8000/health/ready", timeout=3).read()
|
||||
print(f"Backend is ready after attempt {attempt}.")
|
||||
break
|
||||
except Exception as exc:
|
||||
|
||||
@@ -46,6 +46,18 @@ server {
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
location = /health/live {
|
||||
proxy_pass http://127.0.0.1:8000/health/live;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
location = /health/ready {
|
||||
proxy_pass http://127.0.0.1:8000/health/ready;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
+2
-2
@@ -127,7 +127,7 @@ services:
|
||||
db:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5).read()\""]
|
||||
test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/ready', timeout=5).read()\""]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
@@ -142,7 +142,7 @@ services:
|
||||
backend:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1/health | grep -q '\"status\"'"]
|
||||
test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1/health/ready | grep -q '\"status\":\"ok\"'"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
|
||||
+48
-28
@@ -41,48 +41,68 @@ Any valid GeoJSON geometry object. V1 primarily expects `Polygon` and `MultiPoly
|
||||
|
||||
## Health
|
||||
|
||||
### GET `/health`
|
||||
### GET `/health/live`
|
||||
|
||||
Returns service status.
|
||||
Returns process liveness only. It never queries PostgreSQL.
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"service": "geointel-backend",
|
||||
"version": "0.1.0"
|
||||
"version": "0.1.0",
|
||||
"build_sha": null,
|
||||
"build_time": null
|
||||
}
|
||||
```
|
||||
|
||||
### GET `/health`
|
||||
|
||||
Backward-compatible alias for dependency readiness.
|
||||
|
||||
### GET `/health/ready`
|
||||
|
||||
Returns dependency readiness. Both readiness routes return HTTP 503 when the
|
||||
database, PostGIS, single migration head or writable storage check is
|
||||
degraded. Docker uses `/health/ready`.
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"service": "geointel-backend",
|
||||
"version": "0.1.0",
|
||||
"database": "ok",
|
||||
"postgis": "ok:3.x",
|
||||
"migration": "ok:202607160001",
|
||||
"storage": "ok",
|
||||
"checks": {
|
||||
"database": "ok",
|
||||
"postgis": "ok:3.x",
|
||||
"migration": "ok:202607160001",
|
||||
"storage": "ok"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### GET `/api/v1/system/capabilities`
|
||||
|
||||
Returns enabled feature flags and tool availability.
|
||||
Returns enabled feature flags and tool availability in the canonical data
|
||||
envelope. PostGIS and configured YOLO state are derived at runtime.
|
||||
|
||||
```json
|
||||
{
|
||||
"postgis": true,
|
||||
"rasterio": true,
|
||||
"geopandas": true,
|
||||
"yolo": false,
|
||||
"sam": false,
|
||||
"grb": "bounded",
|
||||
"sentinel": "planned",
|
||||
"providers": [
|
||||
{
|
||||
"provider_name": "grb",
|
||||
"display_name": "GRB",
|
||||
"authority_level": "authoritative",
|
||||
"supported_layers": ["buildings", "roads", "water", "parcels"],
|
||||
"supported_geometry_types": ["Polygon", "MultiPolygon", "LineString", "MultiLineString"],
|
||||
"supported_query_modes": ["bbox", "persisted_area"],
|
||||
"fetch_signature": "POST /api/v1/projects/{project_id}/datasets/grb/acquire",
|
||||
"configured": true,
|
||||
"status": "configured",
|
||||
"limitation_message": "Alleen expliciet begrensde selecties tot 20 km per zijde worden opgehaald.",
|
||||
"attribution": "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen",
|
||||
"license_note": "Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen.",
|
||||
"not_configured_reason": null
|
||||
}
|
||||
]
|
||||
"data": {
|
||||
"postgis": true,
|
||||
"rasterio": true,
|
||||
"geopandas": true,
|
||||
"yolo": true,
|
||||
"yolo_status": "configured",
|
||||
"sam": false,
|
||||
"grb": "bounded",
|
||||
"sentinel": "planned",
|
||||
"version": "0.1.0",
|
||||
"build_sha": null,
|
||||
"providers": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
+50
-26
@@ -1,38 +1,62 @@
|
||||
# GeoIntel Build Status
|
||||
|
||||
Current preparation milestone: M7 Implementation Control Layer.
|
||||
Updated: 2026-07-17
|
||||
|
||||
## Done
|
||||
## Current state
|
||||
|
||||
- Product blueprint.
|
||||
- Data specifications.
|
||||
- Architecture specifications.
|
||||
- API/database/service documentation.
|
||||
- Codex build plans and prompts.
|
||||
- Operational readiness docs.
|
||||
- Autonomy pack.
|
||||
- M7 build control, regression traps and self-review layer.
|
||||
GeoIntel is an implemented map-first GeoAI workbench running as an all-in-one
|
||||
Unraid container with embedded PostGIS, FastAPI, React/MapLibre, local
|
||||
Ollama integration and optional local YOLO/PyTorch inference.
|
||||
|
||||
## Ready for Codex
|
||||
The active release program is
|
||||
`docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md`.
|
||||
|
||||
Codex can begin with repository verification and backend foundation using the locked build sequence.
|
||||
## Product scope
|
||||
|
||||
## Must Preserve
|
||||
- Target: all Belgian land and the separately labelled Belgian territorial
|
||||
sea, EEZ and continental shelf.
|
||||
- Existing deep regression references: Mol and the Kempen transport region.
|
||||
- Required new golden areas: Wallonia, Brussels, a cross-region area, coast
|
||||
and Belgian North Sea.
|
||||
- Architecture: federated authoritative providers with one coverage matrix;
|
||||
no assumption that GRB, PICC, UrbIS and NGI are semantically identical.
|
||||
|
||||
- GeoIntel is a GeoAI Workbench for the Kempen.
|
||||
- GRB-first reference strategy.
|
||||
- FastAPI + React + PostGIS.
|
||||
- API-driven frontend.
|
||||
- CRS-aware geospatial processing.
|
||||
- Fixture mode must be clearly labeled.
|
||||
## Implemented foundation
|
||||
|
||||
## Known Limitations Before Code Build
|
||||
- Projects, Areas, Datasets, versions, PostGIS vector features and raster
|
||||
artifacts.
|
||||
- Map selection, semantic metrics, historical comparison and exports.
|
||||
- Governed Flemish sources for buildings, context, orthophotos, terrain,
|
||||
flooding, soil, nature, agriculture, population and policy rasters.
|
||||
- Detection persistence, local configured YOLO/PyTorch path, QA/QC and review
|
||||
evidence.
|
||||
- Segmentation persistence foundation without fake production inference.
|
||||
- Local Ollama assistant grounded in persisted evidence.
|
||||
- Single-container Unraid deployment on port 1202.
|
||||
|
||||
- Real GRB WFS integration still needs implementation.
|
||||
- Real YOLO/SAM inference should follow fixture boundary first.
|
||||
- Sentinel and LiDAR remain post-foundation roadmap items.
|
||||
- No production authentication in V1.
|
||||
## Release blockers
|
||||
|
||||
## Next Recommended Codex Pass
|
||||
- Current production backup and isolated restore proof.
|
||||
- Fail-closed readiness and truthful runtime capability reporting.
|
||||
- Stale job/run reconciliation and correlated exception logging.
|
||||
- Temporal compatibility guard for detection QA.
|
||||
- Production secret/configuration parity, immutable images and rollback.
|
||||
- Full CI, reproducible dependencies and supply-chain evidence.
|
||||
- Critical response-model typing and real frontend/browser E2E coverage.
|
||||
- National and maritime scope/providers/golden areas.
|
||||
|
||||
Run `prompts/codex/PASS_00_REPO_AUDIT.md`, then implement backend foundation according to `docs/12-build-control/BUILD_SEQUENCE_LOCK.md`.
|
||||
## Latest evidence
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
python scripts/capture_release_evidence.py \
|
||||
--output storage/release-evidence/rc-current/baseline.json \
|
||||
--release-id rc-belgium-north-sea \
|
||||
--live-base-url http://192.168.10.150:1202
|
||||
```
|
||||
|
||||
The evidence manifest is runtime output and is deliberately ignored by Git.
|
||||
The first captured baseline reported one Alembic head (`202607160001`) and
|
||||
reachable live health/capability routes. Health truthfulness is an open RC-2
|
||||
blocker until readiness becomes fail-closed.
|
||||
|
||||
@@ -1,3 +1,42 @@
|
||||
## Autonomous RC program for Belgium and the Belgian North Sea (2026-07-17)
|
||||
|
||||
- Froze the RC geography as all Belgian land plus the separately labelled
|
||||
territorial sea, EEZ and continental shelf.
|
||||
- Retained Mol and the Kempen as validated regression areas instead of the
|
||||
final product boundary.
|
||||
- Added `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md`.
|
||||
- Added `docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md` with autonomous stop rules,
|
||||
exact phase gates, evidence requirements and RC-0 through RC-11.
|
||||
- Removed the need for a separate RC-12 phase. Fresh-install, upgrade,
|
||||
rollback and runtime evidence remain mandatory inside RC-5 and RC-11.
|
||||
- Verified the official source families for NGI/IGN, Statbel, SPW Wallonia,
|
||||
UrbIS Brussels, federal marine planning, RBINS/BMDC and MDK before freezing
|
||||
the coverage architecture.
|
||||
- Started RC-0 implementation; the release-evidence command is the first code
|
||||
gate.
|
||||
- Implemented and tested `scripts/capture_release_evidence.py`. The first
|
||||
local/live baseline recorded commit `9513f86`, dirty-state evidence, one
|
||||
Alembic head `202607160001`, dependency/configuration checksums and reachable
|
||||
Tower health/capability routes without storing secret values.
|
||||
- Replaced the obsolete pre-implementation `docs/BUILD_STATUS.md` with the
|
||||
current RC state and moved runtime release evidence outside Git.
|
||||
- RC-0 is complete. RC-1 backup/restore safety is active.
|
||||
- Added atomic custom-format PostgreSQL backup, read-only verification and
|
||||
generated-database restore-smoke scripts. Their focused safety/syntax tests
|
||||
pass; Tower backup and isolated restore execution remain the RC-1 live gate.
|
||||
- Implemented RC-2 truthful runtime state: independent liveness, fail-closed
|
||||
DB/PostGIS/Alembic/storage readiness, runtime-derived YOLO capability,
|
||||
request correlation IDs, exception trace logging, SQL logging at WARNING
|
||||
and all-in-one startup reconciliation for orphaned `running` jobs/runs.
|
||||
- Docker, Nginx, API contracts and health documentation now use
|
||||
`/health/ready`; `/health` remains a compatibility readiness alias.
|
||||
- Validation passed: targeted Ruff, backend compile, 963 backend tests,
|
||||
frontend typecheck/build, 122-route contract audit, one Alembic head,
|
||||
complete offline migration SQL and the full readiness gate.
|
||||
- Local Docker CLI is unavailable on the Windows Codex host. Docker config,
|
||||
image build, live health truthfulness, backup and restore are therefore
|
||||
scheduled on the Docker-enabled Tower after push.
|
||||
|
||||
## Sprint 229 - Governed ALZ definitive release promotion (2026-07-17)
|
||||
|
||||
Implemented:
|
||||
|
||||
@@ -5,8 +5,15 @@ Elke runtime-component moet een eenvoudige en machineleesbare healthcheck hebben
|
||||
|
||||
## Backend
|
||||
|
||||
### Endpoint
|
||||
`GET /health`
|
||||
### Liveness
|
||||
|
||||
`GET /health/live` returns HTTP 200 while the FastAPI process can serve a
|
||||
request. It has no database or storage dependency.
|
||||
|
||||
### Readiness
|
||||
|
||||
`GET /health/ready` is the Docker healthcheck. `GET /health` is the
|
||||
backward-compatible alias.
|
||||
|
||||
### Response 200
|
||||
```json
|
||||
@@ -15,8 +22,15 @@ Elke runtime-component moet een eenvoudige en machineleesbare healthcheck hebben
|
||||
"service": "geointel-backend",
|
||||
"version": "0.1.0",
|
||||
"database": "ok",
|
||||
"redis": "ok",
|
||||
"storage": "ok"
|
||||
"postgis": "ok:3.x",
|
||||
"migration": "ok:202607160001",
|
||||
"storage": "ok",
|
||||
"checks": {
|
||||
"database": "ok",
|
||||
"postgis": "ok:3.x",
|
||||
"migration": "ok:202607160001",
|
||||
"storage": "ok"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -25,10 +39,10 @@ Elke runtime-component moet een eenvoudige en machineleesbare healthcheck hebben
|
||||
{
|
||||
"status": "degraded",
|
||||
"service": "geointel-backend",
|
||||
"database": "unavailable",
|
||||
"redis": "ok",
|
||||
"storage": "ok",
|
||||
"errors": ["database connection failed"]
|
||||
"database": "degraded",
|
||||
"postgis": "degraded",
|
||||
"migration": "degraded",
|
||||
"storage": "ok"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -51,7 +65,8 @@ Codex mag kiezen tussen een internal endpoint of CLI-command, maar het contract
|
||||
Frontend moet een `/status` of settings/statuspanel hebben dat toont: backend bereikbaar, API version, auth status, feature flags en laatste healthchecktijd.
|
||||
|
||||
## Storage
|
||||
Storage healthcheck controleert of upload, originals, derived en exports writable zijn.
|
||||
De readinesscheck maakt en verwijdert een tijdelijk bestand in `STORAGE_ROOT`.
|
||||
|
||||
## Database
|
||||
Database healthcheck controleert connectie, PostGIS extension en migrationstatus.
|
||||
De readinesscheck controleert `SELECT 1`, `PostGIS_Version()` en dat de
|
||||
databaseversie exact overeenkomt met de enige Alembic-head in de image.
|
||||
|
||||
@@ -0,0 +1,471 @@
|
||||
# Autonomous RC Roadmap: Belgium and the Belgian North Sea
|
||||
|
||||
## Mission
|
||||
|
||||
Turn the current GeoIntel implementation into a reproducible, demonstrable and
|
||||
release-candidate-quality map-first workbench for all of Belgium and the
|
||||
Belgian North Sea.
|
||||
|
||||
This file is an executable runbook for Codex. It is both the implementation
|
||||
order and the release evidence index. Work continues autonomously between
|
||||
phases. A separate RC-12 phase is deliberately omitted.
|
||||
|
||||
## Sources of truth
|
||||
|
||||
Read in this order before changing behavior:
|
||||
|
||||
1. `AGENTS.md`
|
||||
2. `docs/CODEX_BOOTSTRAP_PROMPT.md`
|
||||
3. `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md`
|
||||
4. `docs/governance/GEOINTEL_CONSTITUTION.md`
|
||||
5. `docs/governance/ARCHITECTURE_INVARIANTS.md`
|
||||
6. `docs/governance/FORBIDDEN_DECISIONS.md`
|
||||
7. `docs/API_CONTRACTS.md`
|
||||
8. `docs/DATABASE_IMPLEMENTATION_PLAN.md`
|
||||
9. `docs/DATA_SPECIFICATION.md`
|
||||
10. `docs/DATA_SOURCES.md`
|
||||
11. `docs/STORAGE_ARCHITECTURE.md`
|
||||
12. `docs/DEFINITION_OF_DONE.md`
|
||||
|
||||
## Autonomous execution contract
|
||||
|
||||
Codex shall:
|
||||
|
||||
- execute phases in order and continue without requesting routine approval;
|
||||
- use existing service, dataset, vector-feature, raster and job boundaries;
|
||||
- keep every provider bounded, fail-closed and provenance-complete;
|
||||
- add or update tests with every behavior change;
|
||||
- update `docs/CODEX_EXECUTION_LOG.md`, `docs/TODO.md` and `CHANGELOG.md`
|
||||
after each completed phase;
|
||||
- run the phase gate and record exact output under
|
||||
`storage/release-evidence/<release-id>/`;
|
||||
- commit and push only after the relevant local gate is green;
|
||||
- deploy only a commit that passed the local gate;
|
||||
- run browser and live API acceptance after deployment;
|
||||
- retain Mol/Kempen as regression references while adding Wallonia, Brussels,
|
||||
cross-region, coast and North Sea golden areas.
|
||||
|
||||
Codex stops only when:
|
||||
|
||||
- an operation would destructively mutate or delete unbacked production data;
|
||||
- an external licence, credential or authority decision cannot be derived from
|
||||
public source metadata;
|
||||
- an official endpoint fails security validation and no authoritative
|
||||
alternative exists;
|
||||
- the same mandatory gate fails three times for the same external reason; or
|
||||
- continuing would require fabricating data, metrics, provenance or model
|
||||
output.
|
||||
|
||||
For a stop, write the blocker, evidence and exact resume command to the
|
||||
execution log. Continue with independent work when possible.
|
||||
|
||||
## Release priorities
|
||||
|
||||
### P0 - release blockers
|
||||
|
||||
- truthful health/readiness and capability reporting;
|
||||
- current backup plus isolated restore proof;
|
||||
- stale job/run reconciliation and useful logs;
|
||||
- temporally correct detection QA;
|
||||
- production secrets, upload limits, configuration parity and rollback;
|
||||
- full CI and reproducible dependencies;
|
||||
- fresh install and upgrade proof.
|
||||
|
||||
### P1 - national product completeness
|
||||
|
||||
- national and maritime scope model;
|
||||
- governed source coverage matrix;
|
||||
- Wallonia, Brussels and North Sea adapters;
|
||||
- nationwide bounded selection and mixed-zone result handling;
|
||||
- historical compatibility matrix;
|
||||
- golden areas outside Flanders.
|
||||
|
||||
### P2 - quality and maintainability
|
||||
|
||||
- typed response models on critical routes;
|
||||
- frontend unit and browser E2E coverage;
|
||||
- loading/accessibility/performance hardening;
|
||||
- retention and operator documentation;
|
||||
- remaining bounded modularization where protected by tests.
|
||||
|
||||
## Global gates
|
||||
|
||||
Run after every behavior-changing phase:
|
||||
|
||||
```bash
|
||||
python -m compileall backend/app
|
||||
cd backend && python -m pytest
|
||||
cd frontend && npm run typecheck
|
||||
cd frontend && npm run build
|
||||
bash scripts/run_readiness_check.sh
|
||||
cd backend && python -m alembic heads
|
||||
cd backend && python -m alembic upgrade head --sql
|
||||
bash -n scripts/live_migration_smoke.sh
|
||||
docker compose config
|
||||
```
|
||||
|
||||
When Tower is reachable:
|
||||
|
||||
```powershell
|
||||
.\scripts\deploy_tower.ps1
|
||||
```
|
||||
|
||||
Then verify:
|
||||
|
||||
```bash
|
||||
curl -fsS http://192.168.10.150:1202/health/live
|
||||
curl -fsS http://192.168.10.150:1202/health/ready
|
||||
curl -fsS http://192.168.10.150:1202/api/v1/system/capabilities
|
||||
```
|
||||
|
||||
The in-app browser acceptance must cover desktop, widescreen and a narrow
|
||||
viewport. It must inspect console errors, failed requests and the complete
|
||||
map-select-analyse-export flow.
|
||||
|
||||
## Official source baseline
|
||||
|
||||
The roadmap starts from these verified official families. Machine endpoints,
|
||||
editions and licences must still pass source-specific probes before activation.
|
||||
|
||||
| Scope | Candidate | Role |
|
||||
| --- | --- | --- |
|
||||
| Belgium | NGI/IGN AdminVector and Top10Vector | authoritative boundary and common topographic baseline |
|
||||
| Belgium | Statbel statistical sectors/population | authoritative statistical geometry and time series |
|
||||
| Flanders | existing Digitaal Vlaanderen/VMM/DOV/INBO/ALZ adapters | detailed authoritative regional layers |
|
||||
| Wallonia | SPW Géoportail, PICC, orthophoto, MNT and thematic services | detailed authoritative regional layers |
|
||||
| Brussels | Paradigm UrbIS and regional environmental geodata | detailed authoritative urban layers |
|
||||
| North Sea | FPS Marine Environment marine spatial plan | legal/use zones |
|
||||
| North Sea | RBINS/BMDC and Marine Atlas | marine environment and observations |
|
||||
| North Sea | MDK/Flemish Hydrography | bathymetry and nautical source candidate |
|
||||
| North Sea | EMODnet | contextual fallback only when explicitly labelled |
|
||||
|
||||
## Phase status
|
||||
|
||||
| Phase | State | Purpose |
|
||||
| --- | --- | --- |
|
||||
| RC-0 | complete | freeze scope and produce release evidence baseline |
|
||||
| RC-1 | in progress | backup, restore and data safety |
|
||||
| RC-2 | verification pending | health, capabilities and stale-runtime correctness |
|
||||
| RC-3 | pending | temporal detection/QA correctness and observability |
|
||||
| RC-4 | pending | national/maritime scope and provider coverage contracts |
|
||||
| RC-5 | pending | deployment, secrets, configuration, fresh install and rollback |
|
||||
| RC-6 | pending | complete CI, dependency and supply-chain gates |
|
||||
| RC-7 | pending | critical API envelope typing and contract validation |
|
||||
| RC-8 | pending | frontend and browser E2E release journeys |
|
||||
| RC-9 | pending | loading, accessibility and performance hardening |
|
||||
| RC-10 | pending | retention, cleanup and national data operations |
|
||||
| RC-11 | pending | final package, upgrade proof, release tag and handoff |
|
||||
|
||||
## RC-0 - Scope freeze and evidence baseline
|
||||
|
||||
**State: complete.** The read-only baseline command and focused tests pass.
|
||||
Runtime evidence is stored outside Git under `storage/release-evidence/`.
|
||||
|
||||
### Work
|
||||
|
||||
- Freeze Belgium plus Belgian North Sea scope.
|
||||
- Record current commit, migration head, dependency versions, container image,
|
||||
database/storage sizes and readiness results.
|
||||
- Create a release-evidence manifest command.
|
||||
- Replace stale product-status wording that still treats Kempen as the product
|
||||
boundary.
|
||||
- Add a coverage matrix schema with `operational`, `partial`,
|
||||
`not_configured`, `unsupported`.
|
||||
|
||||
### Likely files
|
||||
|
||||
- `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md`
|
||||
- `docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md`
|
||||
- `docs/BUILD_STATUS.md`
|
||||
- `docs/DATA_SOURCES.md`
|
||||
- `scripts/capture_release_evidence.py`
|
||||
- `backend/tests/test_rc_release_evidence.py`
|
||||
|
||||
### Evidence and exit
|
||||
|
||||
- scope documents contain Belgium, all three regions and maritime zones;
|
||||
- evidence command is read-only and deterministic;
|
||||
- manifest contains commit, dirty state, Alembic head and gate status;
|
||||
- compile and focused tests pass.
|
||||
|
||||
## RC-1 - Backup, restore and data safety
|
||||
|
||||
### Work
|
||||
|
||||
- Add a safe production backup command for PostgreSQL custom-format dumps.
|
||||
- Capture schema head, PostGIS version, table counts, model checksums and a
|
||||
storage checksum inventory without duplicating all large artifacts.
|
||||
- Reject default/empty production database passwords.
|
||||
- Add backup verification with `pg_restore --list` and checksums.
|
||||
- Add an isolated restore drill using a temporary database/container; never
|
||||
restore over production.
|
||||
- Document retention, encryption boundary and cleanup.
|
||||
|
||||
### Likely files
|
||||
|
||||
- `scripts/backup_release_state.sh`
|
||||
- `scripts/verify_release_backup.sh`
|
||||
- `scripts/restore_release_backup_smoke.sh`
|
||||
- `deploy/unraid/README.md`
|
||||
- `docs/ROLLBACK_AND_RECOVERY.md`
|
||||
- `backend/tests/test_rc_backup_restore_scripts.py`
|
||||
|
||||
### Exit
|
||||
|
||||
- current Tower database has a dated, checksum-verified backup;
|
||||
- isolated restore reaches the recorded Alembic head;
|
||||
- representative project, dataset, geometry and QA counts reconcile;
|
||||
- no production application data was mutated by the drill.
|
||||
|
||||
## RC-2 - Truthful runtime state
|
||||
|
||||
**State: implementation complete, live verification pending.** Local compile,
|
||||
963 backend tests, frontend typecheck/build and the full readiness gate pass.
|
||||
|
||||
### Work
|
||||
|
||||
- Split process liveness from dependency readiness.
|
||||
- Make readiness return HTTP 503 when DB, PostGIS, migration head or writable
|
||||
storage is unavailable.
|
||||
- Derive version/build identity and AI capability from runtime configuration.
|
||||
- Make Docker use the readiness endpoint.
|
||||
- Reconcile orphaned `running` jobs and analysis runs on process startup.
|
||||
- Add request IDs and exception logging without exposing secrets or full SQL.
|
||||
|
||||
### Likely files
|
||||
|
||||
- `backend/app/api/routes/health.py`
|
||||
- `backend/app/schemas/health.py`
|
||||
- `backend/app/core/config.py`
|
||||
- `backend/app/core/logging.py`
|
||||
- `backend/app/main.py`
|
||||
- `backend/app/services/runtime_reconciliation_service.py`
|
||||
- Docker/nginx files and health tests
|
||||
|
||||
### Exit
|
||||
|
||||
- liveness remains 200 while the process runs;
|
||||
- readiness is fail-closed;
|
||||
- live capabilities match the active local YOLO state;
|
||||
- restart converts impossible orphaned work to a terminal failed state with
|
||||
`PROCESS_INTERRUPTED`;
|
||||
- Docker reports unhealthy for a broken database.
|
||||
|
||||
## RC-3 - Temporal QA correctness and observability
|
||||
|
||||
### Work
|
||||
|
||||
- Stop Detection Lab from silently selecting the first raster.
|
||||
- Require explicit raster/model choice.
|
||||
- Prevent current reference QA against historical imagery unless a compatible
|
||||
reference edition/coverage is selected.
|
||||
- Persist source observation period and QA compatibility decision.
|
||||
- Add structured request/job/run logs and operator diagnostics.
|
||||
- Add a stale-runtime report and safe reconciliation command.
|
||||
|
||||
### Exit
|
||||
|
||||
- no detection run starts with an implicit old raster;
|
||||
- incompatible temporal QA fails with a clear code;
|
||||
- compatible current orthophoto/GRB path remains green;
|
||||
- logs correlate request, job, analysis run and quality check.
|
||||
|
||||
## RC-4 - Belgium and North Sea coverage foundation
|
||||
|
||||
### Work
|
||||
|
||||
1. Persist NGI authoritative Belgium land and administrative scopes.
|
||||
2. Persist territorial sea, EEZ and continental shelf as distinct scope
|
||||
layers.
|
||||
3. Implement the coverage matrix and source authority resolver.
|
||||
4. Add bounded provider contracts for:
|
||||
- NGI/Statbel national baseline;
|
||||
- SPW/PICC/orthophoto/relief for Wallonia;
|
||||
- UrbIS/environmental context for Brussels;
|
||||
- federal marine zones and BMDC/MDK marine sources.
|
||||
5. Normalize the public theme vocabulary without discarding provider-native
|
||||
semantics.
|
||||
6. Add Wallonia, Brussels, cross-region, coast and North Sea golden areas.
|
||||
7. Make mixed-zone selection split into bounded provider requests and merge
|
||||
only semantically compatible metrics.
|
||||
|
||||
### Required rules
|
||||
|
||||
- all persistence flows through DatasetService and geospatial services;
|
||||
- no provider writes directly to feature tables;
|
||||
- no country-wide startup import;
|
||||
- no TLS bypass;
|
||||
- no implicit vertical datum conversion;
|
||||
- unsupported metrics remain unavailable.
|
||||
|
||||
### Likely files
|
||||
|
||||
- provider registry/base and new regional provider modules;
|
||||
- geographic scope and source coverage services;
|
||||
- dataset/source schemas;
|
||||
- map source portfolio and scope UI;
|
||||
- source, API, data and storage documentation;
|
||||
- source-specific tests and fixtures.
|
||||
|
||||
### Exit
|
||||
|
||||
- a user can draw within every golden area and receive an honest coverage
|
||||
matrix;
|
||||
- at least the common national baseline is operational on land;
|
||||
- regional detailed layers are operational or explicitly partial;
|
||||
- maritime legal zones are operational;
|
||||
- bathymetry remains gated until a source passes strict acquisition evidence.
|
||||
|
||||
## RC-5 - Production deployment and rollback
|
||||
|
||||
### Work
|
||||
|
||||
- align backend, nginx and proxy upload/time limits;
|
||||
- make every runtime setting intentionally configured, internal or documented;
|
||||
- reject production default secrets;
|
||||
- add image labels with version, commit and build time;
|
||||
- deploy immutable tags in addition to `latest`;
|
||||
- preserve previous image for one-command rollback;
|
||||
- prove fresh install against empty volumes;
|
||||
- prove upgrade from a copied current database/storage state;
|
||||
- prove rollback without schema or data loss.
|
||||
|
||||
### Exit
|
||||
|
||||
- Unraid edit fields cover all operator-owned settings;
|
||||
- fresh install, upgrade and rollback commands are documented and executed;
|
||||
- deployed UI exposes build identity;
|
||||
- previous image remains addressable;
|
||||
- no mutable-only release is accepted.
|
||||
|
||||
## RC-6 - CI and supply-chain gates
|
||||
|
||||
### Work
|
||||
|
||||
- make CI run backend compile/tests, frontend typecheck/build, readiness,
|
||||
Alembic single-head/offline SQL and Docker config/build;
|
||||
- add Python and npm dependency vulnerability checks;
|
||||
- add container scan and SBOM;
|
||||
- introduce reproducible Python resolution/lock output;
|
||||
- preserve optional AI dependency separation;
|
||||
- publish gate artefacts.
|
||||
|
||||
### Exit
|
||||
|
||||
- a broken compile, test, build, migration or contract blocks CI;
|
||||
- dependency and container findings have severity policy;
|
||||
- a clean checkout can reproduce dependency versions and image metadata.
|
||||
|
||||
## RC-7 - Critical API contract hardening
|
||||
|
||||
### Work
|
||||
|
||||
- replace `response_model=dict` first on health, projects, areas, datasets,
|
||||
jobs, analysis runs, QA, exports and map-critical endpoints;
|
||||
- retain canonical `{data: ...}` success and canonical error envelopes;
|
||||
- add OpenAPI response validation tests;
|
||||
- document intentionally streaming/download responses separately.
|
||||
|
||||
### Exit
|
||||
|
||||
- all critical map-first routes have concrete response models;
|
||||
- actual payloads pass FastAPI validation;
|
||||
- API contract audit and frontend typecheck pass.
|
||||
|
||||
## RC-8 - Release journey automation
|
||||
|
||||
### Work
|
||||
|
||||
- add frontend unit tests for selection, coverage, loading and temporal guards;
|
||||
- add browser E2E for:
|
||||
- open Belgium map;
|
||||
- select each golden area;
|
||||
- load a theme;
|
||||
- inspect metric/provenance;
|
||||
- compare compatible history;
|
||||
- run a bounded configured detection path;
|
||||
- export;
|
||||
- ask Ollama with persisted context;
|
||||
- include no-data, partial coverage, provider failure and unsupported metric
|
||||
journeys.
|
||||
|
||||
### Exit
|
||||
|
||||
- E2E runs against a real backend/PostGIS fixture;
|
||||
- no workflow depends only on source-text assertion tests;
|
||||
- console and failed-request audits are clean.
|
||||
|
||||
## RC-9 - UX, accessibility and performance
|
||||
|
||||
### Work
|
||||
|
||||
- replace initial false `missing` source states with explicit loading;
|
||||
- provide accessible names, labels, keyboard focus and status announcements;
|
||||
- enforce desktop, widescreen and narrow layouts;
|
||||
- record API and map-selection budgets for golden areas;
|
||||
- virtualize or page large panels;
|
||||
- split large modules only where tests protect behavior.
|
||||
|
||||
### Exit
|
||||
|
||||
- no core control lacks an accessible name;
|
||||
- loading, empty, partial, error and ready states are distinct;
|
||||
- map selection remains usable at target viewports;
|
||||
- no release journey exceeds its documented budget without a visible warning.
|
||||
|
||||
## RC-10 - Data operations and retention
|
||||
|
||||
### Work
|
||||
|
||||
- define lifecycle for raw, normalized, derived, export, AI and evidence
|
||||
artifacts;
|
||||
- add dry-run-first cleanup for stale technical projects, failed work and
|
||||
superseded caches;
|
||||
- retain immutable official source editions and release evidence;
|
||||
- report disk pressure before acquisition;
|
||||
- schedule nothing implicitly; provide explicit operator commands;
|
||||
- add source freshness reports for national/regional/maritime families.
|
||||
|
||||
### Exit
|
||||
|
||||
- cleanup cannot remove release evidence or current authoritative editions;
|
||||
- every destructive action requires explicit confirmation and a current
|
||||
backup;
|
||||
- storage growth and retained provenance are reportable.
|
||||
|
||||
## RC-11 - Final release package
|
||||
|
||||
### Work
|
||||
|
||||
- rerun fresh-install and upgrade proof using the release image;
|
||||
- rerun backup/isolated restore and rollback proof;
|
||||
- execute every golden area and core browser journey;
|
||||
- generate coverage, security, dependency, migration, performance and known
|
||||
limitation reports;
|
||||
- update operator/end-user documentation;
|
||||
- remove stale status documents from active navigation without deleting
|
||||
historical evidence;
|
||||
- assign semantic version, create signed/checksummed release manifest, tag,
|
||||
push and deploy the immutable image.
|
||||
|
||||
### Final acceptance
|
||||
|
||||
- all P0 findings are closed;
|
||||
- every mandatory global gate is green;
|
||||
- backup and isolated restore are proven;
|
||||
- one Alembic head applies to empty and upgraded PostGIS;
|
||||
- Belgium and maritime golden areas pass;
|
||||
- no unsupported metric or unavailable source appears as successful;
|
||||
- live browser acceptance passes at port 1202;
|
||||
- remaining P1/P2 limitations are explicit, bounded and non-deceptive.
|
||||
|
||||
## Next command
|
||||
|
||||
After creating this roadmap, start RC-0 immediately:
|
||||
|
||||
```bash
|
||||
python scripts/capture_release_evidence.py --output storage/release-evidence/rc-current/baseline.json
|
||||
```
|
||||
|
||||
If the command does not exist yet, implementing it and its tests is the first
|
||||
code task.
|
||||
@@ -0,0 +1,148 @@
|
||||
# RC Scope Freeze: Belgium and the Belgian North Sea
|
||||
|
||||
## Status and precedence
|
||||
|
||||
This document freezes the geographic and release scope for the GeoIntel
|
||||
release-candidate program. It supersedes the geographic limitation in
|
||||
`docs/V1_SCOPE_FREEZE.md`. The existing V1 domain, persistence, provenance,
|
||||
GIS-correctness and no-fake-data rules remain in force.
|
||||
|
||||
Mol and the Kempen remain golden regression areas. They are no longer the
|
||||
product boundary.
|
||||
|
||||
## Product identity
|
||||
|
||||
GeoIntel RC is a map-first geospatial analysis workbench for:
|
||||
|
||||
- the complete land territory of Belgium;
|
||||
- the Belgian territorial sea;
|
||||
- the Belgian exclusive economic zone and continental shelf, labelled
|
||||
according to their legal meaning;
|
||||
- bounded user-drawn areas that may cross municipal, provincial, regional or
|
||||
land/sea boundaries.
|
||||
|
||||
The primary flow remains:
|
||||
|
||||
1. open the map;
|
||||
2. choose one or more understandable themes;
|
||||
3. draw or select an area;
|
||||
4. load only the authoritative or explicitly contextual data available for
|
||||
that area;
|
||||
5. calculate source-appropriate metrics;
|
||||
6. inspect provenance, limitations and time;
|
||||
7. compare compatible historical editions;
|
||||
8. export the result or ask the local assistant about persisted evidence.
|
||||
|
||||
## Geographic federation
|
||||
|
||||
Belgian coverage is a federation of governed adapters. GeoIntel must never
|
||||
pretend that regional datasets have identical semantics merely because they
|
||||
can be displayed together.
|
||||
|
||||
| Zone | Baseline authority | Detailed authoritative families |
|
||||
| --- | --- | --- |
|
||||
| Belgium-wide | NGI/IGN, Statbel, federal geo.be | AdminVector, Top10Vector/CartoWeb, statistical sectors and population |
|
||||
| Flanders | Digitaal Vlaanderen and competent Flemish agencies | GRB, orthophotos, DHMV, VMM/VHA, DOV, INBO, ALZ |
|
||||
| Wallonia | Service public de Wallonie | PICC, SPW orthophotos, relief, hydrography, land cover and thematic catalogues |
|
||||
| Brussels | Brussels Capital Region/Paradigm and competent administrations | UrbIS, BruGIS and Brussels open geodata |
|
||||
| Belgian North Sea | federal Marine Environment, RBINS/BMDC, MDK and legally competent publishers | maritime plan and zones, marine environment, hydrography and validated bathymetry |
|
||||
|
||||
NGI Top10Vector is the preferred common topographic baseline candidate when
|
||||
its machine access, licence, edition and bounded retrieval contract have been
|
||||
validated. Regional large-scale products remain preferred where their
|
||||
authority and semantic detail are higher.
|
||||
|
||||
## Required RC coverage
|
||||
|
||||
The RC must provide:
|
||||
|
||||
- a persisted authoritative Belgium land boundary;
|
||||
- persisted and legally labelled maritime scope boundaries;
|
||||
- area selection anywhere inside the combined Belgium/maritime scope;
|
||||
- a coverage matrix that reports every theme as `operational`, `partial`,
|
||||
`not_configured` or `unsupported` for the selected zone;
|
||||
- source, authority, observation time, publication time, CRS, licence,
|
||||
attribution, coverage and limitation metadata for every result;
|
||||
- bounded acquisition and analysis; no browser-side source fetch and no
|
||||
unbounded country-wide request to a public service;
|
||||
- compatible historical comparison only where editions, units, coverage and
|
||||
methods are comparable;
|
||||
- a stable national theme vocabulary while preserving provider-native fields
|
||||
and semantics.
|
||||
|
||||
The minimum common theme vocabulary is:
|
||||
|
||||
- administrative context;
|
||||
- buildings and built environment;
|
||||
- roads and mobility context;
|
||||
- surface water and hydrography;
|
||||
- land cover and land use;
|
||||
- vegetation, forest and nature;
|
||||
- population and statistical context;
|
||||
- elevation and terrain;
|
||||
- orthophoto/aerial imagery;
|
||||
- flood or climate context where authoritative data exists;
|
||||
- maritime planning, marine environment and bathymetry for the Belgian North
|
||||
Sea where authoritative data exists.
|
||||
|
||||
Not every theme must have the same resolution or historical depth in every
|
||||
zone. Absence or partial coverage must be visible and must not be replaced by
|
||||
fabricated values.
|
||||
|
||||
## Golden release areas
|
||||
|
||||
National support is validated through bounded golden areas, not one monolithic
|
||||
import:
|
||||
|
||||
- Mol municipality: existing deep land and AI regression reference;
|
||||
- Kempen transport region: existing cross-municipality and partition test;
|
||||
- one Walloon urban/rural boundary-crossing area;
|
||||
- one Brussels urban area;
|
||||
- one cross-region or language-boundary area;
|
||||
- one coastal land/sea area;
|
||||
- one Belgian North Sea area containing at least two legal/use zones.
|
||||
|
||||
Exact golden area identifiers and checksums must be stored in release
|
||||
evidence. A source is not nationally operational merely because the Mol test
|
||||
passes.
|
||||
|
||||
## Maritime and bathymetry rules
|
||||
|
||||
- Territorial sea, EEZ and continental shelf are separate legal scopes.
|
||||
- TAW, LAT, mDNG, NAP and product-specific vertical references remain
|
||||
separate dimensions.
|
||||
- No value is transformed between vertical datums without an authoritative,
|
||||
tested transformation and an uncertainty statement.
|
||||
- Bathymetry is bed elevation or depth relative to a stated reference plane.
|
||||
It is not water volume.
|
||||
- Water volume requires compatible bed elevation, water-surface elevation,
|
||||
time, coverage and uncertainty contracts.
|
||||
- WMTS/WMS imagery may provide visual context but is not an analytical raster
|
||||
unless pixel values and georeferencing are governed and validated.
|
||||
|
||||
## Explicit exclusions from this RC
|
||||
|
||||
- automatic whole-country mirroring of every source;
|
||||
- bypassing TLS, licence or source-integrity failures;
|
||||
- claiming semantic parity between GRB, PICC, UrbIS and NGI products;
|
||||
- hidden external downloads at application startup;
|
||||
- browser-direct provider access;
|
||||
- production multi-user authentication;
|
||||
- real-time surveillance or operational maritime navigation;
|
||||
- unsupported water-volume estimates;
|
||||
- a separate RC-12 soak phase.
|
||||
|
||||
Fresh-install, upgrade, rollback and runtime-stability evidence are mandatory
|
||||
but are incorporated into RC-5 and RC-11.
|
||||
|
||||
## Scope change control
|
||||
|
||||
During the RC program, a scope change is allowed only when it:
|
||||
|
||||
1. fixes a release blocker;
|
||||
2. adds missing coverage required by this document;
|
||||
3. corrects source authority, provenance, CRS, time or metric semantics; or
|
||||
4. improves verification without adding unrelated product capability.
|
||||
|
||||
Every accepted change must update the autonomous roadmap, execution log and
|
||||
relevant source/API documentation.
|
||||
@@ -11,3 +11,24 @@ Bestanden worden niet stilzwijgend verwijderd. Gebruik soft-delete waar mogelijk
|
||||
|
||||
## AI/model rollback
|
||||
Elke analysis run bewaart model id, version, parameters, confidence threshold en pipeline version.
|
||||
# RC backup and isolated restore gate
|
||||
|
||||
Before any release migration, credential rotation or image promotion:
|
||||
|
||||
1. run `scripts/backup_release_state.sh` against the running all-in-one
|
||||
container;
|
||||
2. run `scripts/verify_release_backup.sh` read-only;
|
||||
3. run `scripts/restore_release_backup_smoke.sh` with
|
||||
`--confirm-isolated-restore`;
|
||||
4. retain the backup directory and the separate restore-smoke JSON with the
|
||||
release evidence.
|
||||
|
||||
The restore smoke may only create databases whose name starts with
|
||||
`geointel_restore_verify_`. It refuses the production database name, does not
|
||||
use `pg_restore --clean` and drops the temporary database unless an operator
|
||||
explicitly asks to retain it.
|
||||
|
||||
Storage and model files are inventoried rather than copied into the database
|
||||
dump. Release backups must therefore be paired with the persistent storage
|
||||
volume backup policy. Use `--inventory-mode sha256` for final release
|
||||
evidence.
|
||||
|
||||
@@ -1,5 +1,35 @@
|
||||
# GeoIntel TODO
|
||||
|
||||
## Actief RC-programma: Belgie en de Belgische Noordzee
|
||||
|
||||
`docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md` is vanaf 2026-07-17 het enige actieve
|
||||
releasebord. Mol en de Kempen blijven regressiereferenties, maar de
|
||||
productscope is nu heel Belgie plus de juridisch correct gelabelde Belgische
|
||||
maritieme zones.
|
||||
|
||||
- [x] RC-0: Belgische en maritieme scopefreeze vastleggen.
|
||||
- [x] RC-0: autonome roadmap zonder afzonderlijke RC-12 vastleggen.
|
||||
- [x] RC-0: deterministisch release-evidence manifest implementeren en draaien.
|
||||
- [ ] RC-1: actuele productiebackup, verificatie en geisoleerde restore bewijzen.
|
||||
- [ ] RC-2: liveness/readiness/capabilities fail-closed en waarheidsgetrouw maken.
|
||||
- [ ] RC-2: verweesde jobs en analysis runs na een procesherstart verzoenen.
|
||||
- [ ] RC-3: expliciete rasterkeuze en temporeel compatibele detectie-QA afdwingen.
|
||||
- [ ] RC-4: nationale basisdekking, Wallonie, Brussel en Belgische Noordzee via
|
||||
beheerde providers en golden areas operationaliseren.
|
||||
- [ ] RC-5: secrets/configuratie/uploadlimieten/immutable deploy en rollback
|
||||
bewijzen.
|
||||
- [ ] RC-6: volledige CI, dependency-audit, containerscan en SBOM toevoegen.
|
||||
- [ ] RC-7: kritieke API-routes concrete responsemodellen geven.
|
||||
- [ ] RC-8: echte frontend- en browser-E2E-releaseflows toevoegen.
|
||||
- [ ] RC-9: loading, toegankelijkheid, widescreen/mobile en performance afronden.
|
||||
- [ ] RC-10: dataretentie, diskdruk en veilige cleanup operationaliseren.
|
||||
- [ ] RC-11: fresh install, upgrade, rollback, releasepakket, tag en live
|
||||
acceptatie afronden.
|
||||
|
||||
Een aparte RC-12/soakfase wordt niet uitgevoerd. De relevante fresh-install-,
|
||||
upgrade-, rollback- en runtimebewijzen zijn verplicht opgenomen in RC-5 en
|
||||
RC-11.
|
||||
|
||||
## Actuele V1-productstatus
|
||||
|
||||
Dit is het enige actuele afwerkingsbord. De lange sprint- en
|
||||
|
||||
@@ -36,6 +36,18 @@ server {
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
location = /health/live {
|
||||
proxy_pass http://backend:8000/health/live;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
location = /health/ready {
|
||||
proxy_pass http://backend:8000/health/ready;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
@@ -1938,3 +1938,41 @@ docker exec geointel python /app/scripts/probe_mdk_bathymetry.py
|
||||
|
||||
This performs only `GetCapabilities`, keeps strict TLS verification enabled
|
||||
and returns exit code `2` for an honest non-ready source.
|
||||
# Release backup and restore proof
|
||||
|
||||
The release-candidate safety path is host-operated against the running
|
||||
all-in-one container:
|
||||
|
||||
```bash
|
||||
bash scripts/backup_release_state.sh \
|
||||
--container geointel \
|
||||
--output-root /mnt/user/appdata/geointel/backups \
|
||||
--storage-path /mnt/user/appdata/geointel/storage \
|
||||
--models-path /mnt/user/appdata/geointel/models \
|
||||
--inventory-mode sha256
|
||||
```
|
||||
|
||||
The backup is written atomically and contains a PostgreSQL custom-format dump,
|
||||
archive listing, Alembic/PostGIS metadata, critical table counts, optional
|
||||
storage/model inventories and SHA-256 checksums. An empty or known-default
|
||||
database password leaves the release gate failed. For an emergency backup
|
||||
before rotating that password, add `--allow-insecure-password`; the manifest
|
||||
still records the insecure state.
|
||||
|
||||
Verify without changing any database:
|
||||
|
||||
```bash
|
||||
bash scripts/verify_release_backup.sh \
|
||||
--backup-dir /mnt/user/appdata/geointel/backups/<release-id>
|
||||
```
|
||||
|
||||
Prove restoration only in a generated temporary database:
|
||||
|
||||
```bash
|
||||
bash scripts/restore_release_backup_smoke.sh \
|
||||
--backup-dir /mnt/user/appdata/geointel/backups/<release-id> \
|
||||
--confirm-isolated-restore
|
||||
```
|
||||
|
||||
The restore smoke rejects the production database name, compares PostGIS,
|
||||
Alembic and retained table counts, and removes its temporary database.
|
||||
|
||||
@@ -11,6 +11,8 @@ DOCS = ROOT / "docs" / "API_CONTRACTS.md" # docs/API_CONTRACTS.md
|
||||
|
||||
ALLOWED_NON_ENVELOPE_ENDPOINTS = {
|
||||
("GET", "/health"),
|
||||
("GET", "/health/live"),
|
||||
("GET", "/health/ready"),
|
||||
("GET", "/api/v1/exports/{export_id}/download"),
|
||||
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/image"),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
CONTAINER="geointel"
|
||||
OUTPUT_ROOT="backups"
|
||||
RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
STORAGE_PATH=""
|
||||
MODELS_PATH=""
|
||||
INVENTORY_MODE="metadata"
|
||||
ALLOW_INSECURE_PASSWORD="false"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: bash scripts/backup_release_state.sh [options]
|
||||
|
||||
Creates an atomic, read-only release backup from the running all-in-one
|
||||
GeoIntel container. It never deletes or restores application data.
|
||||
|
||||
Options:
|
||||
--container NAME Docker container (default: geointel)
|
||||
--output-root PATH Host backup root (default: backups)
|
||||
--release-id ID Safe backup directory name
|
||||
--storage-path PATH Optional host storage path to inventory
|
||||
--models-path PATH Optional host model path to inventory
|
||||
--inventory-mode metadata|sha256 Hash all inventoried files only with sha256
|
||||
--allow-insecure-password Complete emergency backup despite an
|
||||
empty/default production DB password
|
||||
EOF
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--container) CONTAINER="$2"; shift 2 ;;
|
||||
--output-root) OUTPUT_ROOT="$2"; shift 2 ;;
|
||||
--release-id) RELEASE_ID="$2"; shift 2 ;;
|
||||
--storage-path) STORAGE_PATH="$2"; shift 2 ;;
|
||||
--models-path) MODELS_PATH="$2"; shift 2 ;;
|
||||
--inventory-mode) INVENTORY_MODE="$2"; shift 2 ;;
|
||||
--allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;;
|
||||
--help|-h) usage; exit 0 ;;
|
||||
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if ! [[ "$RELEASE_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]]; then
|
||||
echo "Unsafe release id: $RELEASE_ID" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ "$INVENTORY_MODE" != "metadata" ] && [ "$INVENTORY_MODE" != "sha256" ]; then
|
||||
echo "--inventory-mode must be metadata or sha256" >&2
|
||||
exit 2
|
||||
fi
|
||||
for required in docker python3 sha256sum git; do
|
||||
if ! command -v "$required" >/dev/null 2>&1; then
|
||||
echo "Missing required command: $required" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||
echo "Container '$CONTAINER' is not running." >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
OUTPUT_ROOT="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$OUTPUT_ROOT")"
|
||||
mkdir -p "$OUTPUT_ROOT"
|
||||
PARTIAL="$OUTPUT_ROOT/.${RELEASE_ID}.partial"
|
||||
FINAL="$OUTPUT_ROOT/$RELEASE_ID"
|
||||
if [ -e "$PARTIAL" ] || [ -e "$FINAL" ]; then
|
||||
echo "Backup target already exists: $FINAL" >&2
|
||||
exit 3
|
||||
fi
|
||||
mkdir -p "$PARTIAL"
|
||||
|
||||
cleanup_partial() {
|
||||
if [ -d "$PARTIAL" ]; then
|
||||
rm -rf -- "$PARTIAL"
|
||||
fi
|
||||
}
|
||||
trap cleanup_partial EXIT
|
||||
|
||||
DB_NAME="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"' )"
|
||||
DB_USER="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"' )"
|
||||
if [ -z "$DB_NAME" ] || [ -z "$DB_USER" ]; then
|
||||
echo "Could not resolve database identity from the container." >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
PASSWORD_SECURE="true"
|
||||
if ! docker exec "$CONTAINER" sh -c '
|
||||
password="${POSTGRES_PASSWORD:-${GEOINTEL_POSTGRES_PASSWORD:-}}"
|
||||
test -n "$password" &&
|
||||
test "$password" != "geointel" &&
|
||||
test "$password" != "postgres" &&
|
||||
test "$password" != "password"
|
||||
'; then
|
||||
PASSWORD_SECURE="false"
|
||||
fi
|
||||
|
||||
echo "Creating PostgreSQL custom-format dump..."
|
||||
docker exec "$CONTAINER" pg_dump \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-Fc \
|
||||
--no-owner \
|
||||
--no-privileges > "$PARTIAL/database.dump"
|
||||
test -s "$PARTIAL/database.dump"
|
||||
|
||||
docker exec -i "$CONTAINER" pg_restore --list \
|
||||
< "$PARTIAL/database.dump" \
|
||||
> "$PARTIAL/database.list"
|
||||
test -s "$PARTIAL/database.list"
|
||||
|
||||
IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")"
|
||||
IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")"
|
||||
GIT_COMMIT="$(git rev-parse HEAD)"
|
||||
GIT_DIRTY="false"
|
||||
if [ -n "$(git status --porcelain=v1)" ]; then
|
||||
GIT_DIRTY="true"
|
||||
fi
|
||||
|
||||
docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \
|
||||
-c "SELECT 'alembic_head', version_num FROM alembic_version
|
||||
UNION ALL SELECT 'postgis_version', postgis_version()
|
||||
UNION ALL SELECT 'database_size_bytes', pg_database_size(current_database())::text
|
||||
ORDER BY 1;" > "$PARTIAL/database-metadata.tsv"
|
||||
|
||||
: > "$PARTIAL/table-counts.tsv"
|
||||
for table in projects areas datasets dataset_versions vector_features jobs analysis_runs detections segmentations quality_checks metrics exports; do
|
||||
count="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -At \
|
||||
-c "SELECT count(*) FROM public.${table};")"
|
||||
printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv"
|
||||
done
|
||||
|
||||
inventory_path() {
|
||||
local source_path="$1"
|
||||
local output_path="$2"
|
||||
if [ -z "$source_path" ]; then
|
||||
printf 'not_requested\n' > "$output_path"
|
||||
return
|
||||
fi
|
||||
python3 - "$source_path" "$output_path" "$INVENTORY_MODE" <<'PY'
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
root = pathlib.Path(sys.argv[1]).expanduser().resolve()
|
||||
output = pathlib.Path(sys.argv[2])
|
||||
mode = sys.argv[3]
|
||||
if not root.is_dir():
|
||||
raise SystemExit(f"Inventory root is not a directory: {root}")
|
||||
|
||||
def digest(path: pathlib.Path) -> str:
|
||||
value = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
value.update(chunk)
|
||||
return value.hexdigest()
|
||||
|
||||
with output.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write("relative_path\tsize_bytes\tmtime_ns\tsha256\n")
|
||||
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
continue
|
||||
stat = path.stat()
|
||||
checksum = digest(path) if mode == "sha256" else ""
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if "\t" in relative or "\n" in relative:
|
||||
raise SystemExit(f"Unsupported inventory path: {relative!r}")
|
||||
handle.write(f"{relative}\t{stat.st_size}\t{stat.st_mtime_ns}\t{checksum}\n")
|
||||
PY
|
||||
}
|
||||
|
||||
inventory_path "$STORAGE_PATH" "$PARTIAL/storage-manifest.tsv"
|
||||
inventory_path "$MODELS_PATH" "$PARTIAL/models-manifest.tsv"
|
||||
|
||||
python3 - "$PARTIAL/manifest.json" <<PY
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"release_id": ${RELEASE_ID@Q},
|
||||
"created_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"read_only_source": True,
|
||||
"container": ${CONTAINER@Q},
|
||||
"database_name": ${DB_NAME@Q},
|
||||
"database_user": ${DB_USER@Q},
|
||||
"database_password_secure": ${PASSWORD_SECURE@Q} == "true",
|
||||
"image_id": ${IMAGE_ID@Q},
|
||||
"image_name": ${IMAGE_NAME@Q},
|
||||
"git_commit": ${GIT_COMMIT@Q},
|
||||
"git_dirty": ${GIT_DIRTY@Q} == "true",
|
||||
"inventory_mode": ${INVENTORY_MODE@Q},
|
||||
"storage_inventory_requested": bool(${STORAGE_PATH@Q}),
|
||||
"models_inventory_requested": bool(${MODELS_PATH@Q}),
|
||||
}
|
||||
path = pathlib.Path(__import__("sys").argv[1])
|
||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
PY
|
||||
|
||||
(
|
||||
cd "$PARTIAL"
|
||||
find . -maxdepth 1 -type f ! -name CHECKSUMS.sha256 -printf '%f\n' \
|
||||
| LC_ALL=C sort \
|
||||
| xargs sha256sum > CHECKSUMS.sha256
|
||||
)
|
||||
|
||||
mv "$PARTIAL" "$FINAL"
|
||||
trap - EXIT
|
||||
echo "Release backup created: $FINAL"
|
||||
|
||||
if [ "$PASSWORD_SECURE" != "true" ] && [ "$ALLOW_INSECURE_PASSWORD" != "true" ]; then
|
||||
echo "SECURITY GATE FAILED: production database password is empty or a known default." >&2
|
||||
echo "The emergency backup is valid, but the release remains blocked." >&2
|
||||
exit 4
|
||||
fi
|
||||
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture a read-only, secret-free GeoIntel release evidence manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Sequence
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BACKEND = ROOT / "backend"
|
||||
DEFAULT_HASHED_FILES = (
|
||||
"AGENTS.md",
|
||||
"backend/pyproject.toml",
|
||||
"backend/alembic.ini",
|
||||
"frontend/package.json",
|
||||
"frontend/package-lock.json",
|
||||
"docker-compose.yml",
|
||||
"docker-compose.unraid.yml",
|
||||
"deploy/unraid/Dockerfile.all-in-one",
|
||||
"docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md",
|
||||
"docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md",
|
||||
)
|
||||
DEPENDENCIES = (
|
||||
"alembic",
|
||||
"fastapi",
|
||||
"geoalchemy2",
|
||||
"geopandas",
|
||||
"numpy",
|
||||
"pydantic",
|
||||
"pyproj",
|
||||
"rasterio",
|
||||
"shapely",
|
||||
"sqlalchemy",
|
||||
"torch",
|
||||
"ultralytics",
|
||||
"uvicorn",
|
||||
)
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def run_command(
|
||||
command: Sequence[str],
|
||||
*,
|
||||
cwd: Path = ROOT,
|
||||
timeout_seconds: int = 30,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
list(command),
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"exit_code": None,
|
||||
"stdout": "",
|
||||
"stderr": str(exc),
|
||||
}
|
||||
return {
|
||||
"ok": result.returncode == 0,
|
||||
"exit_code": result.returncode,
|
||||
"stdout": result.stdout.strip(),
|
||||
"stderr": result.stderr.strip(),
|
||||
}
|
||||
|
||||
|
||||
def git_evidence() -> dict[str, Any]:
|
||||
commit = run_command(("git", "rev-parse", "HEAD"))
|
||||
branch = run_command(("git", "branch", "--show-current"))
|
||||
status = run_command(("git", "status", "--porcelain=v1"))
|
||||
remote = run_command(("git", "remote", "get-url", "origin"))
|
||||
dirty_paths = [
|
||||
line[3:].strip()
|
||||
for line in status["stdout"].splitlines()
|
||||
if len(line) >= 4
|
||||
]
|
||||
return {
|
||||
"commit": commit["stdout"] or None,
|
||||
"branch": branch["stdout"] or None,
|
||||
"origin": remote["stdout"] or None,
|
||||
"dirty": bool(dirty_paths),
|
||||
"dirty_paths": dirty_paths,
|
||||
"commands_ok": all(item["ok"] for item in (commit, branch, status)),
|
||||
}
|
||||
|
||||
|
||||
def migration_evidence() -> dict[str, Any]:
|
||||
heads = run_command((sys.executable, "-m", "alembic", "heads"), cwd=BACKEND)
|
||||
head_lines = [
|
||||
line.strip()
|
||||
for line in heads["stdout"].splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
return {
|
||||
"command_ok": heads["ok"],
|
||||
"heads": head_lines,
|
||||
"single_head": heads["ok"] and len(head_lines) == 1,
|
||||
"error": heads["stderr"] or None,
|
||||
}
|
||||
|
||||
|
||||
def dependency_evidence() -> dict[str, str | None]:
|
||||
versions: dict[str, str | None] = {}
|
||||
for name in DEPENDENCIES:
|
||||
try:
|
||||
versions[name] = importlib.metadata.version(name)
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
versions[name] = None
|
||||
return versions
|
||||
|
||||
|
||||
def file_evidence() -> dict[str, dict[str, Any]]:
|
||||
evidence: dict[str, dict[str, Any]] = {}
|
||||
for relative_path in DEFAULT_HASHED_FILES:
|
||||
path = ROOT / relative_path
|
||||
evidence[relative_path] = {
|
||||
"exists": path.is_file(),
|
||||
"size_bytes": path.stat().st_size if path.is_file() else None,
|
||||
"sha256": sha256_file(path) if path.is_file() else None,
|
||||
}
|
||||
return evidence
|
||||
|
||||
|
||||
def storage_evidence(storage_root: Path | None) -> dict[str, Any]:
|
||||
if storage_root is None:
|
||||
return {"requested": False}
|
||||
resolved = storage_root.expanduser().resolve()
|
||||
if not resolved.is_dir():
|
||||
return {
|
||||
"requested": True,
|
||||
"root": str(resolved),
|
||||
"available": False,
|
||||
}
|
||||
total_size = 0
|
||||
file_count = 0
|
||||
errors: list[str] = []
|
||||
for path in resolved.rglob("*"):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
continue
|
||||
try:
|
||||
total_size += path.stat().st_size
|
||||
file_count += 1
|
||||
except OSError as exc:
|
||||
errors.append(f"{path}: {exc}")
|
||||
if len(errors) >= 20:
|
||||
break
|
||||
return {
|
||||
"requested": True,
|
||||
"root": str(resolved),
|
||||
"available": True,
|
||||
"file_count": file_count,
|
||||
"size_bytes": total_size,
|
||||
"scan_complete": not errors,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def fetch_json(url: str, timeout_seconds: int) -> dict[str, Any]:
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
headers={"Accept": "application/json", "User-Agent": "GeoIntel-RC-Evidence/1.0"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
|
||||
body = response.read(2 * 1024 * 1024 + 1)
|
||||
if len(body) > 2 * 1024 * 1024:
|
||||
raise ValueError("response exceeds 2 MiB evidence limit")
|
||||
return {
|
||||
"ok": 200 <= response.status < 300,
|
||||
"status_code": response.status,
|
||||
"payload": json.loads(body.decode("utf-8")),
|
||||
"error": None,
|
||||
}
|
||||
except (urllib.error.URLError, ValueError, json.JSONDecodeError) as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"status_code": getattr(exc, "code", None),
|
||||
"payload": None,
|
||||
"error": str(exc),
|
||||
}
|
||||
|
||||
|
||||
def live_evidence(base_url: str | None, timeout_seconds: int) -> dict[str, Any]:
|
||||
if not base_url:
|
||||
return {"requested": False}
|
||||
normalized = base_url.rstrip("/")
|
||||
return {
|
||||
"requested": True,
|
||||
"base_url": normalized,
|
||||
"health": fetch_json(f"{normalized}/health", timeout_seconds),
|
||||
"capabilities": fetch_json(
|
||||
f"{normalized}/api/v1/system/capabilities",
|
||||
timeout_seconds,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def build_manifest(args: argparse.Namespace) -> dict[str, Any]:
|
||||
storage_root = Path(args.storage_root) if args.storage_root else None
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"release_id": args.release_id,
|
||||
"captured_at": utc_now(),
|
||||
"read_only": True,
|
||||
"scope": "Belgium and the Belgian North Sea",
|
||||
"host": {
|
||||
"platform": platform.platform(),
|
||||
"python": platform.python_version(),
|
||||
"hostname": platform.node(),
|
||||
},
|
||||
"git": git_evidence(),
|
||||
"migrations": migration_evidence(),
|
||||
"dependencies": dependency_evidence(),
|
||||
"files": file_evidence(),
|
||||
"storage": storage_evidence(storage_root),
|
||||
"live": live_evidence(args.live_base_url, args.timeout_seconds),
|
||||
"environment_presence": {
|
||||
"DATABASE_URL": bool(os.environ.get("DATABASE_URL")),
|
||||
"STORAGE_ROOT": bool(os.environ.get("STORAGE_ROOT")),
|
||||
"YOLO_ENABLED": bool(os.environ.get("YOLO_ENABLED")),
|
||||
"YOLO_MODEL_PATH": bool(os.environ.get("YOLO_MODEL_PATH")),
|
||||
"OLLAMA_ENABLED": bool(os.environ.get("OLLAMA_ENABLED")),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--release-id", default="rc-current")
|
||||
parser.add_argument("--live-base-url")
|
||||
parser.add_argument("--storage-root")
|
||||
parser.add_argument("--timeout-seconds", type=int, default=10)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
if args.timeout_seconds < 1 or args.timeout_seconds > 120:
|
||||
raise SystemExit("--timeout-seconds must be between 1 and 120")
|
||||
manifest = build_manifest(args)
|
||||
output = args.output.expanduser().resolve()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"Wrote read-only release evidence to {output}")
|
||||
if not manifest["git"]["commands_ok"]:
|
||||
return 2
|
||||
if not manifest["migrations"]["single_head"]:
|
||||
return 3
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
CONTAINER="geointel"
|
||||
BACKUP_DIR=""
|
||||
CONFIRM="false"
|
||||
KEEP_DATABASE="false"
|
||||
OUTPUT=""
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: bash scripts/restore_release_backup_smoke.sh --backup-dir PATH \
|
||||
--confirm-isolated-restore [options]
|
||||
|
||||
Restores into a generated geointel_restore_verify_* database only, verifies
|
||||
PostGIS, Alembic and retained table counts, then drops that temporary database.
|
||||
It refuses the production database name and never uses pg_restore --clean.
|
||||
|
||||
Options:
|
||||
--container NAME
|
||||
--output PATH Write result JSON outside the immutable backup directory
|
||||
--keep-database Keep the isolated verification database for inspection
|
||||
EOF
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--backup-dir) BACKUP_DIR="$2"; shift 2 ;;
|
||||
--container) CONTAINER="$2"; shift 2 ;;
|
||||
--confirm-isolated-restore) CONFIRM="true"; shift ;;
|
||||
--keep-database) KEEP_DATABASE="true"; shift ;;
|
||||
--output) OUTPUT="$2"; shift 2 ;;
|
||||
--help|-h) usage; exit 0 ;;
|
||||
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$CONFIRM" != "true" ] || [ -z "$BACKUP_DIR" ]; then
|
||||
echo "Explicit --confirm-isolated-restore and --backup-dir are required." >&2
|
||||
exit 2
|
||||
fi
|
||||
for required in docker python3; do
|
||||
command -v "$required" >/dev/null 2>&1 || {
|
||||
echo "Missing required command: $required" >&2
|
||||
exit 2
|
||||
}
|
||||
done
|
||||
BACKUP_DIR="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$BACKUP_DIR")"
|
||||
bash "$(dirname "$0")/verify_release_backup.sh" --backup-dir "$BACKUP_DIR" --container "$CONTAINER"
|
||||
|
||||
DB_NAME="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"' )"
|
||||
DB_USER="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"' )"
|
||||
TARGET_DB="geointel_restore_verify_$(date -u +%Y%m%d%H%M%S)_$$"
|
||||
if [ "$TARGET_DB" = "$DB_NAME" ] || ! [[ "$TARGET_DB" =~ ^geointel_restore_verify_[0-9_]+$ ]]; then
|
||||
echo "Unsafe restore target: $TARGET_DB" >&2
|
||||
exit 3
|
||||
fi
|
||||
if docker exec "$CONTAINER" psql -X -U "$DB_USER" -d postgres -Atqc \
|
||||
"SELECT 1 FROM pg_database WHERE datname = '$TARGET_DB';" | grep -q 1; then
|
||||
echo "Generated restore database already exists." >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
cleanup_database() {
|
||||
if [ "$KEEP_DATABASE" != "true" ]; then
|
||||
docker exec "$CONTAINER" dropdb --if-exists -U "$DB_USER" "$TARGET_DB" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap cleanup_database EXIT
|
||||
|
||||
docker exec "$CONTAINER" createdb -U "$DB_USER" "$TARGET_DB"
|
||||
docker exec -i "$CONTAINER" pg_restore \
|
||||
--exit-on-error \
|
||||
--no-owner \
|
||||
--no-privileges \
|
||||
-U "$DB_USER" \
|
||||
-d "$TARGET_DB" < "$BACKUP_DIR/database.dump"
|
||||
|
||||
POSTGIS_VERSION="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$TARGET_DB" -Atqc \
|
||||
"SELECT postgis_version();")"
|
||||
ALEMBIC_HEAD="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$TARGET_DB" -Atqc \
|
||||
"SELECT version_num FROM alembic_version;")"
|
||||
EXPECTED_HEAD="$(awk -F $'\t' '$1 == "alembic_head" { print $2 }' "$BACKUP_DIR/database-metadata.tsv")"
|
||||
if [ "$ALEMBIC_HEAD" != "$EXPECTED_HEAD" ]; then
|
||||
echo "Restored Alembic head '$ALEMBIC_HEAD' differs from '$EXPECTED_HEAD'." >&2
|
||||
exit 4
|
||||
fi
|
||||
|
||||
while IFS=$'\t' read -r table expected; do
|
||||
[[ "$table" =~ ^[a-z_]+$ ]] || {
|
||||
echo "Unsafe table name in retained counts: $table" >&2
|
||||
exit 4
|
||||
}
|
||||
actual="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$TARGET_DB" -Atqc \
|
||||
"SELECT count(*) FROM public.${table};")"
|
||||
if [ "$actual" != "$expected" ]; then
|
||||
echo "Restored count mismatch for $table: expected $expected, got $actual." >&2
|
||||
exit 4
|
||||
fi
|
||||
done < "$BACKUP_DIR/table-counts.tsv"
|
||||
|
||||
if [ -z "$OUTPUT" ]; then
|
||||
OUTPUT="${BACKUP_DIR%/}-restore-smoke.json"
|
||||
fi
|
||||
python3 - "$OUTPUT" <<PY
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"verified_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"backup_dir": ${BACKUP_DIR@Q},
|
||||
"temporary_database": ${TARGET_DB@Q},
|
||||
"production_database_untouched": True,
|
||||
"temporary_database_retained": ${KEEP_DATABASE@Q} == "true",
|
||||
"alembic_head": ${ALEMBIC_HEAD@Q},
|
||||
"postgis_version": ${POSTGIS_VERSION@Q},
|
||||
"table_counts_match": True,
|
||||
}
|
||||
path = pathlib.Path(__import__("sys").argv[1]).expanduser().resolve()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
PY
|
||||
|
||||
echo "Isolated restore verified: $TARGET_DB"
|
||||
if [ "$KEEP_DATABASE" != "true" ]; then
|
||||
cleanup_database
|
||||
trap - EXIT
|
||||
echo "Temporary verification database removed."
|
||||
else
|
||||
trap - EXIT
|
||||
echo "Temporary verification database retained by explicit request: $TARGET_DB"
|
||||
fi
|
||||
@@ -27,6 +27,11 @@ if [ -z "${PYTHON_BIN}" ]; then
|
||||
fi
|
||||
|
||||
echo "== GeoIntel run readiness check =="
|
||||
|
||||
"$PYTHON_BIN" -m py_compile scripts/capture_release_evidence.py
|
||||
bash -n scripts/backup_release_state.sh
|
||||
bash -n scripts/verify_release_backup.sh
|
||||
bash -n scripts/restore_release_backup_smoke.sh
|
||||
echo "Using Python: ${PYTHON_BIN}"
|
||||
bash scripts/check_repo_structure.sh
|
||||
${PYTHON_BIN} scripts/smoke_docs.py
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
CONTAINER="geointel"
|
||||
BACKUP_DIR=""
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: bash scripts/verify_release_backup.sh --backup-dir PATH [--container NAME]
|
||||
|
||||
Verifies checksums, manifest shape and PostgreSQL archive readability. This
|
||||
command is read-only and never creates, restores or drops a database.
|
||||
EOF
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--backup-dir) BACKUP_DIR="$2"; shift 2 ;;
|
||||
--container) CONTAINER="$2"; shift 2 ;;
|
||||
--help|-h) usage; exit 0 ;;
|
||||
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$BACKUP_DIR" ]; then
|
||||
usage >&2
|
||||
exit 2
|
||||
fi
|
||||
for required in docker python3 sha256sum; do
|
||||
command -v "$required" >/dev/null 2>&1 || {
|
||||
echo "Missing required command: $required" >&2
|
||||
exit 2
|
||||
}
|
||||
done
|
||||
BACKUP_DIR="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$BACKUP_DIR")"
|
||||
for required_file in manifest.json database.dump database.list database-metadata.tsv table-counts.tsv CHECKSUMS.sha256; do
|
||||
test -s "$BACKUP_DIR/$required_file" || {
|
||||
echo "Missing or empty backup artifact: $required_file" >&2
|
||||
exit 3
|
||||
}
|
||||
done
|
||||
|
||||
(
|
||||
cd "$BACKUP_DIR"
|
||||
sha256sum -c CHECKSUMS.sha256
|
||||
)
|
||||
|
||||
python3 - "$BACKUP_DIR/manifest.json" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
payload = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))
|
||||
required = {
|
||||
"schema_version",
|
||||
"release_id",
|
||||
"created_at",
|
||||
"read_only_source",
|
||||
"database_name",
|
||||
"database_user",
|
||||
"image_id",
|
||||
"git_commit",
|
||||
}
|
||||
missing = sorted(required - payload.keys())
|
||||
if missing:
|
||||
raise SystemExit(f"Backup manifest misses: {', '.join(missing)}")
|
||||
if payload["schema_version"] != 1 or payload["read_only_source"] is not True:
|
||||
raise SystemExit("Unsupported or unsafe backup manifest")
|
||||
PY
|
||||
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||
echo "Container '$CONTAINER' is required to run pg_restore --list." >&2
|
||||
exit 3
|
||||
fi
|
||||
TMP_LIST="$(mktemp)"
|
||||
trap 'rm -f -- "$TMP_LIST"' EXIT
|
||||
docker exec -i "$CONTAINER" pg_restore --list \
|
||||
< "$BACKUP_DIR/database.dump" \
|
||||
> "$TMP_LIST"
|
||||
cmp -s "$TMP_LIST" "$BACKUP_DIR/database.list" || {
|
||||
echo "The current pg_restore listing differs from the retained listing." >&2
|
||||
exit 3
|
||||
}
|
||||
grep -q "TABLE DATA" "$TMP_LIST" || {
|
||||
echo "Archive contains no table data entries." >&2
|
||||
exit 3
|
||||
}
|
||||
|
||||
echo "Release backup verified read-only: $BACKUP_DIR"
|
||||
Reference in New Issue
Block a user