Start autonomous Belgium and North Sea RC
This commit is contained in:
@@ -28,6 +28,7 @@ build/
|
|||||||
/storage/models/*
|
/storage/models/*
|
||||||
/storage/operator-data/*
|
/storage/operator-data/*
|
||||||
/storage/operator-evidence/*
|
/storage/operator-evidence/*
|
||||||
|
/storage/release-evidence/*
|
||||||
/storage/previews/*
|
/storage/previews/*
|
||||||
/storage/training/*
|
/storage/training/*
|
||||||
/storage/ultralytics/*
|
/storage/ultralytics/*
|
||||||
|
|||||||
@@ -7,6 +7,27 @@
|
|||||||
|
|
||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## Autonomous Belgium and North Sea RC program (2026-07-17)
|
||||||
|
|
||||||
|
- Expanded the release-candidate geography from Mol/Kempen to all of Belgium
|
||||||
|
and the Belgian North Sea while retaining the existing areas as golden
|
||||||
|
regression references.
|
||||||
|
- Added an executable autonomous RC-0 through RC-11 roadmap and explicit
|
||||||
|
national/maritime scope freeze.
|
||||||
|
- Folded fresh-install, upgrade, rollback and runtime proof into RC-5 and
|
||||||
|
RC-11 instead of creating a separate RC-12 phase.
|
||||||
|
- Added a read-only release-evidence manifest command with Git, migration,
|
||||||
|
dependency, configuration checksum and optional live endpoint evidence.
|
||||||
|
- Replaced the obsolete pre-build status with the current implemented
|
||||||
|
foundation and active release blockers.
|
||||||
|
- Added atomic PostgreSQL release backup, read-only checksum verification and
|
||||||
|
isolated generated-database restore-smoke tooling.
|
||||||
|
- Split health into process liveness and fail-closed readiness covering
|
||||||
|
PostgreSQL, PostGIS, Alembic head and writable storage.
|
||||||
|
- Made system capabilities report real PostGIS and configured local YOLO
|
||||||
|
state, added request IDs and exception logging, reduced SQL engine logging,
|
||||||
|
and terminalized impossible orphaned work after all-in-one restarts.
|
||||||
|
|
||||||
## Sprint 240 Operational forest, agriculture, nature and soil themes (2026-07-17)
|
## Sprint 240 Operational forest, agriculture, nature and soil themes (2026-07-17)
|
||||||
|
|
||||||
- Extended the governed Landgebruik Vlaanderen 2025 raster registry with
|
- Extended the governed Landgebruik Vlaanderen 2025 raster registry with
|
||||||
|
|||||||
+14
-1
@@ -1,6 +1,19 @@
|
|||||||
# GeoIntel Backend (Sprint 3 foundation layer)
|
# GeoIntel Backend (Sprint 3 foundation layer)
|
||||||
|
|
||||||
FastAPI backend for GeoIntel Kempen Foundation Sprints.
|
FastAPI backend for the GeoIntel Belgium and Belgian North Sea workbench.
|
||||||
|
|
||||||
|
Runtime probes:
|
||||||
|
|
||||||
|
- `GET /health/live`: process liveness, always independent from PostgreSQL.
|
||||||
|
- `GET /health/ready`: fail-closed database/PostGIS/migration/storage
|
||||||
|
readiness used by Docker.
|
||||||
|
- `GET /health`: compatibility alias for readiness.
|
||||||
|
- `GET /api/v1/system/capabilities`: runtime-derived PostGIS, GIS dependency,
|
||||||
|
configured YOLO and provider state.
|
||||||
|
|
||||||
|
The all-in-one production runtime enables interrupted job/analysis-run
|
||||||
|
reconciliation at startup. Local tests and development leave it disabled
|
||||||
|
unless `GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP=true`.
|
||||||
|
|
||||||
The map-first explorer uses the existing persisted vector selection endpoint. Its bounded GeoJSON preview reports `feature_count`, while `total_feature_count` reports the exact PostGIS intersection count before the 1,000-feature response cap. When an active `area_id` is supplied, vector, temporal, derived-dataset and export paths all use `bbox ∩ Area`. A full-work-area bbox resolves to the exact persisted Area geometry; a boundary-crossing rectangle is clipped to the official boundary.
|
The map-first explorer uses the existing persisted vector selection endpoint. Its bounded GeoJSON preview reports `feature_count`, while `total_feature_count` reports the exact PostGIS intersection count before the 1,000-feature response cap. When an active `area_id` is supplied, vector, temporal, derived-dataset and export paths all use `bbox ∩ Area`. A full-work-area bbox resolves to the exact persisted Area geometry; a boundary-crossing rectangle is clipped to the official boundary.
|
||||||
|
|
||||||
|
|||||||
@@ -1,12 +1,23 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from importlib import import_module
|
from importlib import import_module
|
||||||
from sqlalchemy import text
|
from pathlib import Path
|
||||||
from fastapi import APIRouter
|
from tempfile import NamedTemporaryFile
|
||||||
|
|
||||||
from app.schemas.health import HealthResponse, SystemCapabilities
|
from alembic.config import Config
|
||||||
from app.providers.registry import list_provider_capabilities
|
from alembic.script import ScriptDirectory
|
||||||
|
from fastapi import APIRouter, Response, status
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
from app.db.session import get_engine
|
from app.db.session import get_engine
|
||||||
|
from app.providers.registry import list_provider_capabilities
|
||||||
|
from app.schemas.health import (
|
||||||
|
HealthResponse,
|
||||||
|
SystemCapabilities,
|
||||||
|
SystemCapabilitiesEnvelope,
|
||||||
|
)
|
||||||
|
from app.services.model_registry_service import ModelRegistryService
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
@@ -19,27 +30,136 @@ def _dependency_enabled(module: str) -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
@router.get("/health")
|
def _expected_migration_heads() -> list[str]:
|
||||||
def readiness() -> HealthResponse:
|
backend_root = Path(__file__).resolve().parents[3]
|
||||||
db_status = "ok"
|
config = Config(str(backend_root / "alembic.ini"))
|
||||||
|
config.set_main_option("script_location", str(backend_root / "alembic"))
|
||||||
|
return list(ScriptDirectory.from_config(config).get_heads())
|
||||||
|
|
||||||
|
|
||||||
|
def _database_checks() -> dict[str, str]:
|
||||||
|
checks = {
|
||||||
|
"database": "degraded",
|
||||||
|
"postgis": "degraded",
|
||||||
|
"migration": "degraded",
|
||||||
|
}
|
||||||
try:
|
try:
|
||||||
with get_engine().connect() as connection:
|
with get_engine().connect() as connection:
|
||||||
connection.execute(text("SELECT 1"))
|
connection.execute(text("SELECT 1"))
|
||||||
|
checks["database"] = "ok"
|
||||||
|
postgis_version = connection.execute(
|
||||||
|
text("SELECT PostGIS_Version()")
|
||||||
|
).scalar_one()
|
||||||
|
checks["postgis"] = f"ok:{postgis_version}"
|
||||||
|
database_head = connection.execute(
|
||||||
|
text("SELECT version_num FROM alembic_version")
|
||||||
|
).scalar_one()
|
||||||
|
expected_heads = _expected_migration_heads()
|
||||||
|
if len(expected_heads) == 1 and database_head == expected_heads[0]:
|
||||||
|
checks["migration"] = f"ok:{database_head}"
|
||||||
|
else:
|
||||||
|
checks["migration"] = (
|
||||||
|
f"degraded:database={database_head};"
|
||||||
|
f"expected={','.join(expected_heads) or 'none'}"
|
||||||
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
db_status = "degraded"
|
return checks
|
||||||
return HealthResponse(status="ok", service="geointel-backend", version="0.1.0", database=db_status)
|
return checks
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/v1/system/capabilities")
|
def _storage_check(storage_root: str) -> str:
|
||||||
def capabilities() -> dict:
|
root = Path(storage_root).expanduser()
|
||||||
|
try:
|
||||||
|
root.mkdir(parents=True, exist_ok=True)
|
||||||
|
with NamedTemporaryFile(
|
||||||
|
prefix=".geointel-readiness-",
|
||||||
|
dir=root,
|
||||||
|
delete=True,
|
||||||
|
) as handle:
|
||||||
|
handle.write(b"ok")
|
||||||
|
handle.flush()
|
||||||
|
return "ok"
|
||||||
|
except OSError:
|
||||||
|
return "degraded"
|
||||||
|
|
||||||
|
|
||||||
|
def _readiness_payload() -> HealthResponse:
|
||||||
|
settings = get_settings()
|
||||||
|
checks = _database_checks()
|
||||||
|
checks["storage"] = _storage_check(settings.storage_root)
|
||||||
|
ready = all(
|
||||||
|
value == "ok" or value.startswith("ok:")
|
||||||
|
for value in checks.values()
|
||||||
|
)
|
||||||
|
return HealthResponse(
|
||||||
|
status="ok" if ready else "degraded",
|
||||||
|
service="geointel-backend",
|
||||||
|
version=settings.app_version,
|
||||||
|
build_sha=settings.build_sha,
|
||||||
|
build_time=settings.build_time,
|
||||||
|
database=checks["database"],
|
||||||
|
postgis=checks["postgis"],
|
||||||
|
migration=checks["migration"],
|
||||||
|
storage=checks["storage"],
|
||||||
|
checks=checks,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/health/live", response_model=HealthResponse)
|
||||||
|
def liveness() -> HealthResponse:
|
||||||
|
settings = get_settings()
|
||||||
|
return HealthResponse(
|
||||||
|
status="ok",
|
||||||
|
service="geointel-backend",
|
||||||
|
version=settings.app_version,
|
||||||
|
build_sha=settings.build_sha,
|
||||||
|
build_time=settings.build_time,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _readiness_response(response: Response) -> HealthResponse:
|
||||||
|
payload = _readiness_payload()
|
||||||
|
if payload.status != "ok":
|
||||||
|
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/health", response_model=HealthResponse)
|
||||||
|
def readiness(response: Response) -> HealthResponse:
|
||||||
|
return _readiness_response(response)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/health/ready", response_model=HealthResponse)
|
||||||
|
def readiness_explicit(response: Response) -> HealthResponse:
|
||||||
|
return _readiness_response(response)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/api/v1/system/capabilities",
|
||||||
|
response_model=SystemCapabilitiesEnvelope,
|
||||||
|
)
|
||||||
|
def capabilities() -> SystemCapabilitiesEnvelope:
|
||||||
|
settings = get_settings()
|
||||||
providers = [item.to_dict() for item in list_provider_capabilities()]
|
providers = [item.to_dict() for item in list_provider_capabilities()]
|
||||||
return {"data": SystemCapabilities(
|
configured_yolo = ModelRegistryService.get_model_capability(
|
||||||
postgis=True,
|
settings.yolo_model_id,
|
||||||
|
settings=settings,
|
||||||
|
)
|
||||||
|
yolo_configured = bool(configured_yolo and configured_yolo.configured)
|
||||||
|
yolo_status = configured_yolo.status if configured_yolo else "not_configured"
|
||||||
|
postgis_ready = _database_checks()["postgis"].startswith("ok:")
|
||||||
|
return SystemCapabilitiesEnvelope(
|
||||||
|
data=SystemCapabilities(
|
||||||
|
postgis=postgis_ready,
|
||||||
rasterio=_dependency_enabled("rasterio"),
|
rasterio=_dependency_enabled("rasterio"),
|
||||||
geopandas=_dependency_enabled("geopandas"),
|
geopandas=_dependency_enabled("geopandas"),
|
||||||
yolo=False,
|
yolo=yolo_configured,
|
||||||
|
yolo_status=yolo_status,
|
||||||
sam=False,
|
sam=False,
|
||||||
grb="bounded",
|
grb="bounded",
|
||||||
sentinel="planned",
|
sentinel="planned",
|
||||||
|
version=settings.app_version,
|
||||||
|
build_sha=settings.build_sha,
|
||||||
providers=providers,
|
providers=providers,
|
||||||
).model_dump()}
|
)
|
||||||
|
)
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ class Settings(BaseSettings):
|
|||||||
|
|
||||||
app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV")
|
app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV")
|
||||||
app_version: str = Field(default="0.1.0")
|
app_version: str = Field(default="0.1.0")
|
||||||
|
build_sha: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_SHA")
|
||||||
|
build_time: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_TIME")
|
||||||
api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX")
|
api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX")
|
||||||
database_url: str = Field(
|
database_url: str = Field(
|
||||||
default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1",
|
default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1",
|
||||||
@@ -231,6 +233,11 @@ class Settings(BaseSettings):
|
|||||||
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
|
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
|
||||||
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
|
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
|
||||||
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
|
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
|
||||||
|
sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL")
|
||||||
|
reconcile_interrupted_runs_on_startup: bool = Field(
|
||||||
|
default=False,
|
||||||
|
validation_alias="GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP",
|
||||||
|
)
|
||||||
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
|
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
|
||||||
yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED")
|
yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED")
|
||||||
yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR")
|
yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR")
|
||||||
|
|||||||
@@ -2,11 +2,13 @@ import logging
|
|||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
|
||||||
def configure_logging(level: str = "INFO") -> None:
|
def configure_logging(level: str = "INFO", sql_level: str = "WARNING") -> None:
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=level,
|
level=level,
|
||||||
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
||||||
stream=sys.stdout,
|
stream=sys.stdout,
|
||||||
|
force=True,
|
||||||
)
|
)
|
||||||
for name in ["uvicorn", "uvicorn.error", "uvicorn.access", "sqlalchemy.engine"]:
|
for name in ["uvicorn", "uvicorn.error", "uvicorn.access"]:
|
||||||
logging.getLogger(name).setLevel(level)
|
logging.getLogger(name).setLevel(level)
|
||||||
|
logging.getLogger("sqlalchemy.engine").setLevel(sql_level)
|
||||||
|
|||||||
+50
-7
@@ -1,5 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
from fastapi import FastAPI, HTTPException, Request
|
from fastapi import FastAPI, HTTPException, Request
|
||||||
from fastapi.exceptions import RequestValidationError
|
from fastapi.exceptions import RequestValidationError
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
@@ -9,6 +13,11 @@ from app.api.routes import analysis, areas, assistant, datasets, demo, detection
|
|||||||
from app.core.config import get_settings
|
from app.core.config import get_settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.core.logging import configure_logging
|
from app.core.logging import configure_logging
|
||||||
|
from app.db.session import SessionLocal
|
||||||
|
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger("geointel")
|
||||||
|
|
||||||
|
|
||||||
def _to_error_payload(
|
def _to_error_payload(
|
||||||
@@ -27,13 +36,33 @@ def _to_error_payload(
|
|||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
configure_logging(settings.log_level)
|
configure_logging(settings.log_level, settings.sql_log_level)
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def lifespan(_: FastAPI):
|
||||||
|
if settings.reconcile_interrupted_runs_on_startup:
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
result = RuntimeReconciliationService.reconcile(db)
|
||||||
|
logger.info(
|
||||||
|
"Runtime reconciliation completed: jobs=%s analysis_runs=%s",
|
||||||
|
result.interrupted_jobs,
|
||||||
|
result.interrupted_analysis_runs,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
db.rollback()
|
||||||
|
logger.exception("Runtime reconciliation failed")
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
yield
|
||||||
|
|
||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
title="GeoIntel Kempen",
|
title="GeoIntel",
|
||||||
version=settings.app_version,
|
version=settings.app_version,
|
||||||
docs_url="/docs",
|
docs_url="/docs",
|
||||||
redoc_url="/redoc",
|
redoc_url="/redoc",
|
||||||
|
lifespan=lifespan,
|
||||||
)
|
)
|
||||||
|
|
||||||
app.add_middleware(
|
app.add_middleware(
|
||||||
@@ -60,6 +89,14 @@ def create_app() -> FastAPI:
|
|||||||
app.include_router(temporal.router, prefix=settings.api_prefix)
|
app.include_router(temporal.router, prefix=settings.api_prefix)
|
||||||
app.include_router(assistant.router, prefix=settings.api_prefix)
|
app.include_router(assistant.router, prefix=settings.api_prefix)
|
||||||
|
|
||||||
|
@app.middleware("http")
|
||||||
|
async def request_identity(request: Request, call_next):
|
||||||
|
request_id = request.headers.get("x-request-id") or str(uuid.uuid4())
|
||||||
|
request.state.request_id = request_id
|
||||||
|
response = await call_next(request)
|
||||||
|
response.headers["x-request-id"] = request_id
|
||||||
|
return response
|
||||||
|
|
||||||
@app.exception_handler(AppError)
|
@app.exception_handler(AppError)
|
||||||
async def app_error(request: Request, exc: AppError): # noqa: ARG001
|
async def app_error(request: Request, exc: AppError): # noqa: ARG001
|
||||||
return JSONResponse(
|
return JSONResponse(
|
||||||
@@ -68,7 +105,7 @@ def create_app() -> FastAPI:
|
|||||||
exc.code,
|
exc.code,
|
||||||
exc.message,
|
exc.message,
|
||||||
exc.details,
|
exc.details,
|
||||||
request_id=request.headers.get("x-request-id"),
|
request_id=request.state.request_id,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -88,7 +125,7 @@ def create_app() -> FastAPI:
|
|||||||
code,
|
code,
|
||||||
message,
|
message,
|
||||||
details,
|
details,
|
||||||
request_id=request.headers.get("x-request-id"),
|
request_id=request.state.request_id,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -100,19 +137,25 @@ def create_app() -> FastAPI:
|
|||||||
"VALIDATION_ERROR",
|
"VALIDATION_ERROR",
|
||||||
"Validation failed",
|
"Validation failed",
|
||||||
exc.errors(),
|
exc.errors(),
|
||||||
request_id=request.headers.get("x-request-id"),
|
request_id=request.state.request_id,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@app.exception_handler(Exception)
|
@app.exception_handler(Exception)
|
||||||
async def unexpected_error(request: Request, exc: Exception): # noqa: ARG001
|
async def unexpected_error(request: Request, exc: Exception):
|
||||||
|
logger.exception(
|
||||||
|
"Unhandled request error request_id=%s method=%s path=%s",
|
||||||
|
request.state.request_id,
|
||||||
|
request.method,
|
||||||
|
request.url.path,
|
||||||
|
)
|
||||||
return JSONResponse(
|
return JSONResponse(
|
||||||
status_code=500,
|
status_code=500,
|
||||||
content=_to_error_payload(
|
content=_to_error_payload(
|
||||||
"INTERNAL_ERROR",
|
"INTERNAL_ERROR",
|
||||||
"Unexpected server error",
|
"Unexpected server error",
|
||||||
{"type": exc.__class__.__name__},
|
{"type": exc.__class__.__name__},
|
||||||
request_id=request.headers.get("x-request-id"),
|
request_id=request.state.request_id,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,13 @@ class HealthResponse(BaseModel):
|
|||||||
status: str
|
status: str
|
||||||
service: str
|
service: str
|
||||||
version: str
|
version: str
|
||||||
|
build_sha: str | None = None
|
||||||
|
build_time: str | None = None
|
||||||
database: str | None = None
|
database: str | None = None
|
||||||
|
postgis: str | None = None
|
||||||
|
migration: str | None = None
|
||||||
|
storage: str | None = None
|
||||||
|
checks: dict[str, str] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
class SystemCapabilities(BaseModel):
|
class SystemCapabilities(BaseModel):
|
||||||
@@ -31,7 +37,14 @@ class SystemCapabilities(BaseModel):
|
|||||||
rasterio: bool
|
rasterio: bool
|
||||||
geopandas: bool
|
geopandas: bool
|
||||||
yolo: bool | str
|
yolo: bool | str
|
||||||
|
yolo_status: str
|
||||||
sam: bool | str
|
sam: bool | str
|
||||||
grb: str
|
grb: str
|
||||||
sentinel: str
|
sentinel: str
|
||||||
|
version: str
|
||||||
|
build_sha: str | None = None
|
||||||
providers: list[ProviderCapability] = Field(default_factory=list)
|
providers: list[ProviderCapability] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class SystemCapabilitiesEnvelope(BaseModel):
|
||||||
|
data: SystemCapabilities
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.models import AnalysisRun, Job
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ReconciliationResult:
|
||||||
|
interrupted_jobs: int
|
||||||
|
interrupted_analysis_runs: int
|
||||||
|
|
||||||
|
|
||||||
|
class RuntimeReconciliationService:
|
||||||
|
ERROR_MESSAGE = (
|
||||||
|
"PROCESS_INTERRUPTED: the GeoIntel process restarted before this work "
|
||||||
|
"reached a terminal state"
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def reconcile(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
finished_at: datetime | None = None,
|
||||||
|
) -> ReconciliationResult:
|
||||||
|
resolved_finished_at = finished_at or datetime.now(timezone.utc)
|
||||||
|
interrupted_jobs = (
|
||||||
|
db.query(Job)
|
||||||
|
.filter(Job.status == "running")
|
||||||
|
.update(
|
||||||
|
{
|
||||||
|
Job.status: "failed",
|
||||||
|
Job.finished_at: resolved_finished_at,
|
||||||
|
Job.error_message: RuntimeReconciliationService.ERROR_MESSAGE,
|
||||||
|
},
|
||||||
|
synchronize_session=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
interrupted_analysis_runs = (
|
||||||
|
db.query(AnalysisRun)
|
||||||
|
.filter(AnalysisRun.status == "running")
|
||||||
|
.update(
|
||||||
|
{
|
||||||
|
AnalysisRun.status: "failed",
|
||||||
|
AnalysisRun.finished_at: resolved_finished_at,
|
||||||
|
AnalysisRun.error_message: RuntimeReconciliationService.ERROR_MESSAGE,
|
||||||
|
},
|
||||||
|
synchronize_session=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
return ReconciliationResult(
|
||||||
|
interrupted_jobs=interrupted_jobs,
|
||||||
|
interrupted_analysis_runs=interrupted_analysis_runs,
|
||||||
|
)
|
||||||
@@ -204,9 +204,9 @@ def test_compose_mounts_demo_fixtures_for_backend_runtime() -> None:
|
|||||||
def test_compose_has_backend_and_frontend_healthchecks() -> None:
|
def test_compose_has_backend_and_frontend_healthchecks() -> None:
|
||||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert "http://127.0.0.1:8000/health" in compose
|
assert "http://127.0.0.1:8000/health/ready" in compose
|
||||||
assert "urllib.request.urlopen" in compose
|
assert "urllib.request.urlopen" in compose
|
||||||
assert "http://127.0.0.1/health" in compose
|
assert "http://127.0.0.1/health/ready" in compose
|
||||||
assert "wget -q -O -" in compose
|
assert "wget -q -O -" in compose
|
||||||
assert "start_period: 30s" in compose
|
assert "start_period: 30s" in compose
|
||||||
assert "start_period: 10s" in compose
|
assert "start_period: 10s" in compose
|
||||||
|
|||||||
@@ -1,34 +1,100 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
from app.api.routes import health
|
from app.api.routes import health
|
||||||
from app.main import app
|
from app.main import app
|
||||||
|
|
||||||
|
|
||||||
def test_health_endpoint_returns_status_payload() -> None:
|
READY_DATABASE = {
|
||||||
client = TestClient(app)
|
"database": "ok",
|
||||||
response = client.get("/health")
|
"postgis": "ok:3.4 USE_GEOS=1 USE_PROJ=1",
|
||||||
|
"migration": "ok:202607160001",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_liveness_is_independent_from_database(monkeypatch) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
health,
|
||||||
|
"_database_checks",
|
||||||
|
lambda: (_ for _ in ()).throw(AssertionError("must not query DB")),
|
||||||
|
)
|
||||||
|
|
||||||
|
response = TestClient(app).get("/health/live")
|
||||||
|
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
|
assert response.json()["status"] == "ok"
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_returns_ok_only_when_all_checks_pass(monkeypatch) -> None:
|
||||||
|
monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy())
|
||||||
|
monkeypatch.setattr(health, "_storage_check", lambda _: "ok")
|
||||||
|
|
||||||
|
response = TestClient(app).get("/health/ready")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
payload = response.json()
|
payload = response.json()
|
||||||
assert payload["status"] in {"ok", "degraded"}
|
assert payload["status"] == "ok"
|
||||||
assert payload["service"] == "geointel-backend"
|
assert payload["service"] == "geointel-backend"
|
||||||
assert payload["version"] == "0.1.0"
|
assert payload["database"] == "ok"
|
||||||
|
assert payload["postgis"].startswith("ok:")
|
||||||
|
assert payload["migration"] == "ok:202607160001"
|
||||||
|
assert payload["storage"] == "ok"
|
||||||
|
|
||||||
|
|
||||||
def test_system_capabilities_reports_gis_dependency_flags(monkeypatch) -> None:
|
def test_compatibility_health_is_fail_closed(monkeypatch) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
health,
|
||||||
|
"_database_checks",
|
||||||
|
lambda: {
|
||||||
|
"database": "degraded",
|
||||||
|
"postgis": "degraded",
|
||||||
|
"migration": "degraded",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(health, "_storage_check", lambda _: "ok")
|
||||||
|
|
||||||
|
response = TestClient(app).get("/health")
|
||||||
|
|
||||||
|
assert response.status_code == 503
|
||||||
|
assert response.json()["status"] == "degraded"
|
||||||
|
|
||||||
|
|
||||||
|
def test_system_capabilities_report_runtime_state(monkeypatch) -> None:
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
health,
|
health,
|
||||||
"_dependency_enabled",
|
"_dependency_enabled",
|
||||||
lambda module_name: module_name in {"rasterio", "geopandas"},
|
lambda module_name: module_name in {"rasterio", "geopandas"},
|
||||||
)
|
)
|
||||||
|
monkeypatch.setattr(health, "_database_checks", lambda: READY_DATABASE.copy())
|
||||||
|
monkeypatch.setattr(
|
||||||
|
health.ModelRegistryService,
|
||||||
|
"get_model_capability",
|
||||||
|
lambda *args, **kwargs: SimpleNamespace(
|
||||||
|
configured=True,
|
||||||
|
status="configured",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
client = TestClient(app)
|
response = TestClient(app).get("/api/v1/system/capabilities")
|
||||||
response = client.get("/api/v1/system/capabilities")
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
payload = response.json()
|
payload = response.json()["data"]
|
||||||
assert payload["data"]["rasterio"] is True
|
assert payload["postgis"] is True
|
||||||
assert payload["data"]["geopandas"] is True
|
assert payload["rasterio"] is True
|
||||||
|
assert payload["geopandas"] is True
|
||||||
|
assert payload["yolo"] is True
|
||||||
|
assert payload["yolo_status"] == "configured"
|
||||||
|
assert payload["version"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_requests_receive_a_correlation_id() -> None:
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
generated = client.get("/health/live")
|
||||||
|
retained = client.get("/health/live", headers={"x-request-id": "test-request"})
|
||||||
|
|
||||||
|
assert generated.headers["x-request-id"]
|
||||||
|
assert retained.headers["x-request-id"] == "test-request"
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
SCRIPTS = ROOT / "scripts"
|
||||||
|
|
||||||
|
|
||||||
|
def read(name: str) -> str:
|
||||||
|
return (SCRIPTS / name).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_backup_is_atomic_read_only_and_checksum_bound() -> None:
|
||||||
|
script = read("backup_release_state.sh")
|
||||||
|
|
||||||
|
assert "pg_dump" in script
|
||||||
|
assert "-Fc" in script
|
||||||
|
assert "--no-owner" in script
|
||||||
|
assert "CHECKSUMS.sha256" in script
|
||||||
|
assert "database-password" not in script.lower()
|
||||||
|
assert "mv \"$PARTIAL\" \"$FINAL\"" in script
|
||||||
|
assert "rm -rf -- \"$PARTIAL\"" in script
|
||||||
|
assert "DROP DATABASE" not in script
|
||||||
|
assert "pg_restore --clean" not in script
|
||||||
|
|
||||||
|
|
||||||
|
def test_backup_verification_is_read_only() -> None:
|
||||||
|
script = read("verify_release_backup.sh")
|
||||||
|
|
||||||
|
assert "sha256sum -c CHECKSUMS.sha256" in script
|
||||||
|
assert "pg_restore --list" in script
|
||||||
|
assert "createdb" not in script
|
||||||
|
assert "dropdb" not in script
|
||||||
|
assert "pg_restore --clean" not in script
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_smoke_is_forced_to_generated_isolated_database() -> None:
|
||||||
|
script = read("restore_release_backup_smoke.sh")
|
||||||
|
|
||||||
|
assert "--confirm-isolated-restore" in script
|
||||||
|
assert "geointel_restore_verify_" in script
|
||||||
|
assert 'if [ "$TARGET_DB" = "$DB_NAME" ]' in script
|
||||||
|
assert "createdb" in script
|
||||||
|
assert "dropdb --if-exists" in script
|
||||||
|
assert "pg_restore \\\n --clean" not in script
|
||||||
|
assert '"production_database_untouched": True' in script
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_safety_scripts_have_valid_bash_syntax() -> None:
|
||||||
|
for name in (
|
||||||
|
"backup_release_state.sh",
|
||||||
|
"verify_release_backup.sh",
|
||||||
|
"restore_release_backup_smoke.sh",
|
||||||
|
):
|
||||||
|
result = subprocess.run(
|
||||||
|
["bash", "-n", f"scripts/{name}"],
|
||||||
|
cwd=ROOT,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"{name}: {result.stderr}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_gate_checks_release_safety_scripts() -> None:
|
||||||
|
readiness = read("run_readiness_check.sh")
|
||||||
|
|
||||||
|
for name in (
|
||||||
|
"backup_release_state.sh",
|
||||||
|
"verify_release_backup.sh",
|
||||||
|
"restore_release_backup_smoke.sh",
|
||||||
|
):
|
||||||
|
assert f"bash -n scripts/{name}" in readiness
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
SCRIPT = ROOT / "scripts" / "capture_release_evidence.py"
|
||||||
|
|
||||||
|
|
||||||
|
def load_script():
|
||||||
|
spec = importlib.util.spec_from_file_location("capture_release_evidence", SCRIPT)
|
||||||
|
assert spec is not None and spec.loader is not None
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_evidence_manifest_is_secret_free_and_read_only(tmp_path: Path) -> None:
|
||||||
|
module = load_script()
|
||||||
|
args = module.parse_args(
|
||||||
|
[
|
||||||
|
"--output",
|
||||||
|
str(tmp_path / "evidence.json"),
|
||||||
|
"--release-id",
|
||||||
|
"test-rc",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
manifest = module.build_manifest(args)
|
||||||
|
|
||||||
|
assert manifest["schema_version"] == 1
|
||||||
|
assert manifest["release_id"] == "test-rc"
|
||||||
|
assert manifest["read_only"] is True
|
||||||
|
assert manifest["scope"] == "Belgium and the Belgian North Sea"
|
||||||
|
assert "DATABASE_URL" not in json.dumps(manifest).replace(
|
||||||
|
'"DATABASE_URL": false',
|
||||||
|
"",
|
||||||
|
).replace(
|
||||||
|
'"DATABASE_URL": true',
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
assert manifest["storage"] == {"requested": False}
|
||||||
|
assert manifest["live"] == {"requested": False}
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_evidence_cli_writes_single_head_manifest(tmp_path: Path) -> None:
|
||||||
|
output = tmp_path / "baseline.json"
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
str(SCRIPT),
|
||||||
|
"--output",
|
||||||
|
str(output),
|
||||||
|
"--release-id",
|
||||||
|
"test-cli",
|
||||||
|
],
|
||||||
|
cwd=ROOT,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
timeout=60,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
payload = json.loads(output.read_text(encoding="utf-8"))
|
||||||
|
assert payload["git"]["commit"]
|
||||||
|
assert payload["migrations"]["single_head"] is True
|
||||||
|
assert payload["files"]["docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md"]["sha256"]
|
||||||
|
assert payload["files"]["docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md"]["sha256"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_gate_compiles_release_evidence_command() -> None:
|
||||||
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "py_compile scripts/capture_release_evidence.py" in readiness
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.models import AnalysisRun, Job
|
||||||
|
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconciliation_terminalizes_only_running_work() -> None:
|
||||||
|
db = MagicMock()
|
||||||
|
jobs = MagicMock()
|
||||||
|
runs = MagicMock()
|
||||||
|
jobs.filter.return_value.update.return_value = 5
|
||||||
|
runs.filter.return_value.update.return_value = 2
|
||||||
|
db.query.side_effect = [jobs, runs]
|
||||||
|
finished_at = datetime(2026, 7, 17, 20, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
result = RuntimeReconciliationService.reconcile(
|
||||||
|
db,
|
||||||
|
finished_at=finished_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.interrupted_jobs == 5
|
||||||
|
assert result.interrupted_analysis_runs == 2
|
||||||
|
jobs.filter.assert_called_once()
|
||||||
|
runs.filter.assert_called_once()
|
||||||
|
job_values = jobs.filter.return_value.update.call_args.args[0]
|
||||||
|
run_values = runs.filter.return_value.update.call_args.args[0]
|
||||||
|
assert job_values[Job.status] == "failed"
|
||||||
|
assert job_values[Job.finished_at] == finished_at
|
||||||
|
assert "PROCESS_INTERRUPTED" in job_values[Job.error_message]
|
||||||
|
assert run_values[AnalysisRun.status] == "failed"
|
||||||
|
assert run_values[AnalysisRun.finished_at] == finished_at
|
||||||
|
assert "PROCESS_INTERRUPTED" in run_values[AnalysisRun.error_message]
|
||||||
|
db.commit.assert_called_once_with()
|
||||||
|
|
||||||
|
|
||||||
|
def test_startup_reconciliation_is_enabled_only_in_all_in_one_runtime() -> None:
|
||||||
|
start_script = (
|
||||||
|
__import__("pathlib").Path(__file__).resolve().parents[2]
|
||||||
|
/ "deploy"
|
||||||
|
/ "unraid"
|
||||||
|
/ "all-in-one-start.sh"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert (
|
||||||
|
'GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true'
|
||||||
|
in start_script
|
||||||
|
)
|
||||||
@@ -99,7 +99,7 @@ def test_unraid_all_in_one_runtime_starts_embedded_postgis_backend_and_nginx() -
|
|||||||
assert "migrate_compose_volume_if_needed" in dockerman_script
|
assert "migrate_compose_volume_if_needed" in dockerman_script
|
||||||
assert "docker rm -f geointel" in dockerman_script
|
assert "docker rm -f geointel" in dockerman_script
|
||||||
assert "proxy_pass http://127.0.0.1:8000/api/" in nginx_config
|
assert "proxy_pass http://127.0.0.1:8000/api/" in nginx_config
|
||||||
assert "proxy_pass http://127.0.0.1:8000/health" in nginx_config
|
assert "proxy_pass http://127.0.0.1:8000/health/ready" in nginx_config
|
||||||
assert "location = /geointel-icon.png" in nginx_config
|
assert "location = /geointel-icon.png" in nginx_config
|
||||||
assert "frontend/node_modules" in dockerignore
|
assert "frontend/node_modules" in dockerignore
|
||||||
assert "storage" in dockerignore
|
assert "storage" in dockerignore
|
||||||
|
|||||||
@@ -153,6 +153,6 @@ VOLUME ["/var/lib/postgresql/data", "/app/storage"]
|
|||||||
EXPOSE 80
|
EXPOSE 80
|
||||||
|
|
||||||
HEALTHCHECK --interval=30s --timeout=5s --retries=10 --start-period=60s \
|
HEALTHCHECK --interval=30s --timeout=5s --retries=10 --start-period=60s \
|
||||||
CMD curl -fsS http://127.0.0.1/health >/dev/null || exit 1
|
CMD curl -fsS http://127.0.0.1/health/ready >/dev/null || exit 1
|
||||||
|
|
||||||
CMD ["/usr/local/bin/geointel-all-in-one-start"]
|
CMD ["/usr/local/bin/geointel-all-in-one-start"]
|
||||||
|
|||||||
+10
-2
@@ -134,10 +134,12 @@ If multiple model files are present, add `--model-file /mnt/user/appdata/geointe
|
|||||||
The helper does not download weights, does not load a model and does not run
|
The helper does not download weights, does not load a model and does not run
|
||||||
inference; it only updates the env file for the mounted local model.
|
inference; it only updates the env file for the mounted local model.
|
||||||
|
|
||||||
Validate:
|
Validate liveness, dependency readiness and the canonical API:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/health"
|
curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/health/live"
|
||||||
|
curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/health/ready"
|
||||||
|
curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/api/v1/system/capabilities"
|
||||||
curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/api/v1/projects"
|
curl -fsS "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/api/v1/projects"
|
||||||
curl -I "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/geointel-icon.svg"
|
curl -I "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/geointel-icon.svg"
|
||||||
curl -I "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/geointel-icon.png"
|
curl -I "http://192.168.10.150:${GEOINTEL_FRONTEND_PORT:-1202}/geointel-icon.png"
|
||||||
@@ -204,6 +206,12 @@ OLLAMA_MAX_OUTPUT_TOKENS=1200
|
|||||||
OLLAMA_CONTEXT_TOKENS=16384
|
OLLAMA_CONTEXT_TOKENS=16384
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`/health/live` proves only that FastAPI is serving. `/health/ready` returns
|
||||||
|
HTTP 503 when PostgreSQL, PostGIS, the migration head or persistent storage is
|
||||||
|
not ready, and is the container healthcheck. The all-in-one startup also marks
|
||||||
|
work left in `running` by a previous process as failed with
|
||||||
|
`PROCESS_INTERRUPTED`; synchronous work cannot survive a container restart.
|
||||||
|
|
||||||
The model dropdown comes from Ollama `/api/tags`, so changing the installed
|
The model dropdown comes from Ollama `/api/tags`, so changing the installed
|
||||||
models requires no frontend rebuild. Verify after deployment with:
|
models requires no frontend rebuild. Verify after deployment with:
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ export CORS_ORIGINS="${GEOINTEL_CORS_ORIGINS:-${CORS_ORIGINS:-http://localhost:1
|
|||||||
export MAX_UPLOAD_MB="${GEOINTEL_MAX_UPLOAD_MB:-${MAX_UPLOAD_MB:-500}}"
|
export MAX_UPLOAD_MB="${GEOINTEL_MAX_UPLOAD_MB:-${MAX_UPLOAD_MB:-500}}"
|
||||||
export YOLO_MODELS_DIR="${YOLO_MODELS_DIR:-/app/models}"
|
export YOLO_MODELS_DIR="${YOLO_MODELS_DIR:-/app/models}"
|
||||||
export YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-$STORAGE_ROOT/ultralytics}"
|
export YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-$STORAGE_ROOT/ultralytics}"
|
||||||
|
export GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP="${GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true}"
|
||||||
|
|
||||||
mkdir -p "$PGDATA" "$STORAGE_ROOT" "$YOLO_CONFIG_DIR" /run/nginx /var/log/nginx
|
mkdir -p "$PGDATA" "$STORAGE_ROOT" "$YOLO_CONFIG_DIR" /run/nginx /var/log/nginx
|
||||||
chown -R postgres:postgres "$PGDATA"
|
chown -R postgres:postgres "$PGDATA"
|
||||||
@@ -66,7 +67,7 @@ import urllib.request
|
|||||||
last_error = None
|
last_error = None
|
||||||
for attempt in range(1, 61):
|
for attempt in range(1, 61):
|
||||||
try:
|
try:
|
||||||
urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=3).read()
|
urllib.request.urlopen("http://127.0.0.1:8000/health/ready", timeout=3).read()
|
||||||
print(f"Backend is ready after attempt {attempt}.")
|
print(f"Backend is ready after attempt {attempt}.")
|
||||||
break
|
break
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
|||||||
@@ -46,6 +46,18 @@ server {
|
|||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
location = /health/live {
|
||||||
|
proxy_pass http://127.0.0.1:8000/health/live;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
}
|
||||||
|
|
||||||
|
location = /health/ready {
|
||||||
|
proxy_pass http://127.0.0.1:8000/health/ready;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
}
|
||||||
|
|
||||||
location / {
|
location / {
|
||||||
try_files $uri $uri/ /index.html;
|
try_files $uri $uri/ /index.html;
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -127,7 +127,7 @@ services:
|
|||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5).read()\""]
|
test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/ready', timeout=5).read()\""]
|
||||||
interval: 10s
|
interval: 10s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 12
|
retries: 12
|
||||||
@@ -142,7 +142,7 @@ services:
|
|||||||
backend:
|
backend:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1/health | grep -q '\"status\"'"]
|
test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1/health/ready | grep -q '\"status\":\"ok\"'"]
|
||||||
interval: 10s
|
interval: 10s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 12
|
retries: 12
|
||||||
|
|||||||
+41
-21
@@ -41,48 +41,68 @@ Any valid GeoJSON geometry object. V1 primarily expects `Polygon` and `MultiPoly
|
|||||||
|
|
||||||
## Health
|
## Health
|
||||||
|
|
||||||
### GET `/health`
|
### GET `/health/live`
|
||||||
|
|
||||||
Returns service status.
|
Returns process liveness only. It never queries PostgreSQL.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
"service": "geointel-backend",
|
"service": "geointel-backend",
|
||||||
"version": "0.1.0"
|
"version": "0.1.0",
|
||||||
|
"build_sha": null,
|
||||||
|
"build_time": null
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### GET `/health`
|
||||||
|
|
||||||
|
Backward-compatible alias for dependency readiness.
|
||||||
|
|
||||||
|
### GET `/health/ready`
|
||||||
|
|
||||||
|
Returns dependency readiness. Both readiness routes return HTTP 503 when the
|
||||||
|
database, PostGIS, single migration head or writable storage check is
|
||||||
|
degraded. Docker uses `/health/ready`.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"service": "geointel-backend",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"database": "ok",
|
||||||
|
"postgis": "ok:3.x",
|
||||||
|
"migration": "ok:202607160001",
|
||||||
|
"storage": "ok",
|
||||||
|
"checks": {
|
||||||
|
"database": "ok",
|
||||||
|
"postgis": "ok:3.x",
|
||||||
|
"migration": "ok:202607160001",
|
||||||
|
"storage": "ok"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### GET `/api/v1/system/capabilities`
|
### GET `/api/v1/system/capabilities`
|
||||||
|
|
||||||
Returns enabled feature flags and tool availability.
|
Returns enabled feature flags and tool availability in the canonical data
|
||||||
|
envelope. PostGIS and configured YOLO state are derived at runtime.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
|
"data": {
|
||||||
"postgis": true,
|
"postgis": true,
|
||||||
"rasterio": true,
|
"rasterio": true,
|
||||||
"geopandas": true,
|
"geopandas": true,
|
||||||
"yolo": false,
|
"yolo": true,
|
||||||
|
"yolo_status": "configured",
|
||||||
"sam": false,
|
"sam": false,
|
||||||
"grb": "bounded",
|
"grb": "bounded",
|
||||||
"sentinel": "planned",
|
"sentinel": "planned",
|
||||||
"providers": [
|
"version": "0.1.0",
|
||||||
{
|
"build_sha": null,
|
||||||
"provider_name": "grb",
|
"providers": []
|
||||||
"display_name": "GRB",
|
|
||||||
"authority_level": "authoritative",
|
|
||||||
"supported_layers": ["buildings", "roads", "water", "parcels"],
|
|
||||||
"supported_geometry_types": ["Polygon", "MultiPolygon", "LineString", "MultiLineString"],
|
|
||||||
"supported_query_modes": ["bbox", "persisted_area"],
|
|
||||||
"fetch_signature": "POST /api/v1/projects/{project_id}/datasets/grb/acquire",
|
|
||||||
"configured": true,
|
|
||||||
"status": "configured",
|
|
||||||
"limitation_message": "Alleen expliciet begrensde selecties tot 20 km per zijde worden opgehaald.",
|
|
||||||
"attribution": "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen",
|
|
||||||
"license_note": "Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen.",
|
|
||||||
"not_configured_reason": null
|
|
||||||
}
|
}
|
||||||
]
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
+50
-26
@@ -1,38 +1,62 @@
|
|||||||
# GeoIntel Build Status
|
# GeoIntel Build Status
|
||||||
|
|
||||||
Current preparation milestone: M7 Implementation Control Layer.
|
Updated: 2026-07-17
|
||||||
|
|
||||||
## Done
|
## Current state
|
||||||
|
|
||||||
- Product blueprint.
|
GeoIntel is an implemented map-first GeoAI workbench running as an all-in-one
|
||||||
- Data specifications.
|
Unraid container with embedded PostGIS, FastAPI, React/MapLibre, local
|
||||||
- Architecture specifications.
|
Ollama integration and optional local YOLO/PyTorch inference.
|
||||||
- API/database/service documentation.
|
|
||||||
- Codex build plans and prompts.
|
|
||||||
- Operational readiness docs.
|
|
||||||
- Autonomy pack.
|
|
||||||
- M7 build control, regression traps and self-review layer.
|
|
||||||
|
|
||||||
## Ready for Codex
|
The active release program is
|
||||||
|
`docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md`.
|
||||||
|
|
||||||
Codex can begin with repository verification and backend foundation using the locked build sequence.
|
## Product scope
|
||||||
|
|
||||||
## Must Preserve
|
- Target: all Belgian land and the separately labelled Belgian territorial
|
||||||
|
sea, EEZ and continental shelf.
|
||||||
|
- Existing deep regression references: Mol and the Kempen transport region.
|
||||||
|
- Required new golden areas: Wallonia, Brussels, a cross-region area, coast
|
||||||
|
and Belgian North Sea.
|
||||||
|
- Architecture: federated authoritative providers with one coverage matrix;
|
||||||
|
no assumption that GRB, PICC, UrbIS and NGI are semantically identical.
|
||||||
|
|
||||||
- GeoIntel is a GeoAI Workbench for the Kempen.
|
## Implemented foundation
|
||||||
- GRB-first reference strategy.
|
|
||||||
- FastAPI + React + PostGIS.
|
|
||||||
- API-driven frontend.
|
|
||||||
- CRS-aware geospatial processing.
|
|
||||||
- Fixture mode must be clearly labeled.
|
|
||||||
|
|
||||||
## Known Limitations Before Code Build
|
- Projects, Areas, Datasets, versions, PostGIS vector features and raster
|
||||||
|
artifacts.
|
||||||
|
- Map selection, semantic metrics, historical comparison and exports.
|
||||||
|
- Governed Flemish sources for buildings, context, orthophotos, terrain,
|
||||||
|
flooding, soil, nature, agriculture, population and policy rasters.
|
||||||
|
- Detection persistence, local configured YOLO/PyTorch path, QA/QC and review
|
||||||
|
evidence.
|
||||||
|
- Segmentation persistence foundation without fake production inference.
|
||||||
|
- Local Ollama assistant grounded in persisted evidence.
|
||||||
|
- Single-container Unraid deployment on port 1202.
|
||||||
|
|
||||||
- Real GRB WFS integration still needs implementation.
|
## Release blockers
|
||||||
- Real YOLO/SAM inference should follow fixture boundary first.
|
|
||||||
- Sentinel and LiDAR remain post-foundation roadmap items.
|
|
||||||
- No production authentication in V1.
|
|
||||||
|
|
||||||
## Next Recommended Codex Pass
|
- Current production backup and isolated restore proof.
|
||||||
|
- Fail-closed readiness and truthful runtime capability reporting.
|
||||||
|
- Stale job/run reconciliation and correlated exception logging.
|
||||||
|
- Temporal compatibility guard for detection QA.
|
||||||
|
- Production secret/configuration parity, immutable images and rollback.
|
||||||
|
- Full CI, reproducible dependencies and supply-chain evidence.
|
||||||
|
- Critical response-model typing and real frontend/browser E2E coverage.
|
||||||
|
- National and maritime scope/providers/golden areas.
|
||||||
|
|
||||||
Run `prompts/codex/PASS_00_REPO_AUDIT.md`, then implement backend foundation according to `docs/12-build-control/BUILD_SEQUENCE_LOCK.md`.
|
## Latest evidence
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python scripts/capture_release_evidence.py \
|
||||||
|
--output storage/release-evidence/rc-current/baseline.json \
|
||||||
|
--release-id rc-belgium-north-sea \
|
||||||
|
--live-base-url http://192.168.10.150:1202
|
||||||
|
```
|
||||||
|
|
||||||
|
The evidence manifest is runtime output and is deliberately ignored by Git.
|
||||||
|
The first captured baseline reported one Alembic head (`202607160001`) and
|
||||||
|
reachable live health/capability routes. Health truthfulness is an open RC-2
|
||||||
|
blocker until readiness becomes fail-closed.
|
||||||
|
|||||||
@@ -1,3 +1,42 @@
|
|||||||
|
## Autonomous RC program for Belgium and the Belgian North Sea (2026-07-17)
|
||||||
|
|
||||||
|
- Froze the RC geography as all Belgian land plus the separately labelled
|
||||||
|
territorial sea, EEZ and continental shelf.
|
||||||
|
- Retained Mol and the Kempen as validated regression areas instead of the
|
||||||
|
final product boundary.
|
||||||
|
- Added `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md`.
|
||||||
|
- Added `docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md` with autonomous stop rules,
|
||||||
|
exact phase gates, evidence requirements and RC-0 through RC-11.
|
||||||
|
- Removed the need for a separate RC-12 phase. Fresh-install, upgrade,
|
||||||
|
rollback and runtime evidence remain mandatory inside RC-5 and RC-11.
|
||||||
|
- Verified the official source families for NGI/IGN, Statbel, SPW Wallonia,
|
||||||
|
UrbIS Brussels, federal marine planning, RBINS/BMDC and MDK before freezing
|
||||||
|
the coverage architecture.
|
||||||
|
- Started RC-0 implementation; the release-evidence command is the first code
|
||||||
|
gate.
|
||||||
|
- Implemented and tested `scripts/capture_release_evidence.py`. The first
|
||||||
|
local/live baseline recorded commit `9513f86`, dirty-state evidence, one
|
||||||
|
Alembic head `202607160001`, dependency/configuration checksums and reachable
|
||||||
|
Tower health/capability routes without storing secret values.
|
||||||
|
- Replaced the obsolete pre-implementation `docs/BUILD_STATUS.md` with the
|
||||||
|
current RC state and moved runtime release evidence outside Git.
|
||||||
|
- RC-0 is complete. RC-1 backup/restore safety is active.
|
||||||
|
- Added atomic custom-format PostgreSQL backup, read-only verification and
|
||||||
|
generated-database restore-smoke scripts. Their focused safety/syntax tests
|
||||||
|
pass; Tower backup and isolated restore execution remain the RC-1 live gate.
|
||||||
|
- Implemented RC-2 truthful runtime state: independent liveness, fail-closed
|
||||||
|
DB/PostGIS/Alembic/storage readiness, runtime-derived YOLO capability,
|
||||||
|
request correlation IDs, exception trace logging, SQL logging at WARNING
|
||||||
|
and all-in-one startup reconciliation for orphaned `running` jobs/runs.
|
||||||
|
- Docker, Nginx, API contracts and health documentation now use
|
||||||
|
`/health/ready`; `/health` remains a compatibility readiness alias.
|
||||||
|
- Validation passed: targeted Ruff, backend compile, 963 backend tests,
|
||||||
|
frontend typecheck/build, 122-route contract audit, one Alembic head,
|
||||||
|
complete offline migration SQL and the full readiness gate.
|
||||||
|
- Local Docker CLI is unavailable on the Windows Codex host. Docker config,
|
||||||
|
image build, live health truthfulness, backup and restore are therefore
|
||||||
|
scheduled on the Docker-enabled Tower after push.
|
||||||
|
|
||||||
## Sprint 229 - Governed ALZ definitive release promotion (2026-07-17)
|
## Sprint 229 - Governed ALZ definitive release promotion (2026-07-17)
|
||||||
|
|
||||||
Implemented:
|
Implemented:
|
||||||
|
|||||||
@@ -5,8 +5,15 @@ Elke runtime-component moet een eenvoudige en machineleesbare healthcheck hebben
|
|||||||
|
|
||||||
## Backend
|
## Backend
|
||||||
|
|
||||||
### Endpoint
|
### Liveness
|
||||||
`GET /health`
|
|
||||||
|
`GET /health/live` returns HTTP 200 while the FastAPI process can serve a
|
||||||
|
request. It has no database or storage dependency.
|
||||||
|
|
||||||
|
### Readiness
|
||||||
|
|
||||||
|
`GET /health/ready` is the Docker healthcheck. `GET /health` is the
|
||||||
|
backward-compatible alias.
|
||||||
|
|
||||||
### Response 200
|
### Response 200
|
||||||
```json
|
```json
|
||||||
@@ -15,8 +22,15 @@ Elke runtime-component moet een eenvoudige en machineleesbare healthcheck hebben
|
|||||||
"service": "geointel-backend",
|
"service": "geointel-backend",
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
"database": "ok",
|
"database": "ok",
|
||||||
"redis": "ok",
|
"postgis": "ok:3.x",
|
||||||
|
"migration": "ok:202607160001",
|
||||||
|
"storage": "ok",
|
||||||
|
"checks": {
|
||||||
|
"database": "ok",
|
||||||
|
"postgis": "ok:3.x",
|
||||||
|
"migration": "ok:202607160001",
|
||||||
"storage": "ok"
|
"storage": "ok"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -25,10 +39,10 @@ Elke runtime-component moet een eenvoudige en machineleesbare healthcheck hebben
|
|||||||
{
|
{
|
||||||
"status": "degraded",
|
"status": "degraded",
|
||||||
"service": "geointel-backend",
|
"service": "geointel-backend",
|
||||||
"database": "unavailable",
|
"database": "degraded",
|
||||||
"redis": "ok",
|
"postgis": "degraded",
|
||||||
"storage": "ok",
|
"migration": "degraded",
|
||||||
"errors": ["database connection failed"]
|
"storage": "ok"
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -51,7 +65,8 @@ Codex mag kiezen tussen een internal endpoint of CLI-command, maar het contract
|
|||||||
Frontend moet een `/status` of settings/statuspanel hebben dat toont: backend bereikbaar, API version, auth status, feature flags en laatste healthchecktijd.
|
Frontend moet een `/status` of settings/statuspanel hebben dat toont: backend bereikbaar, API version, auth status, feature flags en laatste healthchecktijd.
|
||||||
|
|
||||||
## Storage
|
## Storage
|
||||||
Storage healthcheck controleert of upload, originals, derived en exports writable zijn.
|
De readinesscheck maakt en verwijdert een tijdelijk bestand in `STORAGE_ROOT`.
|
||||||
|
|
||||||
## Database
|
## Database
|
||||||
Database healthcheck controleert connectie, PostGIS extension en migrationstatus.
|
De readinesscheck controleert `SELECT 1`, `PostGIS_Version()` en dat de
|
||||||
|
databaseversie exact overeenkomt met de enige Alembic-head in de image.
|
||||||
|
|||||||
@@ -0,0 +1,471 @@
|
|||||||
|
# Autonomous RC Roadmap: Belgium and the Belgian North Sea
|
||||||
|
|
||||||
|
## Mission
|
||||||
|
|
||||||
|
Turn the current GeoIntel implementation into a reproducible, demonstrable and
|
||||||
|
release-candidate-quality map-first workbench for all of Belgium and the
|
||||||
|
Belgian North Sea.
|
||||||
|
|
||||||
|
This file is an executable runbook for Codex. It is both the implementation
|
||||||
|
order and the release evidence index. Work continues autonomously between
|
||||||
|
phases. A separate RC-12 phase is deliberately omitted.
|
||||||
|
|
||||||
|
## Sources of truth
|
||||||
|
|
||||||
|
Read in this order before changing behavior:
|
||||||
|
|
||||||
|
1. `AGENTS.md`
|
||||||
|
2. `docs/CODEX_BOOTSTRAP_PROMPT.md`
|
||||||
|
3. `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md`
|
||||||
|
4. `docs/governance/GEOINTEL_CONSTITUTION.md`
|
||||||
|
5. `docs/governance/ARCHITECTURE_INVARIANTS.md`
|
||||||
|
6. `docs/governance/FORBIDDEN_DECISIONS.md`
|
||||||
|
7. `docs/API_CONTRACTS.md`
|
||||||
|
8. `docs/DATABASE_IMPLEMENTATION_PLAN.md`
|
||||||
|
9. `docs/DATA_SPECIFICATION.md`
|
||||||
|
10. `docs/DATA_SOURCES.md`
|
||||||
|
11. `docs/STORAGE_ARCHITECTURE.md`
|
||||||
|
12. `docs/DEFINITION_OF_DONE.md`
|
||||||
|
|
||||||
|
## Autonomous execution contract
|
||||||
|
|
||||||
|
Codex shall:
|
||||||
|
|
||||||
|
- execute phases in order and continue without requesting routine approval;
|
||||||
|
- use existing service, dataset, vector-feature, raster and job boundaries;
|
||||||
|
- keep every provider bounded, fail-closed and provenance-complete;
|
||||||
|
- add or update tests with every behavior change;
|
||||||
|
- update `docs/CODEX_EXECUTION_LOG.md`, `docs/TODO.md` and `CHANGELOG.md`
|
||||||
|
after each completed phase;
|
||||||
|
- run the phase gate and record exact output under
|
||||||
|
`storage/release-evidence/<release-id>/`;
|
||||||
|
- commit and push only after the relevant local gate is green;
|
||||||
|
- deploy only a commit that passed the local gate;
|
||||||
|
- run browser and live API acceptance after deployment;
|
||||||
|
- retain Mol/Kempen as regression references while adding Wallonia, Brussels,
|
||||||
|
cross-region, coast and North Sea golden areas.
|
||||||
|
|
||||||
|
Codex stops only when:
|
||||||
|
|
||||||
|
- an operation would destructively mutate or delete unbacked production data;
|
||||||
|
- an external licence, credential or authority decision cannot be derived from
|
||||||
|
public source metadata;
|
||||||
|
- an official endpoint fails security validation and no authoritative
|
||||||
|
alternative exists;
|
||||||
|
- the same mandatory gate fails three times for the same external reason; or
|
||||||
|
- continuing would require fabricating data, metrics, provenance or model
|
||||||
|
output.
|
||||||
|
|
||||||
|
For a stop, write the blocker, evidence and exact resume command to the
|
||||||
|
execution log. Continue with independent work when possible.
|
||||||
|
|
||||||
|
## Release priorities
|
||||||
|
|
||||||
|
### P0 - release blockers
|
||||||
|
|
||||||
|
- truthful health/readiness and capability reporting;
|
||||||
|
- current backup plus isolated restore proof;
|
||||||
|
- stale job/run reconciliation and useful logs;
|
||||||
|
- temporally correct detection QA;
|
||||||
|
- production secrets, upload limits, configuration parity and rollback;
|
||||||
|
- full CI and reproducible dependencies;
|
||||||
|
- fresh install and upgrade proof.
|
||||||
|
|
||||||
|
### P1 - national product completeness
|
||||||
|
|
||||||
|
- national and maritime scope model;
|
||||||
|
- governed source coverage matrix;
|
||||||
|
- Wallonia, Brussels and North Sea adapters;
|
||||||
|
- nationwide bounded selection and mixed-zone result handling;
|
||||||
|
- historical compatibility matrix;
|
||||||
|
- golden areas outside Flanders.
|
||||||
|
|
||||||
|
### P2 - quality and maintainability
|
||||||
|
|
||||||
|
- typed response models on critical routes;
|
||||||
|
- frontend unit and browser E2E coverage;
|
||||||
|
- loading/accessibility/performance hardening;
|
||||||
|
- retention and operator documentation;
|
||||||
|
- remaining bounded modularization where protected by tests.
|
||||||
|
|
||||||
|
## Global gates
|
||||||
|
|
||||||
|
Run after every behavior-changing phase:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m compileall backend/app
|
||||||
|
cd backend && python -m pytest
|
||||||
|
cd frontend && npm run typecheck
|
||||||
|
cd frontend && npm run build
|
||||||
|
bash scripts/run_readiness_check.sh
|
||||||
|
cd backend && python -m alembic heads
|
||||||
|
cd backend && python -m alembic upgrade head --sql
|
||||||
|
bash -n scripts/live_migration_smoke.sh
|
||||||
|
docker compose config
|
||||||
|
```
|
||||||
|
|
||||||
|
When Tower is reachable:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
.\scripts\deploy_tower.ps1
|
||||||
|
```
|
||||||
|
|
||||||
|
Then verify:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -fsS http://192.168.10.150:1202/health/live
|
||||||
|
curl -fsS http://192.168.10.150:1202/health/ready
|
||||||
|
curl -fsS http://192.168.10.150:1202/api/v1/system/capabilities
|
||||||
|
```
|
||||||
|
|
||||||
|
The in-app browser acceptance must cover desktop, widescreen and a narrow
|
||||||
|
viewport. It must inspect console errors, failed requests and the complete
|
||||||
|
map-select-analyse-export flow.
|
||||||
|
|
||||||
|
## Official source baseline
|
||||||
|
|
||||||
|
The roadmap starts from these verified official families. Machine endpoints,
|
||||||
|
editions and licences must still pass source-specific probes before activation.
|
||||||
|
|
||||||
|
| Scope | Candidate | Role |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Belgium | NGI/IGN AdminVector and Top10Vector | authoritative boundary and common topographic baseline |
|
||||||
|
| Belgium | Statbel statistical sectors/population | authoritative statistical geometry and time series |
|
||||||
|
| Flanders | existing Digitaal Vlaanderen/VMM/DOV/INBO/ALZ adapters | detailed authoritative regional layers |
|
||||||
|
| Wallonia | SPW Géoportail, PICC, orthophoto, MNT and thematic services | detailed authoritative regional layers |
|
||||||
|
| Brussels | Paradigm UrbIS and regional environmental geodata | detailed authoritative urban layers |
|
||||||
|
| North Sea | FPS Marine Environment marine spatial plan | legal/use zones |
|
||||||
|
| North Sea | RBINS/BMDC and Marine Atlas | marine environment and observations |
|
||||||
|
| North Sea | MDK/Flemish Hydrography | bathymetry and nautical source candidate |
|
||||||
|
| North Sea | EMODnet | contextual fallback only when explicitly labelled |
|
||||||
|
|
||||||
|
## Phase status
|
||||||
|
|
||||||
|
| Phase | State | Purpose |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| RC-0 | complete | freeze scope and produce release evidence baseline |
|
||||||
|
| RC-1 | in progress | backup, restore and data safety |
|
||||||
|
| RC-2 | verification pending | health, capabilities and stale-runtime correctness |
|
||||||
|
| RC-3 | pending | temporal detection/QA correctness and observability |
|
||||||
|
| RC-4 | pending | national/maritime scope and provider coverage contracts |
|
||||||
|
| RC-5 | pending | deployment, secrets, configuration, fresh install and rollback |
|
||||||
|
| RC-6 | pending | complete CI, dependency and supply-chain gates |
|
||||||
|
| RC-7 | pending | critical API envelope typing and contract validation |
|
||||||
|
| RC-8 | pending | frontend and browser E2E release journeys |
|
||||||
|
| RC-9 | pending | loading, accessibility and performance hardening |
|
||||||
|
| RC-10 | pending | retention, cleanup and national data operations |
|
||||||
|
| RC-11 | pending | final package, upgrade proof, release tag and handoff |
|
||||||
|
|
||||||
|
## RC-0 - Scope freeze and evidence baseline
|
||||||
|
|
||||||
|
**State: complete.** The read-only baseline command and focused tests pass.
|
||||||
|
Runtime evidence is stored outside Git under `storage/release-evidence/`.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
- Freeze Belgium plus Belgian North Sea scope.
|
||||||
|
- Record current commit, migration head, dependency versions, container image,
|
||||||
|
database/storage sizes and readiness results.
|
||||||
|
- Create a release-evidence manifest command.
|
||||||
|
- Replace stale product-status wording that still treats Kempen as the product
|
||||||
|
boundary.
|
||||||
|
- Add a coverage matrix schema with `operational`, `partial`,
|
||||||
|
`not_configured`, `unsupported`.
|
||||||
|
|
||||||
|
### Likely files
|
||||||
|
|
||||||
|
- `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md`
|
||||||
|
- `docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md`
|
||||||
|
- `docs/BUILD_STATUS.md`
|
||||||
|
- `docs/DATA_SOURCES.md`
|
||||||
|
- `scripts/capture_release_evidence.py`
|
||||||
|
- `backend/tests/test_rc_release_evidence.py`
|
||||||
|
|
||||||
|
### Evidence and exit
|
||||||
|
|
||||||
|
- scope documents contain Belgium, all three regions and maritime zones;
|
||||||
|
- evidence command is read-only and deterministic;
|
||||||
|
- manifest contains commit, dirty state, Alembic head and gate status;
|
||||||
|
- compile and focused tests pass.
|
||||||
|
|
||||||
|
## RC-1 - Backup, restore and data safety
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
- Add a safe production backup command for PostgreSQL custom-format dumps.
|
||||||
|
- Capture schema head, PostGIS version, table counts, model checksums and a
|
||||||
|
storage checksum inventory without duplicating all large artifacts.
|
||||||
|
- Reject default/empty production database passwords.
|
||||||
|
- Add backup verification with `pg_restore --list` and checksums.
|
||||||
|
- Add an isolated restore drill using a temporary database/container; never
|
||||||
|
restore over production.
|
||||||
|
- Document retention, encryption boundary and cleanup.
|
||||||
|
|
||||||
|
### Likely files
|
||||||
|
|
||||||
|
- `scripts/backup_release_state.sh`
|
||||||
|
- `scripts/verify_release_backup.sh`
|
||||||
|
- `scripts/restore_release_backup_smoke.sh`
|
||||||
|
- `deploy/unraid/README.md`
|
||||||
|
- `docs/ROLLBACK_AND_RECOVERY.md`
|
||||||
|
- `backend/tests/test_rc_backup_restore_scripts.py`
|
||||||
|
|
||||||
|
### Exit
|
||||||
|
|
||||||
|
- current Tower database has a dated, checksum-verified backup;
|
||||||
|
- isolated restore reaches the recorded Alembic head;
|
||||||
|
- representative project, dataset, geometry and QA counts reconcile;
|
||||||
|
- no production application data was mutated by the drill.
|
||||||
|
|
||||||
|
## RC-2 - Truthful runtime state
|
||||||
|
|
||||||
|
**State: implementation complete, live verification pending.** Local compile,
|
||||||
|
963 backend tests, frontend typecheck/build and the full readiness gate pass.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
- Split process liveness from dependency readiness.
|
||||||
|
- Make readiness return HTTP 503 when DB, PostGIS, migration head or writable
|
||||||
|
storage is unavailable.
|
||||||
|
- Derive version/build identity and AI capability from runtime configuration.
|
||||||
|
- Make Docker use the readiness endpoint.
|
||||||
|
- Reconcile orphaned `running` jobs and analysis runs on process startup.
|
||||||
|
- Add request IDs and exception logging without exposing secrets or full SQL.
|
||||||
|
|
||||||
|
### Likely files
|
||||||
|
|
||||||
|
- `backend/app/api/routes/health.py`
|
||||||
|
- `backend/app/schemas/health.py`
|
||||||
|
- `backend/app/core/config.py`
|
||||||
|
- `backend/app/core/logging.py`
|
||||||
|
- `backend/app/main.py`
|
||||||
|
- `backend/app/services/runtime_reconciliation_service.py`
|
||||||
|
- Docker/nginx files and health tests
|
||||||
|
|
||||||
|
### Exit
|
||||||
|
|
||||||
|
- liveness remains 200 while the process runs;
|
||||||
|
- readiness is fail-closed;
|
||||||
|
- live capabilities match the active local YOLO state;
|
||||||
|
- restart converts impossible orphaned work to a terminal failed state with
|
||||||
|
`PROCESS_INTERRUPTED`;
|
||||||
|
- Docker reports unhealthy for a broken database.
|
||||||
|
|
||||||
|
## RC-3 - Temporal QA correctness and observability
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
- Stop Detection Lab from silently selecting the first raster.
|
||||||
|
- Require explicit raster/model choice.
|
||||||
|
- Prevent current reference QA against historical imagery unless a compatible
|
||||||
|
reference edition/coverage is selected.
|
||||||
|
- Persist source observation period and QA compatibility decision.
|
||||||
|
- Add structured request/job/run logs and operator diagnostics.
|
||||||
|
- Add a stale-runtime report and safe reconciliation command.
|
||||||
|
|
||||||
|
### Exit
|
||||||
|
|
||||||
|
- no detection run starts with an implicit old raster;
|
||||||
|
- incompatible temporal QA fails with a clear code;
|
||||||
|
- compatible current orthophoto/GRB path remains green;
|
||||||
|
- logs correlate request, job, analysis run and quality check.
|
||||||
|
|
||||||
|
## RC-4 - Belgium and North Sea coverage foundation
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
1. Persist NGI authoritative Belgium land and administrative scopes.
|
||||||
|
2. Persist territorial sea, EEZ and continental shelf as distinct scope
|
||||||
|
layers.
|
||||||
|
3. Implement the coverage matrix and source authority resolver.
|
||||||
|
4. Add bounded provider contracts for:
|
||||||
|
- NGI/Statbel national baseline;
|
||||||
|
- SPW/PICC/orthophoto/relief for Wallonia;
|
||||||
|
- UrbIS/environmental context for Brussels;
|
||||||
|
- federal marine zones and BMDC/MDK marine sources.
|
||||||
|
5. Normalize the public theme vocabulary without discarding provider-native
|
||||||
|
semantics.
|
||||||
|
6. Add Wallonia, Brussels, cross-region, coast and North Sea golden areas.
|
||||||
|
7. Make mixed-zone selection split into bounded provider requests and merge
|
||||||
|
only semantically compatible metrics.
|
||||||
|
|
||||||
|
### Required rules
|
||||||
|
|
||||||
|
- all persistence flows through DatasetService and geospatial services;
|
||||||
|
- no provider writes directly to feature tables;
|
||||||
|
- no country-wide startup import;
|
||||||
|
- no TLS bypass;
|
||||||
|
- no implicit vertical datum conversion;
|
||||||
|
- unsupported metrics remain unavailable.
|
||||||
|
|
||||||
|
### Likely files
|
||||||
|
|
||||||
|
- provider registry/base and new regional provider modules;
|
||||||
|
- geographic scope and source coverage services;
|
||||||
|
- dataset/source schemas;
|
||||||
|
- map source portfolio and scope UI;
|
||||||
|
- source, API, data and storage documentation;
|
||||||
|
- source-specific tests and fixtures.
|
||||||
|
|
||||||
|
### Exit
|
||||||
|
|
||||||
|
- a user can draw within every golden area and receive an honest coverage
|
||||||
|
matrix;
|
||||||
|
- at least the common national baseline is operational on land;
|
||||||
|
- regional detailed layers are operational or explicitly partial;
|
||||||
|
- maritime legal zones are operational;
|
||||||
|
- bathymetry remains gated until a source passes strict acquisition evidence.
|
||||||
|
|
||||||
|
## RC-5 - Production deployment and rollback
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
- align backend, nginx and proxy upload/time limits;
|
||||||
|
- make every runtime setting intentionally configured, internal or documented;
|
||||||
|
- reject production default secrets;
|
||||||
|
- add image labels with version, commit and build time;
|
||||||
|
- deploy immutable tags in addition to `latest`;
|
||||||
|
- preserve previous image for one-command rollback;
|
||||||
|
- prove fresh install against empty volumes;
|
||||||
|
- prove upgrade from a copied current database/storage state;
|
||||||
|
- prove rollback without schema or data loss.
|
||||||
|
|
||||||
|
### Exit
|
||||||
|
|
||||||
|
- Unraid edit fields cover all operator-owned settings;
|
||||||
|
- fresh install, upgrade and rollback commands are documented and executed;
|
||||||
|
- deployed UI exposes build identity;
|
||||||
|
- previous image remains addressable;
|
||||||
|
- no mutable-only release is accepted.
|
||||||
|
|
||||||
|
## RC-6 - CI and supply-chain gates
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
- make CI run backend compile/tests, frontend typecheck/build, readiness,
|
||||||
|
Alembic single-head/offline SQL and Docker config/build;
|
||||||
|
- add Python and npm dependency vulnerability checks;
|
||||||
|
- add container scan and SBOM;
|
||||||
|
- introduce reproducible Python resolution/lock output;
|
||||||
|
- preserve optional AI dependency separation;
|
||||||
|
- publish gate artefacts.
|
||||||
|
|
||||||
|
### Exit
|
||||||
|
|
||||||
|
- a broken compile, test, build, migration or contract blocks CI;
|
||||||
|
- dependency and container findings have severity policy;
|
||||||
|
- a clean checkout can reproduce dependency versions and image metadata.
|
||||||
|
|
||||||
|
## RC-7 - Critical API contract hardening
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
- replace `response_model=dict` first on health, projects, areas, datasets,
|
||||||
|
jobs, analysis runs, QA, exports and map-critical endpoints;
|
||||||
|
- retain canonical `{data: ...}` success and canonical error envelopes;
|
||||||
|
- add OpenAPI response validation tests;
|
||||||
|
- document intentionally streaming/download responses separately.
|
||||||
|
|
||||||
|
### Exit
|
||||||
|
|
||||||
|
- all critical map-first routes have concrete response models;
|
||||||
|
- actual payloads pass FastAPI validation;
|
||||||
|
- API contract audit and frontend typecheck pass.
|
||||||
|
|
||||||
|
## RC-8 - Release journey automation
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
- add frontend unit tests for selection, coverage, loading and temporal guards;
|
||||||
|
- add browser E2E for:
|
||||||
|
- open Belgium map;
|
||||||
|
- select each golden area;
|
||||||
|
- load a theme;
|
||||||
|
- inspect metric/provenance;
|
||||||
|
- compare compatible history;
|
||||||
|
- run a bounded configured detection path;
|
||||||
|
- export;
|
||||||
|
- ask Ollama with persisted context;
|
||||||
|
- include no-data, partial coverage, provider failure and unsupported metric
|
||||||
|
journeys.
|
||||||
|
|
||||||
|
### Exit
|
||||||
|
|
||||||
|
- E2E runs against a real backend/PostGIS fixture;
|
||||||
|
- no workflow depends only on source-text assertion tests;
|
||||||
|
- console and failed-request audits are clean.
|
||||||
|
|
||||||
|
## RC-9 - UX, accessibility and performance
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
- replace initial false `missing` source states with explicit loading;
|
||||||
|
- provide accessible names, labels, keyboard focus and status announcements;
|
||||||
|
- enforce desktop, widescreen and narrow layouts;
|
||||||
|
- record API and map-selection budgets for golden areas;
|
||||||
|
- virtualize or page large panels;
|
||||||
|
- split large modules only where tests protect behavior.
|
||||||
|
|
||||||
|
### Exit
|
||||||
|
|
||||||
|
- no core control lacks an accessible name;
|
||||||
|
- loading, empty, partial, error and ready states are distinct;
|
||||||
|
- map selection remains usable at target viewports;
|
||||||
|
- no release journey exceeds its documented budget without a visible warning.
|
||||||
|
|
||||||
|
## RC-10 - Data operations and retention
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
- define lifecycle for raw, normalized, derived, export, AI and evidence
|
||||||
|
artifacts;
|
||||||
|
- add dry-run-first cleanup for stale technical projects, failed work and
|
||||||
|
superseded caches;
|
||||||
|
- retain immutable official source editions and release evidence;
|
||||||
|
- report disk pressure before acquisition;
|
||||||
|
- schedule nothing implicitly; provide explicit operator commands;
|
||||||
|
- add source freshness reports for national/regional/maritime families.
|
||||||
|
|
||||||
|
### Exit
|
||||||
|
|
||||||
|
- cleanup cannot remove release evidence or current authoritative editions;
|
||||||
|
- every destructive action requires explicit confirmation and a current
|
||||||
|
backup;
|
||||||
|
- storage growth and retained provenance are reportable.
|
||||||
|
|
||||||
|
## RC-11 - Final release package
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
- rerun fresh-install and upgrade proof using the release image;
|
||||||
|
- rerun backup/isolated restore and rollback proof;
|
||||||
|
- execute every golden area and core browser journey;
|
||||||
|
- generate coverage, security, dependency, migration, performance and known
|
||||||
|
limitation reports;
|
||||||
|
- update operator/end-user documentation;
|
||||||
|
- remove stale status documents from active navigation without deleting
|
||||||
|
historical evidence;
|
||||||
|
- assign semantic version, create signed/checksummed release manifest, tag,
|
||||||
|
push and deploy the immutable image.
|
||||||
|
|
||||||
|
### Final acceptance
|
||||||
|
|
||||||
|
- all P0 findings are closed;
|
||||||
|
- every mandatory global gate is green;
|
||||||
|
- backup and isolated restore are proven;
|
||||||
|
- one Alembic head applies to empty and upgraded PostGIS;
|
||||||
|
- Belgium and maritime golden areas pass;
|
||||||
|
- no unsupported metric or unavailable source appears as successful;
|
||||||
|
- live browser acceptance passes at port 1202;
|
||||||
|
- remaining P1/P2 limitations are explicit, bounded and non-deceptive.
|
||||||
|
|
||||||
|
## Next command
|
||||||
|
|
||||||
|
After creating this roadmap, start RC-0 immediately:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python scripts/capture_release_evidence.py --output storage/release-evidence/rc-current/baseline.json
|
||||||
|
```
|
||||||
|
|
||||||
|
If the command does not exist yet, implementing it and its tests is the first
|
||||||
|
code task.
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
# RC Scope Freeze: Belgium and the Belgian North Sea
|
||||||
|
|
||||||
|
## Status and precedence
|
||||||
|
|
||||||
|
This document freezes the geographic and release scope for the GeoIntel
|
||||||
|
release-candidate program. It supersedes the geographic limitation in
|
||||||
|
`docs/V1_SCOPE_FREEZE.md`. The existing V1 domain, persistence, provenance,
|
||||||
|
GIS-correctness and no-fake-data rules remain in force.
|
||||||
|
|
||||||
|
Mol and the Kempen remain golden regression areas. They are no longer the
|
||||||
|
product boundary.
|
||||||
|
|
||||||
|
## Product identity
|
||||||
|
|
||||||
|
GeoIntel RC is a map-first geospatial analysis workbench for:
|
||||||
|
|
||||||
|
- the complete land territory of Belgium;
|
||||||
|
- the Belgian territorial sea;
|
||||||
|
- the Belgian exclusive economic zone and continental shelf, labelled
|
||||||
|
according to their legal meaning;
|
||||||
|
- bounded user-drawn areas that may cross municipal, provincial, regional or
|
||||||
|
land/sea boundaries.
|
||||||
|
|
||||||
|
The primary flow remains:
|
||||||
|
|
||||||
|
1. open the map;
|
||||||
|
2. choose one or more understandable themes;
|
||||||
|
3. draw or select an area;
|
||||||
|
4. load only the authoritative or explicitly contextual data available for
|
||||||
|
that area;
|
||||||
|
5. calculate source-appropriate metrics;
|
||||||
|
6. inspect provenance, limitations and time;
|
||||||
|
7. compare compatible historical editions;
|
||||||
|
8. export the result or ask the local assistant about persisted evidence.
|
||||||
|
|
||||||
|
## Geographic federation
|
||||||
|
|
||||||
|
Belgian coverage is a federation of governed adapters. GeoIntel must never
|
||||||
|
pretend that regional datasets have identical semantics merely because they
|
||||||
|
can be displayed together.
|
||||||
|
|
||||||
|
| Zone | Baseline authority | Detailed authoritative families |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Belgium-wide | NGI/IGN, Statbel, federal geo.be | AdminVector, Top10Vector/CartoWeb, statistical sectors and population |
|
||||||
|
| Flanders | Digitaal Vlaanderen and competent Flemish agencies | GRB, orthophotos, DHMV, VMM/VHA, DOV, INBO, ALZ |
|
||||||
|
| Wallonia | Service public de Wallonie | PICC, SPW orthophotos, relief, hydrography, land cover and thematic catalogues |
|
||||||
|
| Brussels | Brussels Capital Region/Paradigm and competent administrations | UrbIS, BruGIS and Brussels open geodata |
|
||||||
|
| Belgian North Sea | federal Marine Environment, RBINS/BMDC, MDK and legally competent publishers | maritime plan and zones, marine environment, hydrography and validated bathymetry |
|
||||||
|
|
||||||
|
NGI Top10Vector is the preferred common topographic baseline candidate when
|
||||||
|
its machine access, licence, edition and bounded retrieval contract have been
|
||||||
|
validated. Regional large-scale products remain preferred where their
|
||||||
|
authority and semantic detail are higher.
|
||||||
|
|
||||||
|
## Required RC coverage
|
||||||
|
|
||||||
|
The RC must provide:
|
||||||
|
|
||||||
|
- a persisted authoritative Belgium land boundary;
|
||||||
|
- persisted and legally labelled maritime scope boundaries;
|
||||||
|
- area selection anywhere inside the combined Belgium/maritime scope;
|
||||||
|
- a coverage matrix that reports every theme as `operational`, `partial`,
|
||||||
|
`not_configured` or `unsupported` for the selected zone;
|
||||||
|
- source, authority, observation time, publication time, CRS, licence,
|
||||||
|
attribution, coverage and limitation metadata for every result;
|
||||||
|
- bounded acquisition and analysis; no browser-side source fetch and no
|
||||||
|
unbounded country-wide request to a public service;
|
||||||
|
- compatible historical comparison only where editions, units, coverage and
|
||||||
|
methods are comparable;
|
||||||
|
- a stable national theme vocabulary while preserving provider-native fields
|
||||||
|
and semantics.
|
||||||
|
|
||||||
|
The minimum common theme vocabulary is:
|
||||||
|
|
||||||
|
- administrative context;
|
||||||
|
- buildings and built environment;
|
||||||
|
- roads and mobility context;
|
||||||
|
- surface water and hydrography;
|
||||||
|
- land cover and land use;
|
||||||
|
- vegetation, forest and nature;
|
||||||
|
- population and statistical context;
|
||||||
|
- elevation and terrain;
|
||||||
|
- orthophoto/aerial imagery;
|
||||||
|
- flood or climate context where authoritative data exists;
|
||||||
|
- maritime planning, marine environment and bathymetry for the Belgian North
|
||||||
|
Sea where authoritative data exists.
|
||||||
|
|
||||||
|
Not every theme must have the same resolution or historical depth in every
|
||||||
|
zone. Absence or partial coverage must be visible and must not be replaced by
|
||||||
|
fabricated values.
|
||||||
|
|
||||||
|
## Golden release areas
|
||||||
|
|
||||||
|
National support is validated through bounded golden areas, not one monolithic
|
||||||
|
import:
|
||||||
|
|
||||||
|
- Mol municipality: existing deep land and AI regression reference;
|
||||||
|
- Kempen transport region: existing cross-municipality and partition test;
|
||||||
|
- one Walloon urban/rural boundary-crossing area;
|
||||||
|
- one Brussels urban area;
|
||||||
|
- one cross-region or language-boundary area;
|
||||||
|
- one coastal land/sea area;
|
||||||
|
- one Belgian North Sea area containing at least two legal/use zones.
|
||||||
|
|
||||||
|
Exact golden area identifiers and checksums must be stored in release
|
||||||
|
evidence. A source is not nationally operational merely because the Mol test
|
||||||
|
passes.
|
||||||
|
|
||||||
|
## Maritime and bathymetry rules
|
||||||
|
|
||||||
|
- Territorial sea, EEZ and continental shelf are separate legal scopes.
|
||||||
|
- TAW, LAT, mDNG, NAP and product-specific vertical references remain
|
||||||
|
separate dimensions.
|
||||||
|
- No value is transformed between vertical datums without an authoritative,
|
||||||
|
tested transformation and an uncertainty statement.
|
||||||
|
- Bathymetry is bed elevation or depth relative to a stated reference plane.
|
||||||
|
It is not water volume.
|
||||||
|
- Water volume requires compatible bed elevation, water-surface elevation,
|
||||||
|
time, coverage and uncertainty contracts.
|
||||||
|
- WMTS/WMS imagery may provide visual context but is not an analytical raster
|
||||||
|
unless pixel values and georeferencing are governed and validated.
|
||||||
|
|
||||||
|
## Explicit exclusions from this RC
|
||||||
|
|
||||||
|
- automatic whole-country mirroring of every source;
|
||||||
|
- bypassing TLS, licence or source-integrity failures;
|
||||||
|
- claiming semantic parity between GRB, PICC, UrbIS and NGI products;
|
||||||
|
- hidden external downloads at application startup;
|
||||||
|
- browser-direct provider access;
|
||||||
|
- production multi-user authentication;
|
||||||
|
- real-time surveillance or operational maritime navigation;
|
||||||
|
- unsupported water-volume estimates;
|
||||||
|
- a separate RC-12 soak phase.
|
||||||
|
|
||||||
|
Fresh-install, upgrade, rollback and runtime-stability evidence are mandatory
|
||||||
|
but are incorporated into RC-5 and RC-11.
|
||||||
|
|
||||||
|
## Scope change control
|
||||||
|
|
||||||
|
During the RC program, a scope change is allowed only when it:
|
||||||
|
|
||||||
|
1. fixes a release blocker;
|
||||||
|
2. adds missing coverage required by this document;
|
||||||
|
3. corrects source authority, provenance, CRS, time or metric semantics; or
|
||||||
|
4. improves verification without adding unrelated product capability.
|
||||||
|
|
||||||
|
Every accepted change must update the autonomous roadmap, execution log and
|
||||||
|
relevant source/API documentation.
|
||||||
@@ -11,3 +11,24 @@ Bestanden worden niet stilzwijgend verwijderd. Gebruik soft-delete waar mogelijk
|
|||||||
|
|
||||||
## AI/model rollback
|
## AI/model rollback
|
||||||
Elke analysis run bewaart model id, version, parameters, confidence threshold en pipeline version.
|
Elke analysis run bewaart model id, version, parameters, confidence threshold en pipeline version.
|
||||||
|
# RC backup and isolated restore gate
|
||||||
|
|
||||||
|
Before any release migration, credential rotation or image promotion:
|
||||||
|
|
||||||
|
1. run `scripts/backup_release_state.sh` against the running all-in-one
|
||||||
|
container;
|
||||||
|
2. run `scripts/verify_release_backup.sh` read-only;
|
||||||
|
3. run `scripts/restore_release_backup_smoke.sh` with
|
||||||
|
`--confirm-isolated-restore`;
|
||||||
|
4. retain the backup directory and the separate restore-smoke JSON with the
|
||||||
|
release evidence.
|
||||||
|
|
||||||
|
The restore smoke may only create databases whose name starts with
|
||||||
|
`geointel_restore_verify_`. It refuses the production database name, does not
|
||||||
|
use `pg_restore --clean` and drops the temporary database unless an operator
|
||||||
|
explicitly asks to retain it.
|
||||||
|
|
||||||
|
Storage and model files are inventoried rather than copied into the database
|
||||||
|
dump. Release backups must therefore be paired with the persistent storage
|
||||||
|
volume backup policy. Use `--inventory-mode sha256` for final release
|
||||||
|
evidence.
|
||||||
|
|||||||
@@ -1,5 +1,35 @@
|
|||||||
# GeoIntel TODO
|
# GeoIntel TODO
|
||||||
|
|
||||||
|
## Actief RC-programma: Belgie en de Belgische Noordzee
|
||||||
|
|
||||||
|
`docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md` is vanaf 2026-07-17 het enige actieve
|
||||||
|
releasebord. Mol en de Kempen blijven regressiereferenties, maar de
|
||||||
|
productscope is nu heel Belgie plus de juridisch correct gelabelde Belgische
|
||||||
|
maritieme zones.
|
||||||
|
|
||||||
|
- [x] RC-0: Belgische en maritieme scopefreeze vastleggen.
|
||||||
|
- [x] RC-0: autonome roadmap zonder afzonderlijke RC-12 vastleggen.
|
||||||
|
- [x] RC-0: deterministisch release-evidence manifest implementeren en draaien.
|
||||||
|
- [ ] RC-1: actuele productiebackup, verificatie en geisoleerde restore bewijzen.
|
||||||
|
- [ ] RC-2: liveness/readiness/capabilities fail-closed en waarheidsgetrouw maken.
|
||||||
|
- [ ] RC-2: verweesde jobs en analysis runs na een procesherstart verzoenen.
|
||||||
|
- [ ] RC-3: expliciete rasterkeuze en temporeel compatibele detectie-QA afdwingen.
|
||||||
|
- [ ] RC-4: nationale basisdekking, Wallonie, Brussel en Belgische Noordzee via
|
||||||
|
beheerde providers en golden areas operationaliseren.
|
||||||
|
- [ ] RC-5: secrets/configuratie/uploadlimieten/immutable deploy en rollback
|
||||||
|
bewijzen.
|
||||||
|
- [ ] RC-6: volledige CI, dependency-audit, containerscan en SBOM toevoegen.
|
||||||
|
- [ ] RC-7: kritieke API-routes concrete responsemodellen geven.
|
||||||
|
- [ ] RC-8: echte frontend- en browser-E2E-releaseflows toevoegen.
|
||||||
|
- [ ] RC-9: loading, toegankelijkheid, widescreen/mobile en performance afronden.
|
||||||
|
- [ ] RC-10: dataretentie, diskdruk en veilige cleanup operationaliseren.
|
||||||
|
- [ ] RC-11: fresh install, upgrade, rollback, releasepakket, tag en live
|
||||||
|
acceptatie afronden.
|
||||||
|
|
||||||
|
Een aparte RC-12/soakfase wordt niet uitgevoerd. De relevante fresh-install-,
|
||||||
|
upgrade-, rollback- en runtimebewijzen zijn verplicht opgenomen in RC-5 en
|
||||||
|
RC-11.
|
||||||
|
|
||||||
## Actuele V1-productstatus
|
## Actuele V1-productstatus
|
||||||
|
|
||||||
Dit is het enige actuele afwerkingsbord. De lange sprint- en
|
Dit is het enige actuele afwerkingsbord. De lange sprint- en
|
||||||
|
|||||||
@@ -36,6 +36,18 @@ server {
|
|||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
location = /health/live {
|
||||||
|
proxy_pass http://backend:8000/health/live;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
}
|
||||||
|
|
||||||
|
location = /health/ready {
|
||||||
|
proxy_pass http://backend:8000/health/ready;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
}
|
||||||
|
|
||||||
location / {
|
location / {
|
||||||
try_files $uri $uri/ /index.html;
|
try_files $uri $uri/ /index.html;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1938,3 +1938,41 @@ docker exec geointel python /app/scripts/probe_mdk_bathymetry.py
|
|||||||
|
|
||||||
This performs only `GetCapabilities`, keeps strict TLS verification enabled
|
This performs only `GetCapabilities`, keeps strict TLS verification enabled
|
||||||
and returns exit code `2` for an honest non-ready source.
|
and returns exit code `2` for an honest non-ready source.
|
||||||
|
# Release backup and restore proof
|
||||||
|
|
||||||
|
The release-candidate safety path is host-operated against the running
|
||||||
|
all-in-one container:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash scripts/backup_release_state.sh \
|
||||||
|
--container geointel \
|
||||||
|
--output-root /mnt/user/appdata/geointel/backups \
|
||||||
|
--storage-path /mnt/user/appdata/geointel/storage \
|
||||||
|
--models-path /mnt/user/appdata/geointel/models \
|
||||||
|
--inventory-mode sha256
|
||||||
|
```
|
||||||
|
|
||||||
|
The backup is written atomically and contains a PostgreSQL custom-format dump,
|
||||||
|
archive listing, Alembic/PostGIS metadata, critical table counts, optional
|
||||||
|
storage/model inventories and SHA-256 checksums. An empty or known-default
|
||||||
|
database password leaves the release gate failed. For an emergency backup
|
||||||
|
before rotating that password, add `--allow-insecure-password`; the manifest
|
||||||
|
still records the insecure state.
|
||||||
|
|
||||||
|
Verify without changing any database:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash scripts/verify_release_backup.sh \
|
||||||
|
--backup-dir /mnt/user/appdata/geointel/backups/<release-id>
|
||||||
|
```
|
||||||
|
|
||||||
|
Prove restoration only in a generated temporary database:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash scripts/restore_release_backup_smoke.sh \
|
||||||
|
--backup-dir /mnt/user/appdata/geointel/backups/<release-id> \
|
||||||
|
--confirm-isolated-restore
|
||||||
|
```
|
||||||
|
|
||||||
|
The restore smoke rejects the production database name, compares PostGIS,
|
||||||
|
Alembic and retained table counts, and removes its temporary database.
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ DOCS = ROOT / "docs" / "API_CONTRACTS.md" # docs/API_CONTRACTS.md
|
|||||||
|
|
||||||
ALLOWED_NON_ENVELOPE_ENDPOINTS = {
|
ALLOWED_NON_ENVELOPE_ENDPOINTS = {
|
||||||
("GET", "/health"),
|
("GET", "/health"),
|
||||||
|
("GET", "/health/live"),
|
||||||
|
("GET", "/health/ready"),
|
||||||
("GET", "/api/v1/exports/{export_id}/download"),
|
("GET", "/api/v1/exports/{export_id}/download"),
|
||||||
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/image"),
|
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/image"),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,217 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
CONTAINER="geointel"
|
||||||
|
OUTPUT_ROOT="backups"
|
||||||
|
RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||||
|
STORAGE_PATH=""
|
||||||
|
MODELS_PATH=""
|
||||||
|
INVENTORY_MODE="metadata"
|
||||||
|
ALLOW_INSECURE_PASSWORD="false"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: bash scripts/backup_release_state.sh [options]
|
||||||
|
|
||||||
|
Creates an atomic, read-only release backup from the running all-in-one
|
||||||
|
GeoIntel container. It never deletes or restores application data.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--container NAME Docker container (default: geointel)
|
||||||
|
--output-root PATH Host backup root (default: backups)
|
||||||
|
--release-id ID Safe backup directory name
|
||||||
|
--storage-path PATH Optional host storage path to inventory
|
||||||
|
--models-path PATH Optional host model path to inventory
|
||||||
|
--inventory-mode metadata|sha256 Hash all inventoried files only with sha256
|
||||||
|
--allow-insecure-password Complete emergency backup despite an
|
||||||
|
empty/default production DB password
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
while [ "$#" -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--container) CONTAINER="$2"; shift 2 ;;
|
||||||
|
--output-root) OUTPUT_ROOT="$2"; shift 2 ;;
|
||||||
|
--release-id) RELEASE_ID="$2"; shift 2 ;;
|
||||||
|
--storage-path) STORAGE_PATH="$2"; shift 2 ;;
|
||||||
|
--models-path) MODELS_PATH="$2"; shift 2 ;;
|
||||||
|
--inventory-mode) INVENTORY_MODE="$2"; shift 2 ;;
|
||||||
|
--allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;;
|
||||||
|
--help|-h) usage; exit 0 ;;
|
||||||
|
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if ! [[ "$RELEASE_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]]; then
|
||||||
|
echo "Unsafe release id: $RELEASE_ID" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
if [ "$INVENTORY_MODE" != "metadata" ] && [ "$INVENTORY_MODE" != "sha256" ]; then
|
||||||
|
echo "--inventory-mode must be metadata or sha256" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
for required in docker python3 sha256sum git; do
|
||||||
|
if ! command -v "$required" >/dev/null 2>&1; then
|
||||||
|
echo "Missing required command: $required" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||||
|
echo "Container '$CONTAINER' is not running." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
OUTPUT_ROOT="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$OUTPUT_ROOT")"
|
||||||
|
mkdir -p "$OUTPUT_ROOT"
|
||||||
|
PARTIAL="$OUTPUT_ROOT/.${RELEASE_ID}.partial"
|
||||||
|
FINAL="$OUTPUT_ROOT/$RELEASE_ID"
|
||||||
|
if [ -e "$PARTIAL" ] || [ -e "$FINAL" ]; then
|
||||||
|
echo "Backup target already exists: $FINAL" >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
mkdir -p "$PARTIAL"
|
||||||
|
|
||||||
|
cleanup_partial() {
|
||||||
|
if [ -d "$PARTIAL" ]; then
|
||||||
|
rm -rf -- "$PARTIAL"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap cleanup_partial EXIT
|
||||||
|
|
||||||
|
DB_NAME="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"' )"
|
||||||
|
DB_USER="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"' )"
|
||||||
|
if [ -z "$DB_NAME" ] || [ -z "$DB_USER" ]; then
|
||||||
|
echo "Could not resolve database identity from the container." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
PASSWORD_SECURE="true"
|
||||||
|
if ! docker exec "$CONTAINER" sh -c '
|
||||||
|
password="${POSTGRES_PASSWORD:-${GEOINTEL_POSTGRES_PASSWORD:-}}"
|
||||||
|
test -n "$password" &&
|
||||||
|
test "$password" != "geointel" &&
|
||||||
|
test "$password" != "postgres" &&
|
||||||
|
test "$password" != "password"
|
||||||
|
'; then
|
||||||
|
PASSWORD_SECURE="false"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Creating PostgreSQL custom-format dump..."
|
||||||
|
docker exec "$CONTAINER" pg_dump \
|
||||||
|
-U "$DB_USER" \
|
||||||
|
-d "$DB_NAME" \
|
||||||
|
-Fc \
|
||||||
|
--no-owner \
|
||||||
|
--no-privileges > "$PARTIAL/database.dump"
|
||||||
|
test -s "$PARTIAL/database.dump"
|
||||||
|
|
||||||
|
docker exec -i "$CONTAINER" pg_restore --list \
|
||||||
|
< "$PARTIAL/database.dump" \
|
||||||
|
> "$PARTIAL/database.list"
|
||||||
|
test -s "$PARTIAL/database.list"
|
||||||
|
|
||||||
|
IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")"
|
||||||
|
IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")"
|
||||||
|
GIT_COMMIT="$(git rev-parse HEAD)"
|
||||||
|
GIT_DIRTY="false"
|
||||||
|
if [ -n "$(git status --porcelain=v1)" ]; then
|
||||||
|
GIT_DIRTY="true"
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \
|
||||||
|
-c "SELECT 'alembic_head', version_num FROM alembic_version
|
||||||
|
UNION ALL SELECT 'postgis_version', postgis_version()
|
||||||
|
UNION ALL SELECT 'database_size_bytes', pg_database_size(current_database())::text
|
||||||
|
ORDER BY 1;" > "$PARTIAL/database-metadata.tsv"
|
||||||
|
|
||||||
|
: > "$PARTIAL/table-counts.tsv"
|
||||||
|
for table in projects areas datasets dataset_versions vector_features jobs analysis_runs detections segmentations quality_checks metrics exports; do
|
||||||
|
count="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -At \
|
||||||
|
-c "SELECT count(*) FROM public.${table};")"
|
||||||
|
printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv"
|
||||||
|
done
|
||||||
|
|
||||||
|
inventory_path() {
|
||||||
|
local source_path="$1"
|
||||||
|
local output_path="$2"
|
||||||
|
if [ -z "$source_path" ]; then
|
||||||
|
printf 'not_requested\n' > "$output_path"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
python3 - "$source_path" "$output_path" "$INVENTORY_MODE" <<'PY'
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
root = pathlib.Path(sys.argv[1]).expanduser().resolve()
|
||||||
|
output = pathlib.Path(sys.argv[2])
|
||||||
|
mode = sys.argv[3]
|
||||||
|
if not root.is_dir():
|
||||||
|
raise SystemExit(f"Inventory root is not a directory: {root}")
|
||||||
|
|
||||||
|
def digest(path: pathlib.Path) -> str:
|
||||||
|
value = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
value.update(chunk)
|
||||||
|
return value.hexdigest()
|
||||||
|
|
||||||
|
with output.open("w", encoding="utf-8", newline="\n") as handle:
|
||||||
|
handle.write("relative_path\tsize_bytes\tmtime_ns\tsha256\n")
|
||||||
|
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()):
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
continue
|
||||||
|
stat = path.stat()
|
||||||
|
checksum = digest(path) if mode == "sha256" else ""
|
||||||
|
relative = path.relative_to(root).as_posix()
|
||||||
|
if "\t" in relative or "\n" in relative:
|
||||||
|
raise SystemExit(f"Unsupported inventory path: {relative!r}")
|
||||||
|
handle.write(f"{relative}\t{stat.st_size}\t{stat.st_mtime_ns}\t{checksum}\n")
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
inventory_path "$STORAGE_PATH" "$PARTIAL/storage-manifest.tsv"
|
||||||
|
inventory_path "$MODELS_PATH" "$PARTIAL/models-manifest.tsv"
|
||||||
|
|
||||||
|
python3 - "$PARTIAL/manifest.json" <<PY
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"release_id": ${RELEASE_ID@Q},
|
||||||
|
"created_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||||
|
"read_only_source": True,
|
||||||
|
"container": ${CONTAINER@Q},
|
||||||
|
"database_name": ${DB_NAME@Q},
|
||||||
|
"database_user": ${DB_USER@Q},
|
||||||
|
"database_password_secure": ${PASSWORD_SECURE@Q} == "true",
|
||||||
|
"image_id": ${IMAGE_ID@Q},
|
||||||
|
"image_name": ${IMAGE_NAME@Q},
|
||||||
|
"git_commit": ${GIT_COMMIT@Q},
|
||||||
|
"git_dirty": ${GIT_DIRTY@Q} == "true",
|
||||||
|
"inventory_mode": ${INVENTORY_MODE@Q},
|
||||||
|
"storage_inventory_requested": bool(${STORAGE_PATH@Q}),
|
||||||
|
"models_inventory_requested": bool(${MODELS_PATH@Q}),
|
||||||
|
}
|
||||||
|
path = pathlib.Path(__import__("sys").argv[1])
|
||||||
|
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
PY
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$PARTIAL"
|
||||||
|
find . -maxdepth 1 -type f ! -name CHECKSUMS.sha256 -printf '%f\n' \
|
||||||
|
| LC_ALL=C sort \
|
||||||
|
| xargs sha256sum > CHECKSUMS.sha256
|
||||||
|
)
|
||||||
|
|
||||||
|
mv "$PARTIAL" "$FINAL"
|
||||||
|
trap - EXIT
|
||||||
|
echo "Release backup created: $FINAL"
|
||||||
|
|
||||||
|
if [ "$PASSWORD_SECURE" != "true" ] && [ "$ALLOW_INSECURE_PASSWORD" != "true" ]; then
|
||||||
|
echo "SECURITY GATE FAILED: production database password is empty or a known default." >&2
|
||||||
|
echo "The emergency backup is valid, but the release remains blocked." >&2
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
@@ -0,0 +1,284 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Capture a read-only, secret-free GeoIntel release evidence manifest."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import importlib.metadata
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import platform
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Sequence
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
BACKEND = ROOT / "backend"
|
||||||
|
DEFAULT_HASHED_FILES = (
|
||||||
|
"AGENTS.md",
|
||||||
|
"backend/pyproject.toml",
|
||||||
|
"backend/alembic.ini",
|
||||||
|
"frontend/package.json",
|
||||||
|
"frontend/package-lock.json",
|
||||||
|
"docker-compose.yml",
|
||||||
|
"docker-compose.unraid.yml",
|
||||||
|
"deploy/unraid/Dockerfile.all-in-one",
|
||||||
|
"docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md",
|
||||||
|
"docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md",
|
||||||
|
)
|
||||||
|
DEPENDENCIES = (
|
||||||
|
"alembic",
|
||||||
|
"fastapi",
|
||||||
|
"geoalchemy2",
|
||||||
|
"geopandas",
|
||||||
|
"numpy",
|
||||||
|
"pydantic",
|
||||||
|
"pyproj",
|
||||||
|
"rasterio",
|
||||||
|
"shapely",
|
||||||
|
"sqlalchemy",
|
||||||
|
"torch",
|
||||||
|
"ultralytics",
|
||||||
|
"uvicorn",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def utc_now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def run_command(
|
||||||
|
command: Sequence[str],
|
||||||
|
*,
|
||||||
|
cwd: Path = ROOT,
|
||||||
|
timeout_seconds: int = 30,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
list(command),
|
||||||
|
cwd=cwd,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
timeout=timeout_seconds,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||||
|
return {
|
||||||
|
"ok": False,
|
||||||
|
"exit_code": None,
|
||||||
|
"stdout": "",
|
||||||
|
"stderr": str(exc),
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"ok": result.returncode == 0,
|
||||||
|
"exit_code": result.returncode,
|
||||||
|
"stdout": result.stdout.strip(),
|
||||||
|
"stderr": result.stderr.strip(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def git_evidence() -> dict[str, Any]:
|
||||||
|
commit = run_command(("git", "rev-parse", "HEAD"))
|
||||||
|
branch = run_command(("git", "branch", "--show-current"))
|
||||||
|
status = run_command(("git", "status", "--porcelain=v1"))
|
||||||
|
remote = run_command(("git", "remote", "get-url", "origin"))
|
||||||
|
dirty_paths = [
|
||||||
|
line[3:].strip()
|
||||||
|
for line in status["stdout"].splitlines()
|
||||||
|
if len(line) >= 4
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"commit": commit["stdout"] or None,
|
||||||
|
"branch": branch["stdout"] or None,
|
||||||
|
"origin": remote["stdout"] or None,
|
||||||
|
"dirty": bool(dirty_paths),
|
||||||
|
"dirty_paths": dirty_paths,
|
||||||
|
"commands_ok": all(item["ok"] for item in (commit, branch, status)),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def migration_evidence() -> dict[str, Any]:
|
||||||
|
heads = run_command((sys.executable, "-m", "alembic", "heads"), cwd=BACKEND)
|
||||||
|
head_lines = [
|
||||||
|
line.strip()
|
||||||
|
for line in heads["stdout"].splitlines()
|
||||||
|
if line.strip()
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"command_ok": heads["ok"],
|
||||||
|
"heads": head_lines,
|
||||||
|
"single_head": heads["ok"] and len(head_lines) == 1,
|
||||||
|
"error": heads["stderr"] or None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def dependency_evidence() -> dict[str, str | None]:
|
||||||
|
versions: dict[str, str | None] = {}
|
||||||
|
for name in DEPENDENCIES:
|
||||||
|
try:
|
||||||
|
versions[name] = importlib.metadata.version(name)
|
||||||
|
except importlib.metadata.PackageNotFoundError:
|
||||||
|
versions[name] = None
|
||||||
|
return versions
|
||||||
|
|
||||||
|
|
||||||
|
def file_evidence() -> dict[str, dict[str, Any]]:
|
||||||
|
evidence: dict[str, dict[str, Any]] = {}
|
||||||
|
for relative_path in DEFAULT_HASHED_FILES:
|
||||||
|
path = ROOT / relative_path
|
||||||
|
evidence[relative_path] = {
|
||||||
|
"exists": path.is_file(),
|
||||||
|
"size_bytes": path.stat().st_size if path.is_file() else None,
|
||||||
|
"sha256": sha256_file(path) if path.is_file() else None,
|
||||||
|
}
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
|
||||||
|
def storage_evidence(storage_root: Path | None) -> dict[str, Any]:
|
||||||
|
if storage_root is None:
|
||||||
|
return {"requested": False}
|
||||||
|
resolved = storage_root.expanduser().resolve()
|
||||||
|
if not resolved.is_dir():
|
||||||
|
return {
|
||||||
|
"requested": True,
|
||||||
|
"root": str(resolved),
|
||||||
|
"available": False,
|
||||||
|
}
|
||||||
|
total_size = 0
|
||||||
|
file_count = 0
|
||||||
|
errors: list[str] = []
|
||||||
|
for path in resolved.rglob("*"):
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
total_size += path.stat().st_size
|
||||||
|
file_count += 1
|
||||||
|
except OSError as exc:
|
||||||
|
errors.append(f"{path}: {exc}")
|
||||||
|
if len(errors) >= 20:
|
||||||
|
break
|
||||||
|
return {
|
||||||
|
"requested": True,
|
||||||
|
"root": str(resolved),
|
||||||
|
"available": True,
|
||||||
|
"file_count": file_count,
|
||||||
|
"size_bytes": total_size,
|
||||||
|
"scan_complete": not errors,
|
||||||
|
"errors": errors,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_json(url: str, timeout_seconds: int) -> dict[str, Any]:
|
||||||
|
request = urllib.request.Request(
|
||||||
|
url,
|
||||||
|
headers={"Accept": "application/json", "User-Agent": "GeoIntel-RC-Evidence/1.0"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
|
||||||
|
body = response.read(2 * 1024 * 1024 + 1)
|
||||||
|
if len(body) > 2 * 1024 * 1024:
|
||||||
|
raise ValueError("response exceeds 2 MiB evidence limit")
|
||||||
|
return {
|
||||||
|
"ok": 200 <= response.status < 300,
|
||||||
|
"status_code": response.status,
|
||||||
|
"payload": json.loads(body.decode("utf-8")),
|
||||||
|
"error": None,
|
||||||
|
}
|
||||||
|
except (urllib.error.URLError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
return {
|
||||||
|
"ok": False,
|
||||||
|
"status_code": getattr(exc, "code", None),
|
||||||
|
"payload": None,
|
||||||
|
"error": str(exc),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def live_evidence(base_url: str | None, timeout_seconds: int) -> dict[str, Any]:
|
||||||
|
if not base_url:
|
||||||
|
return {"requested": False}
|
||||||
|
normalized = base_url.rstrip("/")
|
||||||
|
return {
|
||||||
|
"requested": True,
|
||||||
|
"base_url": normalized,
|
||||||
|
"health": fetch_json(f"{normalized}/health", timeout_seconds),
|
||||||
|
"capabilities": fetch_json(
|
||||||
|
f"{normalized}/api/v1/system/capabilities",
|
||||||
|
timeout_seconds,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build_manifest(args: argparse.Namespace) -> dict[str, Any]:
|
||||||
|
storage_root = Path(args.storage_root) if args.storage_root else None
|
||||||
|
return {
|
||||||
|
"schema_version": 1,
|
||||||
|
"release_id": args.release_id,
|
||||||
|
"captured_at": utc_now(),
|
||||||
|
"read_only": True,
|
||||||
|
"scope": "Belgium and the Belgian North Sea",
|
||||||
|
"host": {
|
||||||
|
"platform": platform.platform(),
|
||||||
|
"python": platform.python_version(),
|
||||||
|
"hostname": platform.node(),
|
||||||
|
},
|
||||||
|
"git": git_evidence(),
|
||||||
|
"migrations": migration_evidence(),
|
||||||
|
"dependencies": dependency_evidence(),
|
||||||
|
"files": file_evidence(),
|
||||||
|
"storage": storage_evidence(storage_root),
|
||||||
|
"live": live_evidence(args.live_base_url, args.timeout_seconds),
|
||||||
|
"environment_presence": {
|
||||||
|
"DATABASE_URL": bool(os.environ.get("DATABASE_URL")),
|
||||||
|
"STORAGE_ROOT": bool(os.environ.get("STORAGE_ROOT")),
|
||||||
|
"YOLO_ENABLED": bool(os.environ.get("YOLO_ENABLED")),
|
||||||
|
"YOLO_MODEL_PATH": bool(os.environ.get("YOLO_MODEL_PATH")),
|
||||||
|
"OLLAMA_ENABLED": bool(os.environ.get("OLLAMA_ENABLED")),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--output", type=Path, required=True)
|
||||||
|
parser.add_argument("--release-id", default="rc-current")
|
||||||
|
parser.add_argument("--live-base-url")
|
||||||
|
parser.add_argument("--storage-root")
|
||||||
|
parser.add_argument("--timeout-seconds", type=int, default=10)
|
||||||
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: Sequence[str] | None = None) -> int:
|
||||||
|
args = parse_args(argv)
|
||||||
|
if args.timeout_seconds < 1 or args.timeout_seconds > 120:
|
||||||
|
raise SystemExit("--timeout-seconds must be between 1 and 120")
|
||||||
|
manifest = build_manifest(args)
|
||||||
|
output = args.output.expanduser().resolve()
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
output.write_text(
|
||||||
|
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
print(f"Wrote read-only release evidence to {output}")
|
||||||
|
if not manifest["git"]["commands_ok"]:
|
||||||
|
return 2
|
||||||
|
if not manifest["migrations"]["single_head"]:
|
||||||
|
return 3
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
CONTAINER="geointel"
|
||||||
|
BACKUP_DIR=""
|
||||||
|
CONFIRM="false"
|
||||||
|
KEEP_DATABASE="false"
|
||||||
|
OUTPUT=""
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: bash scripts/restore_release_backup_smoke.sh --backup-dir PATH \
|
||||||
|
--confirm-isolated-restore [options]
|
||||||
|
|
||||||
|
Restores into a generated geointel_restore_verify_* database only, verifies
|
||||||
|
PostGIS, Alembic and retained table counts, then drops that temporary database.
|
||||||
|
It refuses the production database name and never uses pg_restore --clean.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--container NAME
|
||||||
|
--output PATH Write result JSON outside the immutable backup directory
|
||||||
|
--keep-database Keep the isolated verification database for inspection
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
while [ "$#" -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--backup-dir) BACKUP_DIR="$2"; shift 2 ;;
|
||||||
|
--container) CONTAINER="$2"; shift 2 ;;
|
||||||
|
--confirm-isolated-restore) CONFIRM="true"; shift ;;
|
||||||
|
--keep-database) KEEP_DATABASE="true"; shift ;;
|
||||||
|
--output) OUTPUT="$2"; shift 2 ;;
|
||||||
|
--help|-h) usage; exit 0 ;;
|
||||||
|
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$CONFIRM" != "true" ] || [ -z "$BACKUP_DIR" ]; then
|
||||||
|
echo "Explicit --confirm-isolated-restore and --backup-dir are required." >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
for required in docker python3; do
|
||||||
|
command -v "$required" >/dev/null 2>&1 || {
|
||||||
|
echo "Missing required command: $required" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
done
|
||||||
|
BACKUP_DIR="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$BACKUP_DIR")"
|
||||||
|
bash "$(dirname "$0")/verify_release_backup.sh" --backup-dir "$BACKUP_DIR" --container "$CONTAINER"
|
||||||
|
|
||||||
|
DB_NAME="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"' )"
|
||||||
|
DB_USER="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"' )"
|
||||||
|
TARGET_DB="geointel_restore_verify_$(date -u +%Y%m%d%H%M%S)_$$"
|
||||||
|
if [ "$TARGET_DB" = "$DB_NAME" ] || ! [[ "$TARGET_DB" =~ ^geointel_restore_verify_[0-9_]+$ ]]; then
|
||||||
|
echo "Unsafe restore target: $TARGET_DB" >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
if docker exec "$CONTAINER" psql -X -U "$DB_USER" -d postgres -Atqc \
|
||||||
|
"SELECT 1 FROM pg_database WHERE datname = '$TARGET_DB';" | grep -q 1; then
|
||||||
|
echo "Generated restore database already exists." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
cleanup_database() {
|
||||||
|
if [ "$KEEP_DATABASE" != "true" ]; then
|
||||||
|
docker exec "$CONTAINER" dropdb --if-exists -U "$DB_USER" "$TARGET_DB" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap cleanup_database EXIT
|
||||||
|
|
||||||
|
docker exec "$CONTAINER" createdb -U "$DB_USER" "$TARGET_DB"
|
||||||
|
docker exec -i "$CONTAINER" pg_restore \
|
||||||
|
--exit-on-error \
|
||||||
|
--no-owner \
|
||||||
|
--no-privileges \
|
||||||
|
-U "$DB_USER" \
|
||||||
|
-d "$TARGET_DB" < "$BACKUP_DIR/database.dump"
|
||||||
|
|
||||||
|
POSTGIS_VERSION="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$TARGET_DB" -Atqc \
|
||||||
|
"SELECT postgis_version();")"
|
||||||
|
ALEMBIC_HEAD="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$TARGET_DB" -Atqc \
|
||||||
|
"SELECT version_num FROM alembic_version;")"
|
||||||
|
EXPECTED_HEAD="$(awk -F $'\t' '$1 == "alembic_head" { print $2 }' "$BACKUP_DIR/database-metadata.tsv")"
|
||||||
|
if [ "$ALEMBIC_HEAD" != "$EXPECTED_HEAD" ]; then
|
||||||
|
echo "Restored Alembic head '$ALEMBIC_HEAD' differs from '$EXPECTED_HEAD'." >&2
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
|
|
||||||
|
while IFS=$'\t' read -r table expected; do
|
||||||
|
[[ "$table" =~ ^[a-z_]+$ ]] || {
|
||||||
|
echo "Unsafe table name in retained counts: $table" >&2
|
||||||
|
exit 4
|
||||||
|
}
|
||||||
|
actual="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$TARGET_DB" -Atqc \
|
||||||
|
"SELECT count(*) FROM public.${table};")"
|
||||||
|
if [ "$actual" != "$expected" ]; then
|
||||||
|
echo "Restored count mismatch for $table: expected $expected, got $actual." >&2
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
|
done < "$BACKUP_DIR/table-counts.tsv"
|
||||||
|
|
||||||
|
if [ -z "$OUTPUT" ]; then
|
||||||
|
OUTPUT="${BACKUP_DIR%/}-restore-smoke.json"
|
||||||
|
fi
|
||||||
|
python3 - "$OUTPUT" <<PY
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"verified_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||||
|
"backup_dir": ${BACKUP_DIR@Q},
|
||||||
|
"temporary_database": ${TARGET_DB@Q},
|
||||||
|
"production_database_untouched": True,
|
||||||
|
"temporary_database_retained": ${KEEP_DATABASE@Q} == "true",
|
||||||
|
"alembic_head": ${ALEMBIC_HEAD@Q},
|
||||||
|
"postgis_version": ${POSTGIS_VERSION@Q},
|
||||||
|
"table_counts_match": True,
|
||||||
|
}
|
||||||
|
path = pathlib.Path(__import__("sys").argv[1]).expanduser().resolve()
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
PY
|
||||||
|
|
||||||
|
echo "Isolated restore verified: $TARGET_DB"
|
||||||
|
if [ "$KEEP_DATABASE" != "true" ]; then
|
||||||
|
cleanup_database
|
||||||
|
trap - EXIT
|
||||||
|
echo "Temporary verification database removed."
|
||||||
|
else
|
||||||
|
trap - EXIT
|
||||||
|
echo "Temporary verification database retained by explicit request: $TARGET_DB"
|
||||||
|
fi
|
||||||
@@ -27,6 +27,11 @@ if [ -z "${PYTHON_BIN}" ]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
echo "== GeoIntel run readiness check =="
|
echo "== GeoIntel run readiness check =="
|
||||||
|
|
||||||
|
"$PYTHON_BIN" -m py_compile scripts/capture_release_evidence.py
|
||||||
|
bash -n scripts/backup_release_state.sh
|
||||||
|
bash -n scripts/verify_release_backup.sh
|
||||||
|
bash -n scripts/restore_release_backup_smoke.sh
|
||||||
echo "Using Python: ${PYTHON_BIN}"
|
echo "Using Python: ${PYTHON_BIN}"
|
||||||
bash scripts/check_repo_structure.sh
|
bash scripts/check_repo_structure.sh
|
||||||
${PYTHON_BIN} scripts/smoke_docs.py
|
${PYTHON_BIN} scripts/smoke_docs.py
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
CONTAINER="geointel"
|
||||||
|
BACKUP_DIR=""
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: bash scripts/verify_release_backup.sh --backup-dir PATH [--container NAME]
|
||||||
|
|
||||||
|
Verifies checksums, manifest shape and PostgreSQL archive readability. This
|
||||||
|
command is read-only and never creates, restores or drops a database.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
while [ "$#" -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--backup-dir) BACKUP_DIR="$2"; shift 2 ;;
|
||||||
|
--container) CONTAINER="$2"; shift 2 ;;
|
||||||
|
--help|-h) usage; exit 0 ;;
|
||||||
|
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z "$BACKUP_DIR" ]; then
|
||||||
|
usage >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
for required in docker python3 sha256sum; do
|
||||||
|
command -v "$required" >/dev/null 2>&1 || {
|
||||||
|
echo "Missing required command: $required" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
done
|
||||||
|
BACKUP_DIR="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$BACKUP_DIR")"
|
||||||
|
for required_file in manifest.json database.dump database.list database-metadata.tsv table-counts.tsv CHECKSUMS.sha256; do
|
||||||
|
test -s "$BACKUP_DIR/$required_file" || {
|
||||||
|
echo "Missing or empty backup artifact: $required_file" >&2
|
||||||
|
exit 3
|
||||||
|
}
|
||||||
|
done
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$BACKUP_DIR"
|
||||||
|
sha256sum -c CHECKSUMS.sha256
|
||||||
|
)
|
||||||
|
|
||||||
|
python3 - "$BACKUP_DIR/manifest.json" <<'PY'
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
payload = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))
|
||||||
|
required = {
|
||||||
|
"schema_version",
|
||||||
|
"release_id",
|
||||||
|
"created_at",
|
||||||
|
"read_only_source",
|
||||||
|
"database_name",
|
||||||
|
"database_user",
|
||||||
|
"image_id",
|
||||||
|
"git_commit",
|
||||||
|
}
|
||||||
|
missing = sorted(required - payload.keys())
|
||||||
|
if missing:
|
||||||
|
raise SystemExit(f"Backup manifest misses: {', '.join(missing)}")
|
||||||
|
if payload["schema_version"] != 1 or payload["read_only_source"] is not True:
|
||||||
|
raise SystemExit("Unsupported or unsafe backup manifest")
|
||||||
|
PY
|
||||||
|
|
||||||
|
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||||
|
echo "Container '$CONTAINER' is required to run pg_restore --list." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
TMP_LIST="$(mktemp)"
|
||||||
|
trap 'rm -f -- "$TMP_LIST"' EXIT
|
||||||
|
docker exec -i "$CONTAINER" pg_restore --list \
|
||||||
|
< "$BACKUP_DIR/database.dump" \
|
||||||
|
> "$TMP_LIST"
|
||||||
|
cmp -s "$TMP_LIST" "$BACKUP_DIR/database.list" || {
|
||||||
|
echo "The current pg_restore listing differs from the retained listing." >&2
|
||||||
|
exit 3
|
||||||
|
}
|
||||||
|
grep -q "TABLE DATA" "$TMP_LIST" || {
|
||||||
|
echo "Archive contains no table data entries." >&2
|
||||||
|
exit 3
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "Release backup verified read-only: $BACKUP_DIR"
|
||||||
Reference in New Issue
Block a user