fix(release): make deployment backup and rollback immutable

This commit is contained in:
Jens
2026-08-30 06:00:43 +02:00
parent a0884d64c9
commit c272220277
47 changed files with 3035 additions and 430 deletions
+6
View File
@@ -4,6 +4,7 @@ GEOINTEL_API_PREFIX=/api/v1
DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1 DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1
STORAGE_ROOT=./storage STORAGE_ROOT=./storage
MAX_UPLOAD_MB=500 MAX_UPLOAD_MB=500
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB=64
CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202 CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202
# Optional single-operator access gate. Store only a PBKDF2-SHA256 hash and # Optional single-operator access gate. Store only a PBKDF2-SHA256 hash and
@@ -14,6 +15,11 @@ GEOINTEL_AUTH_USERNAME=
GEOINTEL_AUTH_PASSWORD_HASH= GEOINTEL_AUTH_PASSWORD_HASH=
GEOINTEL_AUTH_SESSION_SECRET= GEOINTEL_AUTH_SESSION_SECRET=
GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200 GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200
GEOINTEL_PUBLIC_BASE_URL=http://localhost:1202
GEOINTEL_AUTHENTIK_ISSUER=
GEOINTEL_AUTHENTIK_CLIENT_ID=
GEOINTEL_AUTHENTIK_CLIENT_SECRET=
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL=
GEOINTEL_GUEST_ACCESS_ENABLED=true GEOINTEL_GUEST_ACCESS_ENABLED=true
GEOINTEL_GUEST_DISPLAY_NAME=Gast GEOINTEL_GUEST_DISPLAY_NAME=Gast
GEOINTEL_GUEST_SESSION_TTL_SECONDS=7200 GEOINTEL_GUEST_SESSION_TTL_SECONDS=7200
+42 -75
View File
@@ -20,12 +20,22 @@ concurrency:
jobs: jobs:
full: full:
name: full name: ${{ inputs.profile || 'full' }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 30 timeout-minutes: 60
steps: steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Validate repository with a bounded profile - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.11"
cache: pip
cache-dependency-path: backend/requirements-ci.lock
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: "20"
cache: npm
cache-dependency-path: frontend/package-lock.json
- name: Validate the requested profile against the real projects
shell: bash shell: bash
env: env:
REQUESTED_PROFILE: ${{ inputs.profile }} REQUESTED_PROFILE: ${{ inputs.profile }}
@@ -42,77 +52,34 @@ jobs:
echo "Unresolved merge markers detected" >&2 echo "Unresolved merge markers detected" >&2
exit 1 exit 1
fi fi
python scripts/verify_repository_layout.py
if [[ -f pyproject.toml || -f requirements.txt ]]; then python -m pip install --disable-pip-version-check --require-hashes -r backend/requirements-ci.lock
# Compile only tracked Python sources. Running compileall after a python -m pip install --disable-pip-version-check --no-deps -e backend
# Node install would otherwise traverse node_modules and turn a (cd frontend && npm ci)
# lightweight baseline into a large runner workload.
git ls-files -z '*.py' | xargs -0 -r python -m py_compile
if [[ -f uv.lock ]]; then
python -m venv "${RUNNER_TEMP}/managed-uv"
uv_python="${RUNNER_TEMP}/managed-uv/bin/python"
"${uv_python}" -m pip install --disable-pip-version-check uv==0.10.0
managed_uv="${RUNNER_TEMP}/managed-uv/bin/uv"
export UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/managed-project-venv"
"${managed_uv}" sync --locked
export PATH="${UV_PROJECT_ENVIRONMENT}/bin:${PATH}"
if [[ "${profile}" == test || "${profile}" == full ]]; then
if "${managed_uv}" run python -c 'import pytest' 2>/dev/null; then
"${managed_uv}" run python -m pytest
fi
fi
if [[ "${profile}" == lint || "${profile}" == full ]]; then
if "${managed_uv}" run python -c 'import ruff' 2>/dev/null; then
"${managed_uv}" run python -m ruff check .
fi
fi
elif [[ -f requirements.txt ]]; then
python -m venv "${RUNNER_TEMP}/managed-python"
managed_python="${RUNNER_TEMP}/managed-python/bin/python"
"${managed_python}" -m pip install --disable-pip-version-check -r requirements.txt
export PATH="${RUNNER_TEMP}/managed-python/bin:${PATH}"
if [[ "${profile}" == test || "${profile}" == full ]]; then
if "${managed_python}" -c 'import pytest' 2>/dev/null; then
"${managed_python}" -m pytest
fi
fi
fi
fi
# Prepare Python before invoking Node scripts. Polyglot repositories case "${profile}" in
# commonly delegate their test script to Python and need the managed test)
# virtual environment to be active first. (cd backend && python -m pytest -W error::DeprecationWarning)
if [[ -f package.json ]]; then (cd frontend && npm run test:unit)
corepack enable ;;
if [[ -f pnpm-lock.yaml ]]; then lint)
pnpm install --frozen-lockfile python -m ruff check backend scripts tests
[[ "${profile}" == test || "${profile}" == full ]] && pnpm --if-present test (cd frontend && npm run lint --if-present)
[[ "${profile}" == lint || "${profile}" == full ]] && pnpm --if-present lint ;;
[[ "${profile}" == typecheck || "${profile}" == full ]] && pnpm --if-present typecheck typecheck)
[[ "${profile}" == build || "${profile}" == full ]] && pnpm --if-present build (cd frontend && npm run typecheck)
elif [[ -f package-lock.json ]]; then ;;
npm ci build)
[[ "${profile}" == test || "${profile}" == full ]] && npm run --if-present test python -m compileall backend/app
[[ "${profile}" == lint || "${profile}" == full ]] && npm run --if-present lint (cd frontend && npm run build)
if [[ "${profile}" == typecheck || "${profile}" == full ]]; then ;;
npm run --if-present typecheck security)
fi python -m pip install --disable-pip-version-check pip-audit==2.10.1
[[ "${profile}" == build || "${profile}" == full ]] && npm run --if-present build bash scripts/audit_python_dependencies.sh
fi (cd frontend && npm audit --audit-level=high)
fi ;;
full)
if [[ -f go.mod ]]; then PYTHON_BIN=python bash scripts/run_readiness_check.sh
if [[ "${profile}" == test || "${profile}" == build || "${profile}" == full ]]; then ;;
go test ./... esac
fi
fi
if [[ -f Cargo.toml ]]; then
if [[ "${profile}" == test || "${profile}" == build || "${profile}" == full ]]; then
cargo test --locked
fi
fi
if compgen -G '*.sln' >/dev/null; then
if [[ "${profile}" == test || "${profile}" == build || "${profile}" == full ]]; then
dotnet test --configuration Release
fi
fi
+46 -19
View File
@@ -1,6 +1,7 @@
name: GeoIntel release gates name: GeoIntel release gates
on: on:
pull_request:
push: push:
branches: [main, develop] branches: [main, develop]
workflow_dispatch: workflow_dispatch:
@@ -16,21 +17,21 @@ jobs:
quality: quality:
name: Compile, test, contracts and builds name: Compile, test, contracts and builds
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 45 timeout-minutes: 60
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Secret scan - name: Secret scan
run: >- run: >-
docker run --rm docker run --rm
--volume "$PWD:/repo:ro" --volume "$PWD:/repo:ro"
trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3 trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3
filesystem /repo --only-verified --no-update filesystem /repo --only-verified --no-update
- uses: actions/setup-python@v5 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with: with:
python-version: "3.11" python-version: "3.11"
cache: pip cache: pip
cache-dependency-path: backend/requirements-ci.lock cache-dependency-path: backend/requirements-ci.lock
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -57,7 +58,7 @@ jobs:
docker compose config > artifacts/docker-compose.resolved.yml docker compose config > artifacts/docker-compose.resolved.yml
- name: Publish quality evidence - name: Publish quality evidence
if: always() if: always()
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with: with:
name: quality-evidence name: quality-evidence
path: | path: |
@@ -71,13 +72,13 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 20 timeout-minutes: 20
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@v5 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with: with:
python-version: "3.11" python-version: "3.11"
cache: pip cache: pip
cache-dependency-path: backend/requirements-ci.lock cache-dependency-path: backend/requirements-ci.lock
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -94,7 +95,7 @@ jobs:
npm audit --audit-level=high --json > ../artifacts/npm-audit.json npm audit --audit-level=high --json > ../artifacts/npm-audit.json
- name: Publish dependency evidence - name: Publish dependency evidence
if: always() if: always()
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with: with:
name: dependency-audits name: dependency-audits
path: | path: |
@@ -105,41 +106,67 @@ jobs:
retention-days: 30 retention-days: 30
container: container:
name: GIS image, SBOM and container scan name: Production AI image, SBOM and container scan
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 60 timeout-minutes: 120
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Build non-AI release image - name: Build production AI release image
env: env:
RELEASE_SHA: ${{ gitea.sha }} RELEASE_SHA: ${{ gitea.sha }}
run: | run: |
mkdir -p artifacts mkdir -p artifacts
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
APP_VERSION="$(tr -d '[:space:]' < VERSION)"
docker build \ docker build \
-f deploy/unraid/Dockerfile.all-in-one \ -f deploy/unraid/Dockerfile.all-in-one \
--build-arg GEOINTEL_INSTALL_AI=false \ --build-arg GEOINTEL_INSTALL_AI=true \
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \ --build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \ --build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
-t "geointel-ci:$RELEASE_SHA-gis" \ --build-arg GEOINTEL_APP_VERSION="$APP_VERSION" \
-t "geointel-ci:$RELEASE_SHA-ai" \
. .
docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json IMAGE_ID="$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")"
printf '%s\n' "$IMAGE_ID" > artifacts/image-id.txt
docker image inspect "$IMAGE_ID" > artifacts/image-inspect.json
- name: Generate SPDX SBOM - name: Generate SPDX SBOM
env: env:
RELEASE_SHA: ${{ gitea.sha }} RELEASE_SHA: ${{ gitea.sha }}
run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis" run: |
IMAGE_ID="$(cat artifacts/image-id.txt)"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
bash scripts/generate_container_sbom.sh "$IMAGE_ID"
- name: Enforce container vulnerability policy - name: Enforce container vulnerability policy
env: env:
RELEASE_SHA: ${{ gitea.sha }} RELEASE_SHA: ${{ gitea.sha }}
run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis" run: |
IMAGE_ID="$(cat artifacts/image-id.txt)"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
bash scripts/scan_container_image.sh "$IMAGE_ID"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
- name: Publish container evidence - name: Publish container evidence
if: always() if: always()
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with: with:
name: container-evidence name: container-evidence
path: | path: |
artifacts/image-inspect.json artifacts/image-inspect.json
artifacts/image-id.txt
artifacts/geointel-sbom.spdx.json artifacts/geointel-sbom.spdx.json
artifacts/geointel-container-vulnerabilities.json artifacts/geointel-container-vulnerabilities.json
if-no-files-found: warn if-no-files-found: warn
retention-days: 30 retention-days: 30
deploy:
name: Deploy exact gated revision to Unraid
needs: [quality, dependency-audit, container]
if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }}
runs-on: unraid-deploy
timeout-minutes: 180
steps:
- name: Deploy only after every release gate is green
run: |
set -euo pipefail
docker exec gitea-deploy-control \
/opt/gitea-deploy/deploy.py deploy \
"${{ gitea.repository }}" "${{ gitea.sha }}"
-31
View File
@@ -1,31 +0,0 @@
name: Unraid autoredeploy
on:
push:
branches: [main]
paths-ignore:
- ".gitea/**"
- "docs/**"
- "**/*.md"
workflow_dispatch:
concurrency:
group: unraid-production-geointel
cancel-in-progress: false
permissions:
contents: read
jobs:
deploy:
name: Deploy geointel
runs-on: unraid-deploy
timeout-minutes: 180
steps:
- name: Deploy exact Gitea revision
run: |
set -euo pipefail
docker exec gitea-deploy-control \
/opt/gitea-deploy/deploy.py deploy \
"$GITHUB_REPOSITORY" "$GITHUB_SHA"
+30 -18
View File
@@ -20,19 +20,19 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 45 timeout-minutes: 45
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Secret scan - name: Secret scan
run: >- run: >-
docker run --rm docker run --rm
--volume "$PWD:/repo:ro" --volume "$PWD:/repo:ro"
trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3 trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3
filesystem /repo --only-verified --no-update filesystem /repo --only-verified --no-update
- uses: actions/setup-python@v5 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with: with:
python-version: "3.11" python-version: "3.11"
cache: pip cache: pip
cache-dependency-path: backend/requirements-ci.lock cache-dependency-path: backend/requirements-ci.lock
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -59,7 +59,7 @@ jobs:
docker compose config > artifacts/docker-compose.resolved.yml docker compose config > artifacts/docker-compose.resolved.yml
- name: Publish quality evidence - name: Publish quality evidence
if: always() if: always()
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with: with:
name: quality-evidence name: quality-evidence
path: | path: |
@@ -73,13 +73,13 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 20 timeout-minutes: 20
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@v5 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with: with:
python-version: "3.11" python-version: "3.11"
cache: pip cache: pip
cache-dependency-path: backend/requirements-ci.lock cache-dependency-path: backend/requirements-ci.lock
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -96,7 +96,7 @@ jobs:
npm audit --audit-level=high --json > ../artifacts/npm-audit.json npm audit --audit-level=high --json > ../artifacts/npm-audit.json
- name: Publish dependency evidence - name: Publish dependency evidence
if: always() if: always()
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with: with:
name: dependency-audits name: dependency-audits
path: | path: |
@@ -107,40 +107,52 @@ jobs:
retention-days: 30 retention-days: 30
container: container:
name: GIS image, SBOM and container scan name: Production AI image, SBOM and container scan
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 60 timeout-minutes: 120
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Build non-AI release image - name: Build production AI release image
env: env:
RELEASE_SHA: ${{ github.sha }} RELEASE_SHA: ${{ github.sha }}
run: | run: |
mkdir -p artifacts mkdir -p artifacts
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
APP_VERSION="$(tr -d '[:space:]' < VERSION)"
docker build \ docker build \
-f deploy/unraid/Dockerfile.all-in-one \ -f deploy/unraid/Dockerfile.all-in-one \
--build-arg GEOINTEL_INSTALL_AI=false \ --build-arg GEOINTEL_INSTALL_AI=true \
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \ --build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \ --build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
-t "geointel-ci:$RELEASE_SHA-gis" \ --build-arg GEOINTEL_APP_VERSION="$APP_VERSION" \
-t "geointel-ci:$RELEASE_SHA-ai" \
. .
docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json IMAGE_ID="$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")"
printf '%s\n' "$IMAGE_ID" > artifacts/image-id.txt
docker image inspect "$IMAGE_ID" > artifacts/image-inspect.json
- name: Generate SPDX SBOM - name: Generate SPDX SBOM
env: env:
RELEASE_SHA: ${{ github.sha }} RELEASE_SHA: ${{ github.sha }}
run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis" run: |
IMAGE_ID="$(cat artifacts/image-id.txt)"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
bash scripts/generate_container_sbom.sh "$IMAGE_ID"
- name: Enforce container vulnerability policy - name: Enforce container vulnerability policy
env: env:
RELEASE_SHA: ${{ github.sha }} RELEASE_SHA: ${{ github.sha }}
run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis" run: |
IMAGE_ID="$(cat artifacts/image-id.txt)"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
bash scripts/scan_container_image.sh "$IMAGE_ID"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
- name: Publish container evidence - name: Publish container evidence
if: always() if: always()
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with: with:
name: container-evidence name: container-evidence
path: | path: |
artifacts/image-inspect.json artifacts/image-inspect.json
artifacts/image-id.txt
artifacts/geointel-sbom.spdx.json artifacts/geointel-sbom.spdx.json
artifacts/geointel-container-vulnerabilities.json artifacts/geointel-container-vulnerabilities.json
if-no-files-found: warn if-no-files-found: warn
+2
View File
@@ -8,3 +8,5 @@ storage
dist dist
node_modules node_modules
.env .env
.env.*
!.env.example
+4 -2
View File
@@ -6,7 +6,7 @@ readme = "README.md"
requires-python = ">=3.11" requires-python = ">=3.11"
dependencies = [ dependencies = [
"fastapi>=0.112.0", "fastapi>=0.112.0",
"starlette>=0.46.0,<1.0.0", "starlette>=1.3.1,<2.0.0",
"uvicorn[standard]>=0.30.6", "uvicorn[standard]>=0.30.6",
"SQLAlchemy>=2.0.34", "SQLAlchemy>=2.0.34",
"psycopg[binary]>=3.2.1", "psycopg[binary]>=3.2.1",
@@ -16,6 +16,8 @@ dependencies = [
"shapely>=2.0.4", "shapely>=2.0.4",
"pyproj>=3.6.1", "pyproj>=3.6.1",
"python-multipart>=0.0.9", "python-multipart>=0.0.9",
"itsdangerous>=2.2.0",
"PyJWT[crypto]>=2.10.1",
"rdflib>=7.1,<8", "rdflib>=7.1,<8",
"alembic>=1.13.2", "alembic>=1.13.2",
] ]
@@ -37,7 +39,7 @@ ai = [
"ultralytics>=8.3,<9", "ultralytics>=8.3,<9",
"torch>=2.4", "torch>=2.4",
] ]
dev = ["pytest>=8.3.2", "httpx>=0.27.0", "ruff>=0.6.9"] dev = ["pytest>=8.3.2", "httpx>=0.27.0", "httpx2>=2.0.0", "ruff>=0.6.9"]
[project.scripts] [project.scripts]
geointel-backend = "app.main:main" geointel-backend = "app.main:main"
+187 -5
View File
@@ -2,9 +2,9 @@
# This file is autogenerated by pip-compile with Python 3.11 # This file is autogenerated by pip-compile with Python 3.11
# by the following command: # by the following command:
# #
# pip-compile --extra=dev --extra=gis --generate-hashes --output-file=requirements-ci.lock --strip-extras pyproject.toml # pip-compile --extra=dev --extra=gis --generate-hashes --no-index --output-file=requirements-ci.lock --strip-extras pyproject.toml
# #
# geointel-input-sha256: 03c20efedd96474cbe62591b7b70cdad2681688b618bdd76731bd4cfaf85b3d4 # geointel-input-sha256: e1dd11f5b30f4c8c902f33476282da48da386d36093cab68212edc69e75df8e3
affine==2.4.0 \ affine==2.4.0 \
--hash=sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92 \ --hash=sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92 \
--hash=sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea --hash=sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea
@@ -26,6 +26,7 @@ anyio==4.14.2 \
--hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f
# via # via
# httpx # httpx
# httpx2
# starlette # starlette
# watchfiles # watchfiles
attrs==26.1.0 \ attrs==26.1.0 \
@@ -41,6 +42,108 @@ certifi==2026.6.17 \
# pyogrio # pyogrio
# pyproj # pyproj
# rasterio # rasterio
cffi==2.1.1 \
--hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \
--hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \
--hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \
--hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \
--hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \
--hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \
--hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \
--hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \
--hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \
--hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \
--hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \
--hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \
--hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \
--hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \
--hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \
--hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \
--hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \
--hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \
--hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \
--hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \
--hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \
--hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \
--hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \
--hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \
--hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \
--hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \
--hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \
--hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \
--hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \
--hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \
--hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \
--hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \
--hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \
--hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \
--hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \
--hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \
--hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \
--hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \
--hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \
--hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \
--hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \
--hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \
--hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \
--hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \
--hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \
--hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \
--hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \
--hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \
--hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \
--hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \
--hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \
--hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \
--hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \
--hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \
--hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \
--hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \
--hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \
--hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \
--hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \
--hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \
--hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \
--hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \
--hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \
--hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \
--hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \
--hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \
--hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \
--hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \
--hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \
--hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \
--hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \
--hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \
--hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \
--hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \
--hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \
--hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \
--hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \
--hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \
--hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \
--hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \
--hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \
--hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \
--hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \
--hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \
--hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \
--hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \
--hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \
--hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \
--hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \
--hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \
--hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \
--hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \
--hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \
--hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \
--hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \
--hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \
--hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \
--hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \
--hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \
--hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264
# via cryptography
click==8.4.2 \ click==8.4.2 \
--hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \
--hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76
@@ -57,6 +160,54 @@ cligj==0.7.2 \
--hash=sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27 \ --hash=sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27 \
--hash=sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df --hash=sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df
# via rasterio # via rasterio
cryptography==50.0.1 \
--hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \
--hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \
--hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \
--hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \
--hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \
--hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \
--hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \
--hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \
--hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \
--hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \
--hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \
--hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \
--hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \
--hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \
--hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \
--hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \
--hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \
--hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \
--hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \
--hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \
--hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \
--hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \
--hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \
--hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \
--hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \
--hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \
--hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \
--hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \
--hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \
--hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \
--hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \
--hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \
--hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \
--hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \
--hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \
--hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \
--hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \
--hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \
--hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \
--hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \
--hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \
--hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \
--hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \
--hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \
--hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \
--hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef
# via pyjwt
fastapi==0.139.2 \ fastapi==0.139.2 \
--hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \ --hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \
--hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c --hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c
@@ -155,11 +306,16 @@ h11==0.16.0 \
--hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
# via # via
# httpcore # httpcore
# httpcore2
# uvicorn # uvicorn
httpcore==1.0.9 \ httpcore==1.0.9 \
--hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
--hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
# via httpx # via httpx
httpcore2==2.12.0 \
--hash=sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb \
--hash=sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648
# via httpx2
httptools==0.8.0 \ httptools==0.8.0 \
--hash=sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683 \ --hash=sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683 \
--hash=sha256:0ea897f0c729581ebf72131a438a7932d9b14efef72d75ada966700cac3caaeb \ --hash=sha256:0ea897f0c729581ebf72131a438a7932d9b14efef72d75ada966700cac3caaeb \
@@ -216,16 +372,25 @@ httpx==0.28.1 \
--hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
--hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
# via geointel-backend (pyproject.toml) # via geointel-backend (pyproject.toml)
httpx2==2.12.0 \
--hash=sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf \
--hash=sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36
# via geointel-backend (pyproject.toml)
idna==3.18 \ idna==3.18 \
--hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \
--hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848
# via # via
# anyio # anyio
# httpx # httpx
# httpx2
iniconfig==2.3.0 \ iniconfig==2.3.0 \
--hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \
--hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12
# via pytest # via pytest
itsdangerous==2.2.0 \
--hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \
--hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173
# via geointel-backend (pyproject.toml)
mako==1.3.12 \ mako==1.3.12 \
--hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \ --hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \
--hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a --hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a
@@ -613,6 +778,10 @@ psycopg-binary==3.3.4 \
--hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \ --hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \
--hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7 --hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7
# via psycopg # via psycopg
pycparser==3.0 \
--hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \
--hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992
# via cffi
pydantic==2.13.4 \ pydantic==2.13.4 \
--hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \ --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
--hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6 --hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
@@ -750,6 +919,12 @@ pygments==2.20.0 \
--hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \
--hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176
# via pytest # via pytest
pyjwt==2.13.0 \
--hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \
--hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728
# via
# geointel-backend (pyproject.toml)
# pyjwt
pyogrio==0.13.0 \ pyogrio==0.13.0 \
--hash=sha256:1b91f6d6e6757a6ea84b9459d24f479dcb52bbf4ebcdb16baf39e49d2836a1cf \ --hash=sha256:1b91f6d6e6757a6ea84b9459d24f479dcb52bbf4ebcdb16baf39e49d2836a1cf \
--hash=sha256:220a988ce2a26591d6db5c775b07289d4f54cabdf274cc048f0e17a0b9d5be14 \ --hash=sha256:220a988ce2a26591d6db5c775b07289d4f54cabdf274cc048f0e17a0b9d5be14 \
@@ -1127,12 +1302,18 @@ sqlalchemy==2.0.51 \
# alembic # alembic
# geoalchemy2 # geoalchemy2
# geointel-backend (pyproject.toml) # geointel-backend (pyproject.toml)
starlette==0.52.1 \ starlette==1.6.0 \
--hash=sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74 \ --hash=sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c \
--hash=sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933 --hash=sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b
# via # via
# fastapi # fastapi
# geointel-backend (pyproject.toml) # geointel-backend (pyproject.toml)
truststore==0.10.4 \
--hash=sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301 \
--hash=sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981
# via
# httpcore2
# httpx2
typing-extensions==4.16.0 \ typing-extensions==4.16.0 \
--hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
--hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
@@ -1140,6 +1321,7 @@ typing-extensions==4.16.0 \
# alembic # alembic
# anyio # anyio
# fastapi # fastapi
# httpx2
# psycopg # psycopg
# pydantic # pydantic
# pydantic-core # pydantic-core
+169 -5
View File
@@ -2,9 +2,9 @@
# This file is autogenerated by pip-compile with Python 3.11 # This file is autogenerated by pip-compile with Python 3.11
# by the following command: # by the following command:
# #
# pip-compile --extra=gis --generate-hashes --output-file=requirements-runtime.lock --strip-extras pyproject.toml # pip-compile --extra=gis --generate-hashes --no-index --output-file=requirements-runtime.lock --strip-extras pyproject.toml
# #
# geointel-input-sha256: 0d0d2cdceb01da58354610f03b5ce244523130cf9fd29e3f8988dbe684838e8b # geointel-input-sha256: 30318170074aad9b91f570cccc574eaaad0eded5814a464d1207a94d2e1d3317
affine==2.4.0 \ affine==2.4.0 \
--hash=sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92 \ --hash=sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92 \
--hash=sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea --hash=sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea
@@ -38,6 +38,108 @@ certifi==2026.6.17 \
# pyogrio # pyogrio
# pyproj # pyproj
# rasterio # rasterio
cffi==2.1.1 \
--hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \
--hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \
--hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \
--hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \
--hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \
--hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \
--hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \
--hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \
--hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \
--hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \
--hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \
--hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \
--hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \
--hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \
--hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \
--hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \
--hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \
--hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \
--hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \
--hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \
--hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \
--hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \
--hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \
--hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \
--hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \
--hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \
--hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \
--hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \
--hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \
--hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \
--hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \
--hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \
--hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \
--hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \
--hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \
--hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \
--hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \
--hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \
--hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \
--hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \
--hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \
--hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \
--hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \
--hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \
--hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \
--hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \
--hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \
--hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \
--hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \
--hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \
--hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \
--hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \
--hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \
--hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \
--hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \
--hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \
--hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \
--hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \
--hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \
--hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \
--hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \
--hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \
--hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \
--hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \
--hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \
--hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \
--hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \
--hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \
--hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \
--hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \
--hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \
--hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \
--hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \
--hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \
--hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \
--hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \
--hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \
--hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \
--hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \
--hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \
--hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \
--hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \
--hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \
--hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \
--hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \
--hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \
--hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \
--hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \
--hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \
--hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \
--hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \
--hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \
--hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \
--hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \
--hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \
--hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \
--hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \
--hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \
--hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \
--hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264
# via cryptography
click==8.4.2 \ click==8.4.2 \
--hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \
--hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76
@@ -54,6 +156,54 @@ cligj==0.7.2 \
--hash=sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27 \ --hash=sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27 \
--hash=sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df --hash=sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df
# via rasterio # via rasterio
cryptography==50.0.1 \
--hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \
--hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \
--hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \
--hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \
--hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \
--hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \
--hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \
--hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \
--hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \
--hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \
--hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \
--hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \
--hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \
--hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \
--hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \
--hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \
--hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \
--hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \
--hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \
--hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \
--hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \
--hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \
--hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \
--hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \
--hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \
--hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \
--hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \
--hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \
--hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \
--hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \
--hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \
--hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \
--hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \
--hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \
--hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \
--hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \
--hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \
--hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \
--hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \
--hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \
--hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \
--hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \
--hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \
--hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \
--hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \
--hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef
# via pyjwt
fastapi==0.139.2 \ fastapi==0.139.2 \
--hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \ --hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \
--hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c --hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c
@@ -207,6 +357,10 @@ idna==3.18 \
--hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \
--hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848
# via anyio # via anyio
itsdangerous==2.2.0 \
--hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \
--hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173
# via geointel-backend (pyproject.toml)
mako==1.3.12 \ mako==1.3.12 \
--hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \ --hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \
--hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a --hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a
@@ -589,6 +743,10 @@ psycopg-binary==3.3.4 \
--hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \ --hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \
--hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7 --hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7
# via psycopg # via psycopg
pycparser==3.0 \
--hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \
--hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992
# via cffi
pydantic==2.13.4 \ pydantic==2.13.4 \
--hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \ --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
--hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6 --hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
@@ -722,6 +880,12 @@ pydantic-settings==2.14.2 \
--hash=sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440 \ --hash=sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440 \
--hash=sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f --hash=sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f
# via geointel-backend (pyproject.toml) # via geointel-backend (pyproject.toml)
pyjwt==2.13.0 \
--hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \
--hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728
# via
# geointel-backend (pyproject.toml)
# pyjwt
pyogrio==0.13.0 \ pyogrio==0.13.0 \
--hash=sha256:1b91f6d6e6757a6ea84b9459d24f479dcb52bbf4ebcdb16baf39e49d2836a1cf \ --hash=sha256:1b91f6d6e6757a6ea84b9459d24f479dcb52bbf4ebcdb16baf39e49d2836a1cf \
--hash=sha256:220a988ce2a26591d6db5c775b07289d4f54cabdf274cc048f0e17a0b9d5be14 \ --hash=sha256:220a988ce2a26591d6db5c775b07289d4f54cabdf274cc048f0e17a0b9d5be14 \
@@ -1075,9 +1239,9 @@ sqlalchemy==2.0.51 \
# alembic # alembic
# geoalchemy2 # geoalchemy2
# geointel-backend (pyproject.toml) # geointel-backend (pyproject.toml)
starlette==0.52.1 \ starlette==1.6.0 \
--hash=sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74 \ --hash=sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c \
--hash=sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933 --hash=sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b
# via # via
# fastapi # fastapi
# geointel-backend (pyproject.toml) # geointel-backend (pyproject.toml)
@@ -172,6 +172,25 @@ def test_walloon_runtime_settings_are_editable_in_compose_and_unraid() -> None:
assert "WALOUS_MAX_PIXELS" in content assert "WALOUS_MAX_PIXELS" in content
def test_in_memory_vector_limit_is_propagated_and_validated_in_every_runtime() -> None:
expected = "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB"
for path in (
ROOT / ".env.example",
ROOT / "docker-compose.yml",
ROOT / "docker-compose.unraid.yml",
ROOT / "deploy" / "unraid" / "geointel.env.example",
ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml",
):
assert expected in path.read_text(encoding="utf-8"), path
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(
encoding="utf-8"
)
assert 'GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}"' in run_script
assert "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be between 1 and 256." in run_script
assert '-e GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB"' in run_script
def test_frontend_uses_same_origin_api_proxy_by_default() -> None: def test_frontend_uses_same_origin_api_proxy_by_default() -> None:
api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8") api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8") nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
@@ -208,6 +227,45 @@ def test_nginx_runtime_allows_long_ai_and_qa_requests() -> None:
assert "proxy_send_timeout 600s;" in config assert "proxy_send_timeout 600s;" in config
def test_nginx_preserves_outer_https_scheme_for_secure_session_cookies() -> None:
configs = (
(ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"),
(ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8"),
)
for config in configs:
assert "geo $geointel_trusted_forwarder" in config
assert "default 0;" in config
assert "172.16.0.0/12 1;" in config
assert 'map "$geointel_trusted_forwarder:$http_x_forwarded_proto"' in config
assert '"1:https" https;' in config
assert "proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;" in config
assert "proxy_set_header X-Forwarded-Proto $scheme;" not in config
def test_nginx_runtime_sets_security_headers_on_all_cached_locations() -> None:
configs = (
(ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"),
(ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(
encoding="utf-8"
),
)
required = (
'Content-Security-Policy "frame-ancestors \'none\'" always;',
'X-Frame-Options "DENY" always;',
'X-Content-Type-Options "nosniff" always;',
'Referrer-Policy "strict-origin-when-cross-origin" always;',
'Permissions-Policy "camera=(), microphone=(), geolocation=()" always;',
)
for config in configs:
cached_locations = config.count("add_header Cache-Control")
assert cached_locations >= 2
for header in required:
# Nginx 1.27 locations with Cache-Control do not inherit server-level
# add_header directives, so every cached location repeats the policy.
assert config.count(f"add_header {header}") == cached_locations + 1
def test_compose_does_not_publish_postgis_on_default_host_port() -> None: def test_compose_does_not_publish_postgis_on_default_host_port() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
+212 -1
View File
@@ -3,6 +3,8 @@ from __future__ import annotations
import hashlib import hashlib
import importlib.util import importlib.util
import json import json
import os
import subprocess
import sys import sys
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
@@ -18,6 +20,8 @@ SCRIPTS = ROOT / "scripts"
def load_script(name: str): def load_script(name: str):
path = SCRIPTS / name path = SCRIPTS / name
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path) spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
assert spec is not None and spec.loader is not None assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec) module = importlib.util.module_from_spec(spec)
@@ -36,6 +40,9 @@ def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha
"database_password_secure": True, "database_password_secure": True,
"inventory_mode": inventory_mode, "inventory_mode": inventory_mode,
"storage_inventory_requested": True, "storage_inventory_requested": True,
"storage_snapshot_requested": True,
"models_inventory_requested": False,
"models_snapshot_requested": False,
"git_commit": "0123456789abcdef", "git_commit": "0123456789abcdef",
} }
files = { files = {
@@ -48,6 +55,7 @@ def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha
} }
for name, content in files.items(): for name, content in files.items():
(root / name).write_text(content, encoding="utf-8") (root / name).write_text(content, encoding="utf-8")
(root / "storage-snapshot").mkdir()
checksums = [] checksums = []
for name in sorted(files): for name in sorted(files):
digest = hashlib.sha256((root / name).read_bytes()).hexdigest() digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
@@ -337,8 +345,11 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8") readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8") live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
assert "DELETE_STORAGE_ARTIFACTS" in generic assert "QUARANTINE_STORAGE_ARTIFACTS" in generic
assert "verify_current_backup" in generic assert "verify_current_backup" in generic
assert "os.link" in generic
assert 'entry["status"] = "linked"' in generic
assert "cleanup-quarantine" in generic
assert "DELETE_DEMO_EXPORTS" in demo assert "DELETE_DEMO_EXPORTS" in demo
assert "verify_current_backup" in demo assert "verify_current_backup" in demo
assert "/app/backups:ro" in compose assert "/app/backups:ro" in compose
@@ -347,6 +358,8 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
"release_backup_guard.py", "release_backup_guard.py",
"audit_data_operations.py", "audit_data_operations.py",
"cleanup_storage_artifacts.py", "cleanup_storage_artifacts.py",
"restore_storage_quarantine.py",
"release_backup_snapshot.py",
): ):
assert f"COPY scripts/{name}" in dockerfile assert f"COPY scripts/{name}" in dockerfile
assert f"py_compile scripts/{name}" in readiness assert f"py_compile scripts/{name}" in readiness
@@ -356,3 +369,201 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
assert "table-counts-after.tsv" in live_audit assert "table-counts-after.tsv" in live_audit
assert "deleted_count" in live_audit assert "deleted_count" in live_audit
assert "missing_manifest_artifact_count" in live_audit assert "missing_manifest_artifact_count" in live_audit
def test_cleanup_apply_moves_bytes_to_protected_traceable_quarantine(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
monkeypatch.syspath_prepend(str(SCRIPTS))
cleanup = load_script("cleanup_storage_artifacts.py")
storage = tmp_path / "storage"
source = storage / "derived" / "orphan.bin"
source.parent.mkdir(parents=True)
source.write_bytes(b"recoverable-derived-artifact")
candidate = SimpleNamespace(
path=source.resolve(),
relative_path="derived/orphan.bin",
size_bytes=source.stat().st_size,
)
now = datetime.now(timezone.utc)
class SessionContext:
def __enter__(self):
return SimpleNamespace()
def __exit__(self, *_args):
return False
monkeypatch.setattr(
cleanup,
"parse_args",
lambda: SimpleNamespace(
storage_root=storage,
minimum_age_days=7,
max_delete=1,
apply=True,
confirm="QUARANTINE_STORAGE_ARTIFACTS",
backup_dir=tmp_path / "backup",
backup_max_age_hours=24.0,
quarantine_root=None,
),
)
monkeypatch.setattr(cleanup, "SessionLocal", lambda: SessionContext())
monkeypatch.setattr(
cleanup,
"build_report",
lambda *_args, **_kwargs: ({"cleanup": {"protected_prefixes": []}}, [candidate]),
)
monkeypatch.setattr(
cleanup,
"verify_current_backup",
lambda *_args, **_kwargs: SimpleNamespace(
release_id="predeploy-test",
created_at=now,
age_hours=0.1,
backup_tool_revision="0123456789abcdef",
),
)
assert cleanup.main() == 0
payload = json.loads(capsys.readouterr().out)
manifest_path = Path(payload["quarantine_manifest"])
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
quarantined_path = storage / payload["quarantined"][0]["quarantine_relative_path"]
assert not source.exists()
assert quarantined_path.read_bytes() == b"recoverable-derived-artifact"
assert manifest["state"] == "complete"
assert manifest["backup_release_id"] == "predeploy-test"
assert manifest["entries"][0]["status"] == "quarantined"
assert payload["deleted_count"] == 0
restore = subprocess.run(
[
sys.executable,
str(SCRIPTS / "restore_storage_quarantine.py"),
"--storage-root",
str(storage),
"--manifest",
str(manifest_path),
"--confirm",
"RESTORE_QUARANTINED_ARTIFACTS",
],
cwd=ROOT,
capture_output=True,
text=True,
check=False,
)
assert restore.returncode == 0, restore.stderr
assert source.read_bytes() == b"recoverable-derived-artifact"
assert not quarantined_path.exists()
restored_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
assert restored_manifest["state"] == "restored"
assert restored_manifest["entries"][0]["status"] == "restored"
def _write_interrupted_quarantine(
storage: Path,
*,
original_exists: bool,
quarantine_exists: bool,
hard_linked: bool = False,
) -> tuple[Path, Path, Path]:
original = storage / "derived" / "interrupted.bin"
operation = storage / "operator-evidence" / "cleanup-quarantine" / "cleanup-interrupted"
quarantined = operation / "files" / "derived" / "interrupted.bin"
original.parent.mkdir(parents=True, exist_ok=True)
quarantined.parent.mkdir(parents=True, exist_ok=True)
retained = b"interrupted-retained-bytes"
if original_exists:
original.write_bytes(retained)
if quarantine_exists:
if hard_linked:
os.link(original, quarantined)
else:
quarantined.write_bytes(retained)
manifest = operation / "manifest.json"
manifest.write_text(
json.dumps(
{
"schema_version": 1,
"state": "in_progress",
"entries": [
{
"relative_path": "derived/interrupted.bin",
"quarantine_relative_path": quarantined.relative_to(storage).as_posix(),
"size_bytes": len(retained),
"sha256": hashlib.sha256(retained).hexdigest(),
"status": "linked" if hard_linked else "planned",
}
],
}
),
encoding="utf-8",
)
return manifest, original, quarantined
@pytest.mark.parametrize(
("original_exists", "quarantine_exists", "hard_linked"),
((False, True, False), (True, True, True)),
)
def test_quarantine_restore_recovers_each_interrupted_move_window(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
original_exists: bool,
quarantine_exists: bool,
hard_linked: bool,
) -> None:
restore = load_script("restore_storage_quarantine.py")
storage = tmp_path / "storage"
manifest, original, quarantined = _write_interrupted_quarantine(
storage,
original_exists=original_exists,
quarantine_exists=quarantine_exists,
hard_linked=hard_linked,
)
monkeypatch.setattr(
restore,
"parse_args",
lambda: SimpleNamespace(
storage_root=storage,
manifest=manifest,
confirm="RESTORE_QUARANTINED_ARTIFACTS",
),
)
assert restore.main() == 0
assert original.read_bytes() == b"interrupted-retained-bytes"
assert not quarantined.exists()
assert json.loads(manifest.read_text(encoding="utf-8"))["state"] == "restored"
def test_quarantine_restore_never_clobbers_recreated_destination(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
restore = load_script("restore_storage_quarantine.py")
storage = tmp_path / "storage"
manifest, original, quarantined = _write_interrupted_quarantine(
storage,
original_exists=False,
quarantine_exists=True,
)
original.write_bytes(b"new-runtime-bytes")
monkeypatch.setattr(
restore,
"parse_args",
lambda: SimpleNamespace(
storage_root=storage,
manifest=manifest,
confirm="RESTORE_QUARANTINED_ARTIFACTS",
),
)
with pytest.raises(RuntimeError, match="different bytes"):
restore.main()
assert original.read_bytes() == b"new-runtime-bytes"
assert quarantined.read_bytes() == b"interrupted-retained-bytes"
+79 -7
View File
@@ -18,17 +18,16 @@ def test_build_identity_does_not_invalidate_dependency_layers() -> None:
assert 'io.geointel.ai.enabled="${GEOINTEL_INSTALL_AI}"' in dockerfile assert 'io.geointel.ai.enabled="${GEOINTEL_INSTALL_AI}"' in dockerfile
def test_release_deploy_preserves_immutable_and_previous_images() -> None: def test_release_deploy_preserves_immutable_and_backup_specific_rollback_images() -> None:
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8") script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
assert 'GEOINTEL_RELEASE_VARIANT="ai"' in script assert 'GEOINTEL_RELEASE_VARIANT="ai"' in script
assert 'GEOINTEL_RELEASE_VARIANT="gis"' in script assert 'GEOINTEL_RELEASE_VARIANT="gis"' in script
assert 'GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"' in script assert 'GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"' in script
assert 'GEOINTEL_PREVIOUS_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:previous"' in script assert 'GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"' in script
assert 'release_image_id="$(' in script assert 'docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
assert '[ "$current_image_id" != "$release_image_id" ]' in script assert '--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
assert 'docker tag "$current_image_id" "$GEOINTEL_PREVIOUS_IMAGE"' in script assert "GEOINTEL_PREVIOUS_IMAGE" not in script
assert "preserving the existing previous image" in script
assert 'if docker image inspect "$GEOINTEL_RELEASE_IMAGE"' in script assert 'if docker image inspect "$GEOINTEL_RELEASE_IMAGE"' in script
assert "Immutable release tag has conflicting metadata" in script assert "Immutable release tag has conflicting metadata" in script
assert "Reusing existing immutable image" in script assert "Reusing existing immutable image" in script
@@ -36,12 +35,62 @@ def test_release_deploy_preserves_immutable_and_previous_images() -> None:
assert "Deployed immutable image" in script assert "Deployed immutable image" in script
def test_release_creates_verified_backup_before_candidate_migrations() -> None:
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
backup_index = script.index("create_predeploy_backup\n")
scan_index = script.index("scan_release_image\n")
candidate_start_index = script.index('if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"')
assert scan_index < backup_index
assert backup_index < candidate_start_index
assert script.index("docker build") < backup_index
assert "/mnt/user/appdata/geointel/backups" in script
assert "--inventory-mode sha256" in script
assert "scripts/verify_release_backup.sh" in script
assert "refusing an unbacked migration" in script
assert "Quiescing the current backend" in script
assert "restarting the unchanged current release" in script
assert "preflight_backup_capacity" in script
assert "select_verified_link_dest" in script
assert "release_backup_snapshot.py" in script
def test_release_starts_only_the_locally_attested_ai_image() -> None:
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"' in script
assert "Production release deployment requires the gated AI image" in script
assert 'bash scripts/generate_container_sbom.sh "$GEOINTEL_RELEASE_IMAGE"' in script
assert 'bash scripts/scan_container_image.sh "$GEOINTEL_RELEASE_IMAGE"' in script
assert 'running_image_id="$(docker inspect --format \'{{.Image}}\' geointel)"' in script
assert 'if [ "$running_image_id" != "$image" ]' in script
assert "deployment-attestation.json" in script
assert "artifacts/release-evidence/deploy" in script
assert "GITEA_COMMIT_SHA" in script
assert "GITHUB_SHA" in script
assert "must contain one full 40-character Git commit SHA" in script
assert 'running_revision" != "$GEOINTEL_BUILD_SHA"' in script
assert 'running_ai" != "true"' in script
assert '"revision": revision' in script
def test_release_and_container_replacement_are_serialized() -> None: def test_release_and_container_replacement_are_serialized() -> None:
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8") release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8") run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
rollback_script = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
encoding="utf-8"
)
restore_script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
encoding="utf-8"
)
assert "GEOINTEL_DEPLOY_LOCK_FILE" in release_script assert "GEOINTEL_DEPLOY_LOCK_FILE" in release_script
assert "flock -n 9" in release_script assert "flock -n 9" in release_script
assert "GEOINTEL_DEPLOY_LOCK_FILE" in rollback_script
assert "flock -n 9" in rollback_script
assert "GEOINTEL_DEPLOY_LOCK_FILE" in restore_script
assert "flock -n 9" in restore_script
assert "GEOINTEL_DEPLOY_LOCK_HELD=true" in release_script
assert "GEOINTEL_CONTAINER_LOCK_FILE" in run_script assert "GEOINTEL_CONTAINER_LOCK_FILE" in run_script
assert "flock -w 300 8" in run_script assert "flock -w 300 8" in run_script
assert "GeoIntel container removal did not complete within 60 seconds" in run_script assert "GeoIntel container removal did not complete within 60 seconds" in run_script
@@ -89,13 +138,35 @@ def test_manual_rollback_reuses_persistent_paths_and_requires_existing_image() -
rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(encoding="utf-8") rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(encoding="utf-8")
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8") run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
assert "geointel-all-in-one:previous" in rollback assert "Backup manifest lacks an immutable rollback image ID" in rollback
assert 'get("image_id", "")' in rollback
assert 'docker image inspect "$GEOINTEL_ROLLBACK_IMAGE"' in rollback assert 'docker image inspect "$GEOINTEL_ROLLBACK_IMAGE"' in rollback
assert 'GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE"' in rollback assert 'GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE"' in rollback
assert "restore-predeploy-database.sh" in rollback
assert "--confirm-production-database-restore" in rollback
assert "Image-only rollback" in rollback
assert '-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data"' in run_script assert '-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data"' in run_script
assert '-v "${GEOINTEL_STORAGE_PATH}:/app/storage"' in run_script assert '-v "${GEOINTEL_STORAGE_PATH}:/app/storage"' in run_script
def test_same_revision_redeploy_rolls_back_by_backup_bound_image_id() -> None:
release = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
encoding="utf-8"
)
restore = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
encoding="utf-8"
)
backup = (ROOT / "scripts" / "backup_release_state.sh").read_text(encoding="utf-8")
assert "current_image_id" in release
assert "release_image_id" not in release
assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in backup
assert 'if [ -z "$RESTORE_IMAGE" ]; then\n RESTORE_IMAGE="$BACKUP_IMAGE_ID"' in restore
assert 'GEOINTEL_ROLLBACK_IMAGE="$(python3 - "$BACKUP_DIR/manifest.json"' in rollback
assert "geointel-all-in-one:previous" not in release + rollback + restore
def test_readiness_checks_all_release_shell_entrypoints() -> None: def test_readiness_checks_all_release_shell_entrypoints() -> None:
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
@@ -107,6 +178,7 @@ def test_readiness_checks_all_release_shell_entrypoints() -> None:
"deploy/unraid/run-dockerman-container.sh", "deploy/unraid/run-dockerman-container.sh",
"deploy/unraid/deploy-release.sh", "deploy/unraid/deploy-release.sh",
"deploy/unraid/rollback-dockerman-container.sh", "deploy/unraid/rollback-dockerman-container.sh",
"deploy/unraid/restore-predeploy-database.sh",
): ):
assert f"bash -n {path}" in readiness assert f"bash -n {path}" in readiness
+49 -6
View File
@@ -48,13 +48,54 @@ def test_ci_runs_complete_release_and_supply_chain_gates() -> None:
assert "pip-audit==2.10.1" in workflow assert "pip-audit==2.10.1" in workflow
assert "audit_python_dependencies.sh" in workflow assert "audit_python_dependencies.sh" in workflow
assert "npm audit --audit-level=high" in workflow assert "npm audit --audit-level=high" in workflow
assert "GEOINTEL_INSTALL_AI=false" in workflow assert "GEOINTEL_INSTALL_AI=true" in workflow
assert "geointel-ci:$RELEASE_SHA-ai" in workflow
assert "artifacts/image-id.txt" in workflow
assert 'scan_container_image.sh "$IMAGE_ID"' in workflow
assert "generate_container_sbom.sh" in workflow assert "generate_container_sbom.sh" in workflow
assert "scan_container_image.sh" in workflow assert "scan_container_image.sh" in workflow
assert "actions/upload-artifact@v4" in workflow assert "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" in workflow
assert context in workflow assert context in workflow
def test_gitea_production_deploy_depends_on_every_release_gate() -> None:
release = read(".gitea/workflows/release-gates.yml")
legacy_deploy = ROOT / ".gitea" / "workflows" / "unraid-deploy.yml"
assert "pull_request:" in release
assert "needs: [quality, dependency-audit, container]" in release
assert "gitea.event_name == 'push'" in release
assert "gitea.ref == 'refs/heads/main'" in release
assert "/opt/gitea-deploy/deploy.py deploy" in release
assert not legacy_deploy.exists()
assert "workflow_dispatch:" in release
def test_release_workflows_pin_third_party_actions_to_reviewed_commits() -> None:
expected = (
"actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683",
"actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065",
"actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020",
"actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02",
)
for path in (".gitea/workflows/release-gates.yml", ".github/workflows/release-gates.yml"):
workflow = read(path)
for action in expected:
assert action in workflow
def test_managed_validation_targets_the_actual_backend_and_frontend_projects() -> None:
workflow = read(".gitea/workflows/managed-validation.yml")
assert "backend/requirements-ci.lock" in workflow
assert "frontend/package-lock.json" in workflow
assert "python -m pytest -W error::DeprecationWarning" in workflow
assert "cd frontend && npm run test:unit" in workflow
assert "python -m ruff check backend scripts tests" in workflow
assert "python scripts/verify_repository_layout.py" in workflow
assert "[[ -f pyproject.toml" not in workflow
def test_scanner_images_are_versioned_and_digest_pinned() -> None: def test_scanner_images_are_versioned_and_digest_pinned() -> None:
sbom = read("scripts/generate_container_sbom.sh") sbom = read("scripts/generate_container_sbom.sh")
scan = read("scripts/scan_container_image.sh") scan = read("scripts/scan_container_image.sh")
@@ -65,8 +106,9 @@ def test_scanner_images_are_versioned_and_digest_pinned() -> None:
assert "--ignore-unfixed" in scan assert "--ignore-unfixed" in scan
assert "--timeout 20m" in scan assert "--timeout 20m" in scan
assert "--scanners vuln" in scan assert "--scanners vuln" in scan
assert '-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro"' in scan assert 'ignored_container_ids' in scan
assert '--ignorefile "$CONTAINER_IGNORE_FILE"' in scan assert 'ignore_args=(-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro")' in scan
assert 'trivy_ignore_args=(--ignorefile "$CONTAINER_IGNORE_FILE")' in scan
assert "--skip-files /usr/local/bin/gosu" in scan assert "--skip-files /usr/local/bin/gosu" in scan
assert "final filesystem replaces it with the audited setpriv shell wrapper" in scan assert "final filesystem replaces it with the audited setpriv shell wrapper" in scan
assert "geointel-container-vulnerabilities.json" in scan assert "geointel-container-vulnerabilities.json" in scan
@@ -86,11 +128,12 @@ def test_readiness_guards_lock_and_supply_chain_entrypoints() -> None:
assert f"bash -n {path}" in readiness assert f"bash -n {path}" in readiness
def test_python_audit_exceptions_are_timeboxed_and_full_evidence_is_kept() -> None: def test_python_audit_policy_has_no_active_exceptions_and_keeps_full_evidence() -> None:
policy = read("security/pip-audit-exceptions.json") policy = read("security/pip-audit-exceptions.json")
audit_script = read("scripts/audit_python_dependencies.sh") audit_script = read("scripts/audit_python_dependencies.sh")
assert '"review_by": "2026-08-31"' in policy assert '"schema_version": 1' in policy
assert '"advisories": []' in policy
assert "pip-audit-full.json" in audit_script assert "pip-audit-full.json" in audit_script
assert "pip-audit-policy.json" in audit_script assert "pip-audit-policy.json" in audit_script
assert "--ignore-vuln" in audit_script assert "--ignore-vuln" in audit_script
@@ -20,12 +20,35 @@ def test_backup_is_atomic_read_only_and_checksum_bound() -> None:
assert "--no-owner" in script assert "--no-owner" in script
assert "CHECKSUMS.sha256" in script assert "CHECKSUMS.sha256" in script
assert "database-password" not in script.lower() assert "database-password" not in script.lower()
assert 'git -C "$ROOT" rev-parse HEAD' in script assert 'for required in docker python3 sha256sum; do' in script
assert 'git -C "$ROOT" status --porcelain=v1' in script assert 'for required in docker python3 sha256sum git; do' not in script
assert "GITEA_COMMIT_SHA" in script
assert "GITHUB_SHA" in script
assert "GEOINTEL_BUILD_SHA" in script
assert 'if command -v git >/dev/null 2>&1' in script
assert "mv \"$PARTIAL\" \"$FINAL\"" in script assert "mv \"$PARTIAL\" \"$FINAL\"" in script
assert "rm -rf -- \"$PARTIAL\"" in script assert "rm -rf -- \"$PARTIAL\"" in script
assert "DROP DATABASE" not in script assert "DROP DATABASE" not in script
assert "pg_restore --clean" not in script assert "pg_restore --clean" not in script
assert "/mnt/user/appdata/geointel/backups" in script
assert "release_backup_snapshot.py" in script
assert "storage-snapshot" not in script # labels are composed without unsafe path interpolation
assert "--link-dest-backup" in script
assert "--rollback-image-tag" in script
assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in script
def test_backup_binds_prepared_source_without_requiring_dot_git() -> None:
script = read("backup_release_state.sh")
controller_resolution = script.index('local gitea_sha="${GITEA_COMMIT_SHA:-}"')
optional_git_fallback = script.index('if command -v git >/dev/null 2>&1')
docker_access = script.index("docker inspect -f '{{.State.Running}}'")
assert controller_resolution < optional_git_fallback < docker_access
assert 'SOURCE_REVISION="$explicit_sha"' in script
assert '"backup_tool_revision": ${SOURCE_REVISION@Q}' in script
assert '"running_image_revision": ${RUNNING_IMAGE_REVISION@Q}' in script
assert "Cannot bind backup to a source revision" in script
def test_backup_verification_is_read_only() -> None: def test_backup_verification_is_read_only() -> None:
@@ -55,9 +78,11 @@ def test_release_safety_scripts_have_valid_bash_syntax() -> None:
"backup_release_state.sh", "backup_release_state.sh",
"verify_release_backup.sh", "verify_release_backup.sh",
"restore_release_backup_smoke.sh", "restore_release_backup_smoke.sh",
"../deploy/unraid/restore-predeploy-database.sh",
): ):
script_path = f"scripts/{name}" if not name.startswith("../") else name.removeprefix("../")
result = subprocess.run( result = subprocess.run(
["bash", "-n", f"scripts/{name}"], ["bash", "-n", script_path],
cwd=ROOT, cwd=ROOT,
capture_output=True, capture_output=True,
text=True, text=True,
@@ -66,6 +91,30 @@ def test_release_safety_scripts_have_valid_bash_syntax() -> None:
assert result.returncode == 0, f"{name}: {result.stderr}" assert result.returncode == 0, f"{name}: {result.stderr}"
def test_production_restore_is_explicit_bounded_and_verified() -> None:
script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
encoding="utf-8"
)
assert "--confirm-production-database-restore" in script
assert "/mnt/user/appdata/geointel/backups" in script
assert "backup.relative_to(root)" in script
assert "sha256sum -c CHECKSUMS.sha256" in script
assert '"$RESTORE_PROOF_DB"' in script
assert "pg_restore" in script
assert "Restored Alembic head" in script
assert "Restored count mismatch" in script
assert "pg_restore --clean" not in script
assert "geointel_restore_proof_" in script
assert "Isolated predeploy restore proof passed" in script
assert "ALTER DATABASE" in script
assert "Pre-restore production database retained" in script
drop_start = script.index("dropdb --if-exists --force")
drop_command = script[drop_start : script.index("\n fi", drop_start)]
assert '"$RESTORE_PROOF_DB"' in drop_command
assert '"$GEOINTEL_POSTGRES_DB"' not in drop_command
def test_readiness_gate_checks_release_safety_scripts() -> None: def test_readiness_gate_checks_release_safety_scripts() -> None:
readiness = read("run_readiness_check.sh") readiness = read("run_readiness_check.sh")
+3 -2
View File
@@ -1,11 +1,12 @@
from pathlib import Path from pathlib import Path
def test_backend_keeps_starlette_on_the_supported_pre_httpx2_line() -> None: def test_backend_uses_patched_starlette_and_explicit_httpx2_test_client() -> None:
pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml" pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml"
content = pyproject.read_text(encoding="utf-8") content = pyproject.read_text(encoding="utf-8")
assert '"starlette>=0.46.0,<1.0.0"' in content assert '"starlette>=1.3.1,<2.0.0"' in content
assert '"httpx2>=2.0.0"' in content
def test_readiness_gate_treats_deprecation_warnings_as_errors() -> None: def test_readiness_gate_treats_deprecation_warnings_as_errors() -> None:
@@ -0,0 +1,83 @@
from __future__ import annotations
import importlib.util
import os
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "release_backup_snapshot.py"
def load_snapshot_module():
spec = importlib.util.spec_from_file_location("release_backup_snapshot_test", SCRIPT)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_snapshot_is_byte_complete_and_reuses_only_verified_backup_bytes(tmp_path: Path) -> None:
snapshot = load_snapshot_module()
source = tmp_path / "source"
source.mkdir()
(source / "same.bin").write_bytes(b"unchanged")
(source / "changed.bin").write_bytes(b"before")
(source / "empty").mkdir()
prior = tmp_path / "prior"
prior_manifest = tmp_path / "prior.tsv"
snapshot.create_snapshot(source, prior, prior_manifest, label="storage")
snapshot.verify_snapshot(prior, prior_manifest)
(source / "changed.bin").write_bytes(b"after")
current = tmp_path / "current"
current_manifest = tmp_path / "current.tsv"
snapshot.create_snapshot(
source,
current,
current_manifest,
label="storage",
link_dest_snapshot=prior,
link_dest_manifest=prior_manifest,
)
snapshot.verify_snapshot(current, current_manifest)
assert os.path.samefile(prior / "same.bin", current / "same.bin")
assert not os.path.samefile(prior / "changed.bin", current / "changed.bin")
assert (current / "changed.bin").read_bytes() == b"after"
assert (current / "empty").is_dir()
def test_snapshot_rejects_symlinked_content(tmp_path: Path) -> None:
snapshot = load_snapshot_module()
source = tmp_path / "source"
source.mkdir()
target = source / "target.bin"
target.write_bytes(b"target")
try:
(source / "link.bin").symlink_to(target)
except OSError:
pytest.skip("Symlink creation is unavailable on this host")
with pytest.raises(RuntimeError, match="refuses symlinked content"):
snapshot.create_snapshot(source, tmp_path / "snapshot", tmp_path / "manifest.tsv", label="storage")
def test_snapshot_verification_rejects_changed_retained_bytes(tmp_path: Path) -> None:
snapshot = load_snapshot_module()
source = tmp_path / "source"
source.mkdir()
(source / "artifact.bin").write_bytes(b"retained")
retained = tmp_path / "snapshot"
manifest = tmp_path / "manifest.tsv"
snapshot.create_snapshot(source, retained, manifest, label="storage")
(retained / "artifact.bin").chmod(0o644)
(retained / "artifact.bin").write_bytes(b"tampered")
with pytest.raises(RuntimeError, match="checksum differs"):
snapshot.verify_snapshot(retained, manifest)
@@ -150,7 +150,7 @@ def test_tower_deploy_uses_single_container_unraid_compose() -> None:
assert "LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh" in release_script assert "LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh" in release_script
def test_tower_deploy_build_uses_remote_env_ai_setting_by_default() -> None: def test_tower_deploy_build_requires_the_production_ai_variant_by_default() -> None:
powershell = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8") powershell = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8")
bash = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8") bash = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8")
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8") release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
@@ -161,7 +161,8 @@ def test_tower_deploy_build_uses_remote_env_ai_setting_by_default() -> None:
assert "if [ -f .env ]; then" in release_script assert "if [ -f .env ]; then" in release_script
assert ". ./.env" in release_script assert ". ./.env" in release_script
assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-false}"' in release_script assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"' in release_script
assert "Production release deployment requires the gated AI image" in release_script
assert "--build-arg GEOINTEL_INSTALL_AI=" in release_script assert "--build-arg GEOINTEL_INSTALL_AI=" in release_script
+2
View File
@@ -160,8 +160,10 @@ COPY scripts/migrate_runtime_model_provenance.py /app/scripts/migrate_runtime_mo
COPY scripts/archive_technical_projects.py /app/scripts/archive_technical_projects.py COPY scripts/archive_technical_projects.py /app/scripts/archive_technical_projects.py
COPY scripts/runtime_state_report.py /app/scripts/runtime_state_report.py COPY scripts/runtime_state_report.py /app/scripts/runtime_state_report.py
COPY scripts/release_backup_guard.py /app/scripts/release_backup_guard.py COPY scripts/release_backup_guard.py /app/scripts/release_backup_guard.py
COPY scripts/release_backup_snapshot.py /app/scripts/release_backup_snapshot.py
COPY scripts/audit_data_operations.py /app/scripts/audit_data_operations.py COPY scripts/audit_data_operations.py /app/scripts/audit_data_operations.py
COPY scripts/cleanup_storage_artifacts.py /app/scripts/cleanup_storage_artifacts.py COPY scripts/cleanup_storage_artifacts.py /app/scripts/cleanup_storage_artifacts.py
COPY scripts/restore_storage_quarantine.py /app/scripts/restore_storage_quarantine.py
COPY deploy/unraid/nginx-all-in-one.conf /etc/nginx/conf.d/default.conf COPY deploy/unraid/nginx-all-in-one.conf /etc/nginx/conf.d/default.conf
COPY deploy/unraid/all-in-one-start.sh /usr/local/bin/geointel-all-in-one-start COPY deploy/unraid/all-in-one-start.sh /usr/local/bin/geointel-all-in-one-start
COPY --from=frontend-build /frontend/dist/ /usr/share/nginx/html/ COPY --from=frontend-build /frontend/dist/ /usr/share/nginx/html/
+34 -32
View File
@@ -11,9 +11,8 @@ met de naam `geointel`, bereikbaar op `http://192.168.10.150:1202`.
Kopieer de **volledige** map `C:\Projects\geointel` naar Kopieer de **volledige** map `C:\Projects\geointel` naar
`/mnt/user/appdata/geointel` op de server. Verder niets uitzoeken. `/mnt/user/appdata/geointel` op de server. Verder niets uitzoeken.
`.dockerignore` regelt de rest: `node_modules/`, `.git/`, de dubbele `.dockerignore` regelt de rest: `node_modules/`, `.git/`, `docs/`, `artifacts/`
`geointel/`-map, `docs/`, `artifacts/` en testoutput gaan niet mee de en testoutput gaan niet mee de build-context in, ook al staan ze in de map.
build-context in, ook al staan ze in de map.
Eén waarschuwing bij het overschrijven: laat `storage/`, `postgres-data/`, Eén waarschuwing bij het overschrijven: laat `storage/`, `postgres-data/`,
`backups/` en `models/` op de server **staan**. Dat is je bestaande data, en `backups/` en `models/` op de server **staan**. Dat is je bestaande data, en
@@ -71,7 +70,8 @@ die hostname in `GEOINTEL_CORS_ORIGINS` staan — anders blokkeert de browser de
API-calls vanaf het publieke adres. De backend doet geen host-validatie, dus API-calls vanaf het publieke adres. De backend doet geen host-validatie, dus
verder is er niets nodig aan applicatiekant. verder is er niets nodig aan applicatiekant.
Voor GPU-inferentie (optioneel, kan ook later): De productie-image bevat altijd de gepinde AI-runtime. Inferentie zelf kan
uitblijven totdat een lokaal, gevalideerd model beschikbaar is:
```env ```env
GEOINTEL_INSTALL_AI=true GEOINTEL_INSTALL_AI=true
@@ -81,8 +81,8 @@ YOLO_REQUIRE_CUDA=true
YOLO_MODEL_PATH=/app/models/<jouw-model>.pt YOLO_MODEL_PATH=/app/models/<jouw-model>.pt
``` ```
Laat `GEOINTEL_INSTALL_AI=false` staan als je eerst gewoon wilt dat de app Laat `YOLO_ENABLED=false` zolang er geen geschikt modelbestand is. Het
draait — dat scheelt een paar GB aan PyTorch-lagen in de build. release-deployscript weigert bewust een GIS-only productie-image.
--- ---
@@ -134,12 +134,21 @@ bash deploy/unraid/deploy-release.sh
Het script: Het script:
1. ruimt een eventueel achtergebleven Compose-stack op (ook de oude 1. bindt het exacte huidige image-ID aan een unieke backup-specifieke
`rollback-predeploy-*`-tag;
2. bouwt of hergebruikt de AI-candidate terwijl de huidige release beschikbaar
blijft, legt het exacte lokale image-ID vast en maakt daarop SBOM- en
Trivy-evidence;
3. controleert vóór het pauzeren de vrije ruimte en maakt vervolgens een
byte-complete, SHA-256-geverifieerde database-, storage- en modelsnapshot in
`/mnt/user/appdata/geointel/backups`; ongewijzigde bestanden mogen alleen
vanuit een oudere geverifieerde backup worden gehardlinkt;
4. ruimt een eventueel achtergebleven Compose-stack op (ook de oude
3-container dev-stack die óók poort 1202 pakt); 3-container dev-stack die óók poort 1202 pakt);
2. bewaart de huidige image als `geointel-all-in-one:previous`; 5. start één container `geointel` met `-p 1202:80` en `--gpus all`;
3. bouwt `deploy/unraid/Dockerfile.all-in-one`; 6. bewijst een rollbackdump eerst in een geïsoleerde tijdelijke database,
4. start één container `geointel` met `-p 1202:80` en `--gpus all`; bewaart de oude productiedatabase als herstelpad en start pas daarna
5. rolt automatisch terug naar `:previous` als de healthcheck of smoke faalt. automatisch het image-ID dat cryptografisch in die pre-deploybackup staat.
De eerste build duurt lang (PostGIS + GDAL + npm build). Volgende deploys De eerste build duurt lang (PostGIS + GDAL + npm build). Volgende deploys
hergebruiken de Docker-layercache. hergebruiken de Docker-layercache.
@@ -165,24 +174,13 @@ docker logs --tail 200 geointel
--- ---
## 7. Belangrijk: dubbele projectmap lokaal ## 7. Geretireerde dubbele projectmap
In `C:\Projects\geointel` staat een tweede, volledige kopie van het project De vroegere geneste mirror `C:\Projects\geointel\geointel` is geretireerd. De
onder `C:\Projects\geointel\geointel\`. Die bevat dezelfde bestanden en immutable Git-herkomst, niet-getrackte recoverybestanden en verificatiegrens
dezelfde datum, maar staat buiten git. Zolang die er staat: staan in `docs/accuracy-program/13-nested-mirror-retirement.md`. Behandel die
evidence als herstelreferentie; deze handleiding vraagt geen extra kopieer- of
- wordt de Docker build-context onnodig verdubbeld; opruimactie.
- weet je bij het bewerken van bijvoorbeeld `docker-compose.unraid.yml` niet
welke versie je te pakken hebt.
`.dockerignore` sluit hem nu uit, maar ruim hem op zodra je zeker weet dat er
niets unieks in staat. Vergelijk eerst:
```powershell
robocopy C:\Projects\geointel\geointel C:\Projects\geointel /L /E /NJH /NJS /NDL /XF *.pyc
```
Regels die als `New File` verschijnen bestaan alleen in de kopie.
--- ---
@@ -190,11 +188,15 @@ Regels die als `New File` verschijnen bestaan alleen in de kopie.
```bash ```bash
cd /mnt/user/appdata/geointel cd /mnt/user/appdata/geointel
bash deploy/unraid/rollback-dockerman-container.sh bash deploy/unraid/rollback-dockerman-container.sh \
--backup-dir /mnt/user/appdata/geointel/backups/<predeploy-release-id> \
--confirm-production-database-restore
``` ```
Rollback hergebruikt dezelfde PostGIS- en storage-paden en draait nooit een Rollback hergebruikt dezelfde storage-paden, bewijst de geverifieerde dump
Alembic-downgrade. eerst geïsoleerd, wisselt daarna databases via no-clobber namen en draait nooit
een Alembic-downgrade. De oude productiedatabase blijft staan totdat een
operator haar na controle expliciet opruimt.
--- ---
@@ -202,7 +204,7 @@ Alembic-downgrade.
| Bestand | Aanpassing | | Bestand | Aanpassing |
|---|---| |---|---|
| `.dockerignore` | Sluit root-`node_modules`, de dubbele `geointel/`-map, `.git`, `docs/`, `artifacts/` en testoutput uit de build-context | | `.dockerignore` | Sluit root-`node_modules`, `.git`, `docs/`, `artifacts/` en testoutput uit de build-context |
| `deploy/unraid/deploy-release.sh` | `git rev-parse HEAD` crashte op een kopie zonder `.git`. Valt nu terug op `GEOINTEL_BUILD_SHA`, een `RELEASE_SHA`-bestand of een content-hash van de broncode | | `deploy/unraid/deploy-release.sh` | `git rev-parse HEAD` crashte op een kopie zonder `.git`. Valt nu terug op `GEOINTEL_BUILD_SHA`, een `RELEASE_SHA`-bestand of een content-hash van de broncode |
| `deploy/unraid/deploy-release.sh` | Smoke-scripts worden op bestaan getest in plaats van op de execute-bit, die bij een Windows-kopie verloren gaat | | `deploy/unraid/deploy-release.sh` | Smoke-scripts worden op bestaan getest in plaats van op de execute-bit, die bij een Windows-kopie verloren gaat |
| `deploy/unraid/run-dockerman-container.sh` | Ruimt expliciet zowel `docker-compose.yml` (3 containers) als `docker-compose.unraid.yml` op, zodat poort 1202 gegarandeerd vrij is | | `deploy/unraid/run-dockerman-container.sh` | Ruimt expliciet zowel `docker-compose.yml` (3 containers) als `docker-compose.unraid.yml` op, zodat poort 1202 gegarandeerd vrij is |
+30 -22
View File
@@ -106,27 +106,25 @@ customer or operational data. Deploy a separate demo container and storage
root for public or recruiter-facing access. root for public or recruiter-facing access.
The repository deploy scripts run the same flow automatically. They validate The repository deploy scripts run the same flow automatically. They validate
the Compose reference, preserve the current image as the Compose reference, preserve the current image under a unique
`geointel-all-in-one:previous`, build an immutable `<commit-sha>-ai` or backup-specific `rollback-predeploy-*` tag, build the immutable production
`<commit-sha>-gis` tag plus `latest`, install the DockerMan metadata and start `<commit-sha>-ai` tag plus `latest`, attest its exact local image ID, generate
the immutable image. An existing matching tag is reused, never rebuilt. A an SBOM and enforce the Trivy policy before starting that same ID. An existing
failed start, live migration smoke or browser/API smoke automatically attempts matching tag is reused, never rebuilt. A failed start, live migration smoke or
the previous image without changing the configured PostGIS or storage paths. browser/API smoke automatically attempts the previous image without changing
the configured PostGIS or storage paths.
`scripts/deploy_tower.sh` and `scripts/deploy_tower.ps1` source the remote `scripts/deploy_tower.sh` and `scripts/deploy_tower.ps1` source the remote
`.env` before building the image. That means `GEOINTEL_INSTALL_AI=true` in `.env` before building the image. Production deployment requires
`/mnt/user/appdata/geointel/.env` is enough for the automatic deploy to build `GEOINTEL_INSTALL_AI=true`; an explicit false value fails closed before the
the AI-enabled image. Set `GEOINTEL_INSTALL_AI` in the local shell or pass image or running container is replaced.
`-InstallAi true/false` to the PowerShell wrapper only when you intentionally
want to override the remote `.env` for that deploy.
Database credentials are runtime configuration, not image metadata. The Database credentials are runtime configuration, not image metadata. The
all-in-one image does not bake `GEOINTEL_POSTGRES_PASSWORD` into the Dockerfile; all-in-one image does not bake `GEOINTEL_POSTGRES_PASSWORD` into the Dockerfile;
set it through `.env`, the Unraid template or `docker run -e`. set it through `.env`, the Unraid template or `docker run -e`.
AI dependencies are opt-in. Leave `GEOINTEL_INSTALL_AI=false` for the default Production images always include the pinned AI dependencies. Mount models
GIS-only image. Set `GEOINTEL_INSTALL_AI=true`, mount models through through `GEOINTEL_MODELS_PATH` and configure `YOLO_ENABLED=true` plus
`GEOINTEL_MODELS_PATH` and configure `YOLO_ENABLED=true` plus
`YOLO_MODELS_DIR=/app/models` and `YOLO_MODEL_PATH=/app/models/<model>.pt` only `YOLO_MODELS_DIR=/app/models` and `YOLO_MODEL_PATH=/app/models/<model>.pt` only
when you have a local model file. when you have a local model file.
The AI-enabled image installs PyTorch/Ultralytics plus the native OpenCV runtime The AI-enabled image installs PyTorch/Ultralytics plus the native OpenCV runtime
@@ -264,7 +262,8 @@ git reset --hard origin/main
bash deploy/unraid/deploy-release.sh bash deploy/unraid/deploy-release.sh
``` ```
The equivalent low-level build remains available for debugging: The equivalent low-level GIS-only build remains available only for local
debugging; it is not a production deployment path:
```bash ```bash
docker build --build-arg GEOINTEL_INSTALL_AI=${GEOINTEL_INSTALL_AI:-false} -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest . docker build --build-arg GEOINTEL_INSTALL_AI=${GEOINTEL_INSTALL_AI:-false} -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest .
@@ -305,20 +304,27 @@ bash scripts/verify_release_upgrade_smoke.sh \
Return to the image that was active immediately before the latest deployment: Return to the image that was active immediately before the latest deployment:
```bash ```bash
bash deploy/unraid/rollback-dockerman-container.sh bash deploy/unraid/rollback-dockerman-container.sh \
--backup-dir /mnt/user/appdata/geointel/backups/<predeploy-release-id> \
--confirm-production-database-restore
``` ```
For an older retained commit, select its immutable tag explicitly: For an older retained commit, select its immutable tag explicitly:
```bash ```bash
GEOINTEL_ROLLBACK_IMAGE=geointel-all-in-one:<commit-sha>-ai \ GEOINTEL_ROLLBACK_IMAGE=geointel-all-in-one:<commit-sha>-ai \
bash deploy/unraid/rollback-dockerman-container.sh bash deploy/unraid/rollback-dockerman-container.sh \
--backup-dir /mnt/user/appdata/geointel/backups/<matching-predeploy-release-id> \
--confirm-production-database-restore
``` ```
Rollback reuses the configured PostGIS and storage mounts and never runs an Rollback restores and verifies the selected dump in an isolated proof database
Alembic downgrade. If a future release has a backward-incompatible migration, before any production replacement. It then swaps database names, retains the
restore its verified pre-release backup instead of forcing an older app pre-restore production database for operator recovery, reuses the configured
against a newer schema. storage mount and never runs an Alembic downgrade or an older app against an
unknown newer schema. Remove the retained recovery database and old backup
directories only in a separately reviewed operator retention step; deployment
never deletes them automatically.
The configured upload limit is shared by FastAPI and the generated nginx The configured upload limit is shared by FastAPI and the generated nginx
runtime configuration. Values outside `1..2048` MiB are rejected before the runtime configuration. Values outside `1..2048` MiB are rejected before the
@@ -339,7 +345,9 @@ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py
``` ```
The full backup, confirmation, candidate-limit and apply sequence is in The full backup, confirmation, candidate-limit and apply sequence is in
`docs/DATA_OPERATIONS_RUNBOOK.md`. GeoIntel installs no automatic cleanup `docs/DATA_OPERATIONS_RUNBOOK.md`. Apply moves bytes to protected,
checksum-bound quarantine rather than deleting them; a separate confirmed
restore command reverses the move. GeoIntel installs no automatic cleanup
schedule. schedule.
## Safe cleanup ## Safe cleanup
+372 -22
View File
@@ -26,7 +26,11 @@ if [ -n "${DEPLOY_GEOINTEL_INSTALL_AI:-}" ]; then
GEOINTEL_INSTALL_AI="$DEPLOY_GEOINTEL_INSTALL_AI" GEOINTEL_INSTALL_AI="$DEPLOY_GEOINTEL_INSTALL_AI"
fi fi
GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-false}" GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"
if [ "$GEOINTEL_INSTALL_AI" != "true" ]; then
echo "Production release deployment requires the gated AI image (GEOINTEL_INSTALL_AI=true)." >&2
exit 2
fi
GEOINTEL_APP_VERSION="$(tr -d '[:space:]' < VERSION)" GEOINTEL_APP_VERSION="$(tr -d '[:space:]' < VERSION)"
if ! [[ "$GEOINTEL_APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then if ! [[ "$GEOINTEL_APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then
echo "Invalid semantic version in VERSION: ${GEOINTEL_APP_VERSION}" >&2 echo "Invalid semantic version in VERSION: ${GEOINTEL_APP_VERSION}" >&2
@@ -57,7 +61,52 @@ source_tree_hash() {
} }
resolve_build_sha() { resolve_build_sha() {
local head="" content="" local head="" content="" controller_sha="" controller_source=""
if [ -n "${GITEA_COMMIT_SHA:-}" ]; then
controller_sha="$GITEA_COMMIT_SHA"
controller_source="GITEA_COMMIT_SHA"
fi
if [ -n "${GITHUB_SHA:-}" ]; then
if ! [[ "$GITHUB_SHA" =~ ^[0-9A-Fa-f]{40}$ ]]; then
echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2
return 2
fi
if [ -n "$controller_sha" ] && [ "${controller_sha,,}" != "${GITHUB_SHA,,}" ]; then
echo "Controller commit variables disagree." >&2
return 2
fi
controller_sha="$GITHUB_SHA"
controller_source="${controller_source:-GITHUB_SHA}"
fi
if [ -n "$controller_sha" ]; then
if ! [[ "$controller_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
echo "${controller_source} must contain one full 40-character Git commit SHA." >&2
return 2
fi
controller_sha="${controller_sha,,}"
if [ -n "${GEOINTEL_BUILD_SHA:-}" ] && [ "${GEOINTEL_BUILD_SHA,,}" != "$controller_sha" ]; then
echo "Explicit build revision differs from the controller revision." >&2
return 2
fi
if command -v git >/dev/null 2>&1 && git rev-parse --git-dir >/dev/null 2>&1; then
head="$(git rev-parse HEAD 2>/dev/null || true)"
if [ "${head,,}" != "$controller_sha" ]; then
echo "Prepared Git checkout does not match the controller revision." >&2
return 2
fi
if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
echo "Prepared Git checkout contains changes outside the controller revision." >&2
return 2
fi
fi
printf '%s' "$controller_sha"
return 0
fi
if [ -n "${GITEA_REPOSITORY:-}" ] || [ -n "${GITHUB_REPOSITORY:-}" ]; then
echo "Automated deployment context is missing GITEA_COMMIT_SHA/GITHUB_SHA." >&2
return 2
fi
# 1. Explicit override wins. # 1. Explicit override wins.
if [ -n "${GEOINTEL_BUILD_SHA:-}" ]; then if [ -n "${GEOINTEL_BUILD_SHA:-}" ]; then
@@ -107,7 +156,13 @@ if [ -z "$GEOINTEL_BUILD_SHA" ]; then
echo "Could not determine a build revision for this deployment." >&2 echo "Could not determine a build revision for this deployment." >&2
exit 2 exit 2
fi fi
export GEOINTEL_BUILD_SHA
echo "Build revision: ${GEOINTEL_BUILD_SHA}" echo "Build revision: ${GEOINTEL_BUILD_SHA}"
GEOINTEL_RELEASE_TOKEN="$(printf '%s' "$GEOINTEL_BUILD_SHA" | tr -c 'A-Za-z0-9._-' '_' | cut -c1-48)"
if [ -z "$GEOINTEL_RELEASE_TOKEN" ]; then
echo "Could not derive a safe release evidence identifier." >&2
exit 2
fi
GEOINTEL_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" GEOINTEL_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
GEOINTEL_IMAGE_REPOSITORY="${GEOINTEL_IMAGE_REPOSITORY:-geointel-all-in-one}" GEOINTEL_IMAGE_REPOSITORY="${GEOINTEL_IMAGE_REPOSITORY:-geointel-all-in-one}"
if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then
@@ -116,8 +171,23 @@ else
GEOINTEL_RELEASE_VARIANT="gis" GEOINTEL_RELEASE_VARIANT="gis"
fi fi
GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}" GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"
GEOINTEL_PREVIOUS_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:previous"
FRONTEND_URL="${FRONTEND_URL:-http://127.0.0.1:${GEOINTEL_FRONTEND_PORT:-1202}}" FRONTEND_URL="${FRONTEND_URL:-http://127.0.0.1:${GEOINTEL_FRONTEND_PORT:-1202}}"
GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
GEOINTEL_STORAGE_PATH="${GEOINTEL_STORAGE_PATH:-/mnt/user/appdata/geointel/storage}"
GEOINTEL_MODELS_PATH="${GEOINTEL_MODELS_PATH:-/mnt/user/appdata/geointel/models}"
GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}"
GEOINTEL_DEPLOY_EVIDENCE_DIR="${GEOINTEL_DEPLOY_EVIDENCE_DIR:-artifacts/release-evidence/deploy/${GEOINTEL_RELEASE_TOKEN}-ai}"
GEOINTEL_PREDEPLOY_BACKUP_DIR=""
GEOINTEL_RELEASE_IMAGE_ID=""
GEOINTEL_BACKUP_LINK_DEST=""
GEOINTEL_PREDEPLOY_ROLLBACK_TAG=""
case "$GEOINTEL_DEPLOY_EVIDENCE_DIR" in
/*|*..*)
echo "Deployment evidence directory must be repository-relative and must not contain '..'." >&2
exit 2
;;
esac
wait_for_geointel_health() { wait_for_geointel_health() {
local status="" local status=""
@@ -141,35 +211,312 @@ wait_for_geointel_health() {
start_image() { start_image() {
local image="$1" local image="$1"
local running_image_id=""
local running_revision=""
local running_ai=""
GEOINTEL_IMAGE="$image" bash deploy/unraid/run-dockerman-container.sh GEOINTEL_IMAGE="$image" bash deploy/unraid/run-dockerman-container.sh
wait_for_geointel_health wait_for_geointel_health
running_image_id="$(docker inspect --format '{{.Image}}' geointel)"
if [ "$running_image_id" != "$image" ]; then
echo "Running container image ${running_image_id} differs from attested image ${image}." >&2
return 1
fi
running_revision="$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' geointel)"
running_ai="$(docker inspect --format '{{index .Config.Labels "io.geointel.ai.enabled"}}' geointel)"
if [ "$running_revision" != "$GEOINTEL_BUILD_SHA" ] || [ "$running_ai" != "true" ]; then
echo "Running container labels do not match the attested AI revision." >&2
return 1
fi
echo "Running container matches attested image: ${running_image_id}"
}
scan_release_image() {
local scanned_image_id=""
local current_image_id=""
local inspect_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/image-inspect.json"
local sbom_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-sbom.spdx.json"
local vulnerability_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-container-vulnerabilities.json"
local attestation_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/deployment-attestation.json"
scanned_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
test -n "$scanned_image_id"
mkdir -p "$ROOT/$GEOINTEL_DEPLOY_EVIDENCE_DIR"
docker image inspect "$GEOINTEL_RELEASE_IMAGE" > "$ROOT/$inspect_output"
bash scripts/generate_container_sbom.sh "$GEOINTEL_RELEASE_IMAGE" "$sbom_output"
bash scripts/scan_container_image.sh "$GEOINTEL_RELEASE_IMAGE" "$vulnerability_output"
current_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
if [ "$current_image_id" != "$scanned_image_id" ]; then
echo "Release image tag changed while SBOM/scan evidence was being generated." >&2
return 1
fi
test -s "$ROOT/$inspect_output"
test -s "$ROOT/$sbom_output"
test -s "$ROOT/$vulnerability_output"
GEOINTEL_RELEASE_IMAGE_ID="$scanned_image_id"
python3 - \
"$ROOT/$attestation_output" \
"$GEOINTEL_RELEASE_IMAGE" \
"$GEOINTEL_RELEASE_IMAGE_ID" \
"$GEOINTEL_BUILD_SHA" \
"$inspect_output" \
"$sbom_output" \
"$vulnerability_output" <<'PY'
import datetime
import json
import pathlib
import sys
output, image_tag, image_id, revision, inspect_path, sbom_path, vulnerability_path = sys.argv[1:]
payload = {
"schema_version": 1,
"attested_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"image_tag": image_tag,
"image_id": image_id,
"image_config_digest": image_id,
"revision": revision,
"variant": "ai",
"evidence": {
"image_inspect": inspect_path,
"sbom": sbom_path,
"vulnerabilities": vulnerability_path,
},
}
path = pathlib.Path(output)
temporary = path.with_suffix(".json.partial")
temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
temporary.replace(path)
PY
test -s "$ROOT/$attestation_output"
echo "Exact deployment image scanned: ${GEOINTEL_RELEASE_IMAGE_ID}"
}
preflight_backup_capacity() {
local database_name=""
local database_user=""
local database_size_bytes=""
database_name="$(docker exec geointel sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"')"
database_user="$(docker exec geointel sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"')"
database_size_bytes="$(docker exec geointel psql -X -v ON_ERROR_STOP=1 \
-U "$database_user" -d "$database_name" -Atqc \
'SELECT pg_database_size(current_database());')"
mkdir -p "$GEOINTEL_BACKUPS_PATH"
python3 - \
"$GEOINTEL_BACKUPS_PATH" \
"$GEOINTEL_STORAGE_PATH" \
"$GEOINTEL_MODELS_PATH" \
"$database_size_bytes" <<'PY'
import os
import pathlib
import shutil
import stat
import sys
backup_root = pathlib.Path(sys.argv[1]).expanduser().resolve()
sources = [pathlib.Path(value).expanduser().resolve() for value in sys.argv[2:4]]
database_bytes = int(sys.argv[4])
def retained_bytes(root: pathlib.Path) -> int:
if not root.is_dir():
raise SystemExit(f"Mandatory snapshot source is not a directory: {root}")
total = 0
for current, directories, files in os.walk(root, topdown=True, followlinks=False):
current_path = pathlib.Path(current)
for name in [*directories, *files]:
path = current_path / name
details = path.lstat()
if stat.S_ISLNK(details.st_mode):
raise SystemExit(f"Mandatory snapshot refuses symlinked content: {path}")
if name in directories and not stat.S_ISDIR(details.st_mode):
raise SystemExit(f"Snapshot directory changed during capacity preflight: {path}")
if name in files:
if not stat.S_ISREG(details.st_mode):
raise SystemExit(f"Mandatory snapshot refuses non-regular content: {path}")
total += details.st_size
return total
source_bytes = sum(retained_bytes(source) for source in sources)
# Reflink clones are used when the backing filesystem supports them. Budget for
# a complete copy plus two uncompressed database sizes (dump and isolated
# restore/cutover recovery) so fallback still fails before the live backend is
# quiesced rather than midway through the snapshot.
required = source_bytes + (2 * database_bytes)
headroom = max(5 * 1024**3, required // 10)
free = shutil.disk_usage(backup_root).free
if free < required + headroom:
raise SystemExit(
"Insufficient free space for a fail-safe predeploy snapshot: "
f"required={required + headroom} free={free} source={source_bytes} database={database_bytes}"
)
print(
"Predeploy snapshot capacity: "
f"source_bytes={source_bytes} database_bytes={database_bytes} free_bytes={free}"
)
PY
}
select_verified_link_dest() {
local candidate=""
GEOINTEL_BACKUP_LINK_DEST=""
while IFS= read -r candidate; do
if (
cd "$candidate" \
&& sha256sum -c CHECKSUMS.sha256 >/dev/null \
&& python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate"
); then
GEOINTEL_BACKUP_LINK_DEST="$candidate"
echo "Using verified prior byte snapshot as link-dest: ${candidate}"
return 0
fi
echo "Skipping unusable prior backup link-dest: ${candidate}" >&2
done < <(
python3 - "$GEOINTEL_BACKUPS_PATH" <<'PY'
import pathlib
import sys
root = pathlib.Path(sys.argv[1]).expanduser().resolve()
candidates = sorted(
(
path
for path in root.iterdir()
if path.is_dir() and not path.name.startswith(".") and (path / "manifest.json").is_file()
),
key=lambda path: path.stat().st_mtime_ns,
reverse=True,
)
for candidate in candidates:
print(candidate)
PY
)
echo "No verified prior byte snapshot found; this deployment will create a full first snapshot."
}
create_predeploy_backup() {
local container_exists="false"
local container_running="false"
local release_id=""
local backup_link_args=()
local current_image_id=""
local existing_rollback_id=""
if docker ps -a --format '{{.Names}}' | grep -Fxq geointel; then
container_exists="true"
fi
if [ "$(docker inspect -f '{{.State.Running}}' geointel 2>/dev/null || true)" = "true" ]; then
container_running="true"
fi
if [ "$container_exists" = "false" ]; then
if [ -f "$GEOINTEL_POSTGIS_DATA_PATH/PG_VERSION" ]; then
echo "PostGIS data exists without a running GeoIntel container; refusing an unbacked migration." >&2
return 1
fi
echo "No existing GeoIntel state found; pre-deploy backup is not required for this fresh install."
return 0
fi
if [ "$container_running" != "true" ]; then
echo "Existing GeoIntel container is not running; refusing deployment because a consistent backup cannot be created." >&2
return 1
fi
current_image_id="$(docker inspect --format '{{.Image}}' geointel)"
if ! [[ "$current_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
echo "Running release does not expose one immutable Docker image ID." >&2
return 1
fi
release_id="predeploy-${GEOINTEL_RELEASE_TOKEN:0:24}-$(date -u +%Y%m%dT%H%M%SZ)-$$"
GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"
existing_rollback_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" 2>/dev/null || true)"
if [ -n "$existing_rollback_id" ] && [ "$existing_rollback_id" != "$current_image_id" ]; then
echo "Backup-specific rollback tag already identifies different image bytes." >&2
return 1
fi
docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"
# This conservative full-copy fallback estimate runs while the existing
# release is still healthy. Verified backup-to-backup hardlinks normally
# avoid recopying unchanged bytes, but are never assumed for this fail-closed
# capacity decision.
preflight_backup_capacity
select_verified_link_dest
if [ -n "$GEOINTEL_BACKUP_LINK_DEST" ]; then
backup_link_args=(--link-dest-backup "$GEOINTEL_BACKUP_LINK_DEST")
fi
echo "Quiescing the current backend so the rollback point cannot miss concurrent writes..."
if ! docker exec -i geointel python - <<'PY'
import os
import pathlib
import signal
import time
matches = []
for item in pathlib.Path("/proc").iterdir():
if not item.name.isdigit() or int(item.name) in {os.getpid(), os.getppid()}:
continue
try:
command = (item / "cmdline").read_bytes().replace(b"\0", b" ")
except (OSError, PermissionError):
continue
if b"uvicorn" in command and b"app.main:app" in command:
matches.append(int(item.name))
if not matches:
raise SystemExit("Could not identify the running GeoIntel backend")
for process_id in matches:
os.kill(process_id, signal.SIGTERM)
deadline = time.monotonic() + 60
remaining = matches
while remaining and time.monotonic() < deadline:
time.sleep(0.25)
remaining = [process_id for process_id in remaining if pathlib.Path(f"/proc/{process_id}").exists()]
if remaining:
raise SystemExit(f"Backend did not stop cleanly: {remaining}")
print(f"Stopped {len(matches)} backend process(es)")
PY
then
echo "Could not quiesce the current backend; refusing a potentially inconsistent backup." >&2
docker restart geointel >/dev/null || true
wait_for_geointel_health || true
return 1
fi
GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}"
echo "Creating mandatory pre-deploy backup ${release_id}..."
if ! bash scripts/backup_release_state.sh \
--container geointel \
--output-root "$GEOINTEL_BACKUPS_PATH" \
--release-id "$release_id" \
--storage-path "$GEOINTEL_STORAGE_PATH" \
--models-path "$GEOINTEL_MODELS_PATH" \
--inventory-mode sha256 \
--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \
"${backup_link_args[@]}" \
|| ! bash scripts/verify_release_backup.sh \
--container geointel \
--backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then
echo "Pre-deploy backup failed; restarting the unchanged current release." >&2
docker restart geointel >/dev/null || true
wait_for_geointel_health || true
GEOINTEL_PREDEPLOY_BACKUP_DIR=""
return 1
fi
echo "Pre-deploy backup verified: ${GEOINTEL_PREDEPLOY_BACKUP_DIR}"
} }
rollback_previous() { rollback_previous() {
if ! docker image inspect "$GEOINTEL_PREVIOUS_IMAGE" >/dev/null 2>&1; then if [ -z "$GEOINTEL_PREDEPLOY_BACKUP_DIR" ]; then
echo "Automatic rollback unavailable: ${GEOINTEL_PREVIOUS_IMAGE} does not exist." >&2 echo "Automatic rollback unavailable: no verified pre-deploy database backup was created." >&2
return 1 return 1
fi fi
echo "Rolling back to ${GEOINTEL_PREVIOUS_IMAGE}..." echo "Rolling back database and image to the verified pre-deploy state..."
start_image "$GEOINTEL_PREVIOUS_IMAGE" GEOINTEL_DEPLOY_LOCK_HELD=true \
bash deploy/unraid/rollback-dockerman-container.sh \
--backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR" \
--confirm-production-database-restore
} }
docker compose -f docker-compose.unraid.yml config >/dev/null docker compose -f docker-compose.unraid.yml config >/dev/null
current_image_id="$(docker inspect --format '{{.Image}}' geointel 2>/dev/null || true)"
release_image_id="$(
docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE" 2>/dev/null || true
)"
if (
[ -n "$current_image_id" ] &&
[ "$current_image_id" != "$release_image_id" ] &&
docker image inspect "$current_image_id" >/dev/null 2>&1
); then
docker tag "$current_image_id" "$GEOINTEL_PREVIOUS_IMAGE"
elif [ -n "$current_image_id" ] && [ "$current_image_id" = "$release_image_id" ]; then
echo "Current container already uses ${GEOINTEL_RELEASE_IMAGE}; preserving the existing previous image."
fi
if docker image inspect "$GEOINTEL_RELEASE_IMAGE" >/dev/null 2>&1; then if docker image inspect "$GEOINTEL_RELEASE_IMAGE" >/dev/null 2>&1; then
stored_revision="$( stored_revision="$(
docker image inspect \ docker image inspect \
@@ -208,7 +555,10 @@ else
. .
fi fi
if ! start_image "$GEOINTEL_RELEASE_IMAGE"; then scan_release_image
create_predeploy_backup
if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"; then
rollback_previous || true rollback_previous || true
exit 1 exit 1
fi fi
@@ -32,11 +32,17 @@
<Config Name="Postgres Password" Target="GEOINTEL_POSTGRES_PASSWORD" Default="change-me-before-shared-use" Mode="" Description="Embedded PostGIS database password. Change before shared use." Type="Variable" Display="advanced" Required="true" Mask="true">change-me-before-shared-use</Config> <Config Name="Postgres Password" Target="GEOINTEL_POSTGRES_PASSWORD" Default="change-me-before-shared-use" Mode="" Description="Embedded PostGIS database password. Change before shared use." Type="Variable" Display="advanced" Required="true" Mask="true">change-me-before-shared-use</Config>
<Config Name="CORS Origins" Target="GEOINTEL_CORS_ORIGINS" Default="http://localhost:1202,http://127.0.0.1:1202,http://192.168.10.150:1202" Mode="" Description="Comma-separated browser origins allowed to call the backend directly." Type="Variable" Display="advanced" Required="false" Mask="false">http://localhost:1202,http://127.0.0.1:1202,http://192.168.10.150:1202</Config> <Config Name="CORS Origins" Target="GEOINTEL_CORS_ORIGINS" Default="http://localhost:1202,http://127.0.0.1:1202,http://192.168.10.150:1202" Mode="" Description="Comma-separated browser origins allowed to call the backend directly." Type="Variable" Display="advanced" Required="false" Mask="false">http://localhost:1202,http://127.0.0.1:1202,http://192.168.10.150:1202</Config>
<Config Name="Max Upload MB" Target="GEOINTEL_MAX_UPLOAD_MB" Default="500" Mode="" Description="Maximum upload size in MiB enforced consistently by nginx and the backend (1-2048)." Type="Variable" Display="advanced" Required="true" Mask="false">500</Config> <Config Name="Max Upload MB" Target="GEOINTEL_MAX_UPLOAD_MB" Default="500" Mode="" Description="Maximum upload size in MiB enforced consistently by nginx and the backend (1-2048)." Type="Variable" Display="advanced" Required="true" Mask="false">500</Config>
<Config Name="Max In-memory Vector MB" Target="GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" Default="64" Mode="" Description="Maximum decompressed vector payload processed fully in memory by the backend (1-256 MiB)." Type="Variable" Display="advanced" Required="true" Mask="false">64</Config>
<Config Name="Operator Login Enabled" Target="GEOINTEL_AUTH_ENABLED" Default="false" Mode="" Description="Require the single configured operator login before the browser may access workbench APIs." Type="Variable" Display="advanced" Required="true" Mask="false">false</Config> <Config Name="Operator Login Enabled" Target="GEOINTEL_AUTH_ENABLED" Default="false" Mode="" Description="Require the single configured operator login before the browser may access workbench APIs." Type="Variable" Display="advanced" Required="true" Mask="false">false</Config>
<Config Name="Operator Username" Target="GEOINTEL_AUTH_USERNAME" Default="" Mode="" Description="Exact username for the single operator account." Type="Variable" Display="advanced" Required="false" Mask="false"></Config> <Config Name="Operator Username" Target="GEOINTEL_AUTH_USERNAME" Default="" Mode="" Description="Exact username for the single operator account." Type="Variable" Display="advanced" Required="false" Mask="false"></Config>
<Config Name="Operator Password Hash" Target="GEOINTEL_AUTH_PASSWORD_HASH" Default="" Mode="" Description="PBKDF2-SHA256 password hash. Never enter a plaintext password." Type="Variable" Display="advanced" Required="false" Mask="true"></Config> <Config Name="Operator Password Hash" Target="GEOINTEL_AUTH_PASSWORD_HASH" Default="" Mode="" Description="PBKDF2-SHA256 password hash. Never enter a plaintext password." Type="Variable" Display="advanced" Required="false" Mask="true"></Config>
<Config Name="Operator Session Secret" Target="GEOINTEL_AUTH_SESSION_SECRET" Default="" Mode="" Description="Random secret of at least 32 characters used only to sign browser sessions." Type="Variable" Display="advanced" Required="false" Mask="true"></Config> <Config Name="Operator Session Secret" Target="GEOINTEL_AUTH_SESSION_SECRET" Default="" Mode="" Description="Random secret of at least 32 characters used only to sign browser sessions." Type="Variable" Display="advanced" Required="false" Mask="true"></Config>
<Config Name="Operator Session TTL" Target="GEOINTEL_AUTH_SESSION_TTL_SECONDS" Default="43200" Mode="" Description="Session lifetime in seconds (900-604800)." Type="Variable" Display="advanced" Required="true" Mask="false">43200</Config> <Config Name="Operator Session TTL" Target="GEOINTEL_AUTH_SESSION_TTL_SECONDS" Default="43200" Mode="" Description="Session lifetime in seconds (900-604800)." Type="Variable" Display="advanced" Required="true" Mask="false">43200</Config>
<Config Name="Public Base URL" Target="GEOINTEL_PUBLIC_BASE_URL" Default="http://localhost:1202" Mode="" Description="Public browser origin. Must be HTTPS when Authentik is configured." Type="Variable" Display="advanced" Required="true" Mask="false">http://localhost:1202</Config>
<Config Name="Authentik Issuer" Target="GEOINTEL_AUTHENTIK_ISSUER" Default="" Mode="" Description="Optional HTTPS OIDC issuer. Configure all Authentik values together." Type="Variable" Display="advanced" Required="false" Mask="false"></Config>
<Config Name="Authentik Client ID" Target="GEOINTEL_AUTHENTIK_CLIENT_ID" Default="" Mode="" Description="OIDC client identifier for GeoIntel." Type="Variable" Display="advanced" Required="false" Mask="false"></Config>
<Config Name="Authentik Client Secret" Target="GEOINTEL_AUTHENTIK_CLIENT_SECRET" Default="" Mode="" Description="OIDC client secret; never commit this value." Type="Variable" Display="advanced" Required="false" Mask="true"></Config>
<Config Name="Authentik Allowed Email" Target="GEOINTEL_AUTHENTIK_ALLOWED_EMAIL" Default="" Mode="" Description="Exact verified operator e-mail address allowed to create an operator session." Type="Variable" Display="advanced" Required="false" Mask="false"></Config>
<Config Name="Guest Demo Enabled" Target="GEOINTEL_GUEST_ACCESS_ENABLED" Default="true" Mode="" Description="Show a guest button that opens only the seeded, restricted demo workspace. Enabled by default when operator login is active; set false on non-demo instances." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config> <Config Name="Guest Demo Enabled" Target="GEOINTEL_GUEST_ACCESS_ENABLED" Default="true" Mode="" Description="Show a guest button that opens only the seeded, restricted demo workspace. Enabled by default when operator login is active; set false on non-demo instances." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
<Config Name="Guest Display Name" Target="GEOINTEL_GUEST_DISPLAY_NAME" Default="Gast" Mode="" Description="Label shown for the temporary guest session." Type="Variable" Display="advanced" Required="true" Mask="false">Gast</Config> <Config Name="Guest Display Name" Target="GEOINTEL_GUEST_DISPLAY_NAME" Default="Gast" Mode="" Description="Label shown for the temporary guest session." Type="Variable" Display="advanced" Required="true" Mask="false">Gast</Config>
<Config Name="Guest Session TTL" Target="GEOINTEL_GUEST_SESSION_TTL_SECONDS" Default="7200" Mode="" Description="Temporary guest session lifetime in seconds (900-86400)." Type="Variable" Display="advanced" Required="true" Mask="false">7200</Config> <Config Name="Guest Session TTL" Target="GEOINTEL_GUEST_SESSION_TTL_SECONDS" Default="7200" Mode="" Description="Temporary guest session lifetime in seconds (900-86400)." Type="Variable" Display="advanced" Required="true" Mask="false">7200</Config>
+13 -2
View File
@@ -34,6 +34,8 @@ GEOINTEL_CORS_ORIGINS=https://geointel.itworx.tech,http://geointel.itworx.tech,h
# Upload guard in MiB. The same 1-2048 limit is applied by nginx and FastAPI. # Upload guard in MiB. The same 1-2048 limit is applied by nginx and FastAPI.
GEOINTEL_MAX_UPLOAD_MB=500 GEOINTEL_MAX_UPLOAD_MB=500
# Maximum decompressed vector payload processed fully in memory (1-256 MiB).
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB=64
GEOINTEL_AOI_WORKER_ENABLED=true GEOINTEL_AOI_WORKER_ENABLED=true
GEOINTEL_AOI_WORKER_POLL_SECONDS=2 GEOINTEL_AOI_WORKER_POLL_SECONDS=2
@@ -46,6 +48,14 @@ GEOINTEL_AUTH_PASSWORD_HASH=
GEOINTEL_AUTH_SESSION_SECRET= GEOINTEL_AUTH_SESSION_SECRET=
GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200 GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200
# Optional additive Authentik OIDC login. Configure all fields together. The
# local operator credentials above remain the recovery login.
GEOINTEL_PUBLIC_BASE_URL=http://localhost:1202
GEOINTEL_AUTHENTIK_ISSUER=
GEOINTEL_AUTHENTIK_CLIENT_ID=
GEOINTEL_AUTHENTIK_CLIENT_SECRET=
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL=
# Guest access is enabled by default whenever operator authentication is active. # Guest access is enabled by default whenever operator authentication is active.
# It opens the seeded GeoIntel demo in a temporary, API-enforced restricted # It opens the seeded GeoIntel demo in a temporary, API-enforced restricted
# session. Set this to false on installations containing private project data. # session. Set this to false on installations containing private project data.
@@ -153,8 +163,9 @@ SPW_TERRAIN_ANALYSIS_RESOLUTION_M=5
SPW_TERRAIN_MAX_SIDE_M=20000 SPW_TERRAIN_MAX_SIDE_M=20000
SPW_TERRAIN_MAX_PIXELS=12000000 SPW_TERRAIN_MAX_PIXELS=12000000
# Optional configured-YOLO runtime. Keep disabled unless a local model is mounted. # Production releases always contain the pinned AI dependencies. Inference may
GEOINTEL_INSTALL_AI=false # remain disabled until an integrity-bound local model is mounted.
GEOINTEL_INSTALL_AI=true
YOLO_ENABLED=false YOLO_ENABLED=false
YOLO_MODELS_DIR=/app/models YOLO_MODELS_DIR=/app/models
YOLO_MODEL_PATH= YOLO_MODEL_PATH=
+43 -2
View File
@@ -1,3 +1,18 @@
geo $geointel_trusted_forwarder {
default 0;
127.0.0.0/8 1;
::1/128 1;
# The outer Nginx Proxy Manager reaches this container through Docker's
# internal bridge; public/LAN clients are not trusted forwarders.
172.16.0.0/12 1;
}
map "$geointel_trusted_forwarder:$http_x_forwarded_proto" $geointel_forwarded_proto {
default $scheme;
"1:https" https;
"1:http" http;
}
server { server {
listen 80; listen 80;
server_name _; server_name _;
@@ -5,21 +20,42 @@ server {
proxy_read_timeout 600s; proxy_read_timeout 600s;
proxy_send_timeout 600s; proxy_send_timeout 600s;
add_header Content-Security-Policy "frame-ancestors 'none'" always;
add_header X-Frame-Options "DENY" always;
add_header X-Content-Type-Options "nosniff" always;
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
root /usr/share/nginx/html; root /usr/share/nginx/html;
index index.html; index index.html;
location = /index.html { location = /index.html {
add_header Cache-Control "no-cache"; add_header Cache-Control "no-cache";
add_header Content-Security-Policy "frame-ancestors 'none'" always;
add_header X-Frame-Options "DENY" always;
add_header X-Content-Type-Options "nosniff" always;
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
try_files /index.html =404; try_files /index.html =404;
} }
location = /geointel-icon.svg { location = /geointel-icon.svg {
add_header Cache-Control "public, max-age=3600"; add_header Cache-Control "public, max-age=3600";
add_header Content-Security-Policy "frame-ancestors 'none'" always;
add_header X-Frame-Options "DENY" always;
add_header X-Content-Type-Options "nosniff" always;
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
try_files /geointel-icon.svg =404; try_files /geointel-icon.svg =404;
} }
location = /geointel-icon.png { location = /geointel-icon.png {
add_header Cache-Control "public, max-age=3600"; add_header Cache-Control "public, max-age=3600";
add_header Content-Security-Policy "frame-ancestors 'none'" always;
add_header X-Frame-Options "DENY" always;
add_header X-Content-Type-Options "nosniff" always;
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
# Preserve the stable DockerMan/public URL while the frontend keeps # Preserve the stable DockerMan/public URL while the frontend keeps
# its explicit 32px and 180px icon variants. # its explicit 32px and 180px icon variants.
try_files /geointel-icon-180.png =404; try_files /geointel-icon-180.png =404;
@@ -27,6 +63,11 @@ server {
location /assets/ { location /assets/ {
add_header Cache-Control "no-cache"; add_header Cache-Control "no-cache";
add_header Content-Security-Policy "frame-ancestors 'none'" always;
add_header X-Frame-Options "DENY" always;
add_header X-Content-Type-Options "nosniff" always;
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
try_files $uri =404; try_files $uri =404;
} }
@@ -36,7 +77,7 @@ server {
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;
} }
location = /health { location = /health {
@@ -45,7 +86,7 @@ server {
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;
} }
location = /health/live { location = /health/live {
+323
View File
@@ -0,0 +1,323 @@
#!/usr/bin/env bash
set -euo pipefail
# Restore the production database from a verified pre-deploy dump while the
# normal GeoIntel container is stopped. This is intentionally a separate,
# explicitly confirmed operation: starting an older image against a schema
# migrated by a newer image is not a safe rollback strategy.
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$ROOT"
GEOINTEL_DEPLOY_LOCK_FILE="${GEOINTEL_DEPLOY_LOCK_FILE:-/tmp/geointel-release-deploy.lock}"
if [ "${GEOINTEL_DEPLOY_LOCK_HELD:-false}" != "true" ]; then
command -v flock >/dev/null 2>&1 || {
echo "GeoIntel database restore requires flock to prevent concurrent deployment." >&2
exit 2
}
exec 9>"$GEOINTEL_DEPLOY_LOCK_FILE"
if ! flock -n 9; then
echo "Another GeoIntel deployment or rollback is already running." >&2
exit 3
fi
fi
BACKUP_DIR=""
CONFIRMED="false"
RESTORE_IMAGE="${GEOINTEL_ROLLBACK_IMAGE:-}"
GEOINTEL_CONTAINER_NAME="${GEOINTEL_CONTAINER_NAME:-geointel}"
usage() {
cat <<'EOF'
Usage: bash deploy/unraid/restore-predeploy-database.sh \
--backup-dir PATH --confirm-production-database-restore [options]
Stops the normal GeoIntel container, starts an isolated PostGIS recovery
container on the same persistent database path, restores the checksum-verified
custom-format dump, validates Alembic/table counts, and stops recovery again.
The caller must start the rollback image after this command succeeds.
Options:
--image IMAGE Recovery image containing PostgreSQL/PostGIS tools
--container NAME Normal application container (default: geointel)
EOF
}
while [ "$#" -gt 0 ]; do
case "$1" in
--backup-dir) BACKUP_DIR="$2"; shift 2 ;;
--confirm-production-database-restore) CONFIRMED="true"; shift ;;
--image) RESTORE_IMAGE="$2"; shift 2 ;;
--container) GEOINTEL_CONTAINER_NAME="$2"; shift 2 ;;
--help|-h) usage; exit 0 ;;
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
esac
done
if [ "$CONFIRMED" != "true" ] || [ -z "$BACKUP_DIR" ]; then
echo "Explicit --confirm-production-database-restore and --backup-dir are required." >&2
exit 2
fi
for required in docker python3 sha256sum; do
command -v "$required" >/dev/null 2>&1 || {
echo "Missing required command: $required" >&2
exit 2
}
done
if [ -f .env ]; then
set -a
# shellcheck disable=SC1091
. ./.env
set +a
fi
GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}"
GEOINTEL_POSTGRES_DB="${GEOINTEL_POSTGRES_DB:-geointel}"
GEOINTEL_POSTGRES_USER="${GEOINTEL_POSTGRES_USER:-geointel}"
GEOINTEL_POSTGRES_PASSWORD="${GEOINTEL_POSTGRES_PASSWORD:-}"
if ! [[ "$GEOINTEL_POSTGRES_DB" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] \
|| ! [[ "$GEOINTEL_POSTGRES_USER" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
echo "Configured PostGIS database and user names must be simple SQL identifiers." >&2
exit 2
fi
case "$GEOINTEL_POSTGRES_PASSWORD" in
''|geointel|postgres|password|changeme|change-me-before-shared-use)
echo "Refusing database restore with an empty or known-default PostGIS password." >&2
exit 2
;;
esac
test -f "$GEOINTEL_POSTGIS_DATA_PATH/PG_VERSION" || {
echo "Persistent PostGIS data path is not initialized: $GEOINTEL_POSTGIS_DATA_PATH" >&2
exit 3
}
GEOINTEL_BACKUPS_PATH="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$GEOINTEL_BACKUPS_PATH")"
BACKUP_DIR="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$BACKUP_DIR")"
python3 - "$GEOINTEL_BACKUPS_PATH" "$BACKUP_DIR" <<'PY'
import pathlib
import sys
root = pathlib.Path(sys.argv[1])
backup = pathlib.Path(sys.argv[2])
try:
backup.relative_to(root)
except ValueError as exc:
raise SystemExit(f"Backup directory must be below {root}") from exc
if backup == root:
raise SystemExit("Backup directory must identify one immutable backup")
PY
for required_file in manifest.json database.dump database.list database-metadata.tsv table-counts.tsv CHECKSUMS.sha256; do
test -s "$BACKUP_DIR/$required_file" || {
echo "Missing or empty backup artifact: $required_file" >&2
exit 3
}
done
(
cd "$BACKUP_DIR"
sha256sum -c CHECKSUMS.sha256
)
python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$BACKUP_DIR"
IFS=$'\t' read -r BACKUP_DB BACKUP_USER BACKUP_IMAGE_ID BACKUP_RELEASE_ID < <(
python3 - "$BACKUP_DIR/manifest.json" <<'PY'
import json
import pathlib
import sys
payload = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))
if payload.get("schema_version") != 1 or payload.get("read_only_source") is not True:
raise SystemExit("Unsupported or unsafe backup manifest")
print(
f"{payload.get('database_name', '')}\t{payload.get('database_user', '')}\t"
f"{payload.get('image_id', '')}\t{payload.get('release_id', '')}"
)
PY
)
if [ "$BACKUP_DB" != "$GEOINTEL_POSTGRES_DB" ] || [ "$BACKUP_USER" != "$GEOINTEL_POSTGRES_USER" ]; then
echo "Backup database identity does not match the configured production database." >&2
exit 3
fi
if ! [[ "$BACKUP_IMAGE_ID" =~ ^sha256:[0-9a-f]{64}$ ]]; then
echo "Backup manifest does not contain one immutable Docker image ID." >&2
exit 3
fi
if [ -z "$RESTORE_IMAGE" ]; then
RESTORE_IMAGE="$BACKUP_IMAGE_ID"
fi
docker image inspect "$RESTORE_IMAGE" >/dev/null
RESTORE_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$RESTORE_IMAGE")"
if [ -z "$BACKUP_IMAGE_ID" ] || [ "$BACKUP_IMAGE_ID" != "$RESTORE_IMAGE_ID" ]; then
echo "Backup image identity does not match the retained rollback image." >&2
exit 3
fi
case "$BACKUP_RELEASE_ID" in
predeploy-*) ;;
*) echo "Production rollback requires a predeploy backup." >&2; exit 3 ;;
esac
if docker ps -a --format '{{.Names}}' | grep -Fxq "$GEOINTEL_CONTAINER_NAME"; then
docker rm -f "$GEOINTEL_CONTAINER_NAME" >/dev/null
fi
RECOVERY_CONTAINER="geointel-db-restore-$(date -u +%Y%m%d%H%M%S)-$$"
RESTORE_PROOF_DB="geointel_restore_proof_$(date -u +%Y%m%d%H%M%S)_$$"
RECOVERY_DB="geointel_pre_restore_$(date -u +%Y%m%d%H%M%S)_$$"
FAILED_RESTORE_DB="geointel_failed_restore_$(date -u +%Y%m%d%H%M%S)_$$"
SWAP_COMPLETE="false"
cleanup_recovery() {
if [ "$SWAP_COMPLETE" != "true" ] \
&& [ "$(docker inspect -f '{{.State.Running}}' "$RECOVERY_CONTAINER" 2>/dev/null || true)" = "true" ]; then
docker exec "$RECOVERY_CONTAINER" dropdb --if-exists --force \
-U "$GEOINTEL_POSTGRES_USER" "$RESTORE_PROOF_DB" >/dev/null 2>&1 || true
fi
docker rm -f "$RECOVERY_CONTAINER" >/dev/null 2>&1 || true
}
trap cleanup_recovery EXIT
docker run -d \
--name "$RECOVERY_CONTAINER" \
--restart no \
-e PGDATA=/var/lib/postgresql/data \
-e PGPASSWORD="$GEOINTEL_POSTGRES_PASSWORD" \
-v "$GEOINTEL_POSTGIS_DATA_PATH:/var/lib/postgresql/data" \
-v "$BACKUP_DIR:/restore:ro" \
--entrypoint /bin/bash \
"$RESTORE_IMAGE" \
-c 'set -euo pipefail; chown postgres:postgres "$PGDATA"; exec gosu postgres postgres' \
>/dev/null
for attempt in $(seq 1 180); do
if docker exec "$RECOVERY_CONTAINER" pg_isready -h 127.0.0.1 -U "$GEOINTEL_POSTGRES_USER" -d postgres >/dev/null 2>&1; then
break
fi
if [ "$(docker inspect -f '{{.State.Running}}' "$RECOVERY_CONTAINER" 2>/dev/null || true)" != "true" ]; then
echo "Database recovery container exited before PostGIS became ready." >&2
docker logs "$RECOVERY_CONTAINER" >&2 || true
exit 4
fi
if [ "$attempt" -eq 180 ]; then
echo "PostGIS recovery did not become ready within six minutes." >&2
exit 4
fi
sleep 2
done
RESTORED_LIST="$(mktemp)"
trap 'rm -f -- "$RESTORED_LIST"; cleanup_recovery' EXIT
docker exec "$RECOVERY_CONTAINER" pg_restore --list /restore/database.dump > "$RESTORED_LIST"
cmp -s "$RESTORED_LIST" "$BACKUP_DIR/database.list" || {
echo "Recovery image reads a different PostgreSQL archive listing." >&2
exit 4
}
if ! docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
-U "$GEOINTEL_POSTGRES_USER" -d postgres -Atqc \
"SELECT 1 FROM pg_database WHERE datname = '${GEOINTEL_POSTGRES_DB}';" | grep -Fxq 1; then
echo "Configured production database does not exist; refusing replacement." >&2
exit 4
fi
for generated_database in "$RESTORE_PROOF_DB" "$RECOVERY_DB" "$FAILED_RESTORE_DB"; do
if docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
-U "$GEOINTEL_POSTGRES_USER" -d postgres -Atqc \
"SELECT 1 FROM pg_database WHERE datname = '${generated_database}';" | grep -Fxq 1; then
echo "Generated recovery database already exists: ${generated_database}" >&2
exit 4
fi
done
# Prove the complete archive in a separate database before touching production.
docker exec "$RECOVERY_CONTAINER" createdb \
-U "$GEOINTEL_POSTGRES_USER" "$RESTORE_PROOF_DB"
docker exec "$RECOVERY_CONTAINER" pg_restore \
--exit-on-error \
--no-owner \
--no-privileges \
-U "$GEOINTEL_POSTGRES_USER" \
-d "$RESTORE_PROOF_DB" \
/restore/database.dump
EXPECTED_HEAD="$(awk -F $'\t' '$1 == "alembic_head" { print $2 }' "$BACKUP_DIR/database-metadata.tsv")"
validate_restored_database() {
local database_name="$1"
local restored_head=""
restored_head="$(docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
-U "$GEOINTEL_POSTGRES_USER" -d "$database_name" -Atqc \
'SELECT version_num FROM alembic_version;')"
if [ -z "$EXPECTED_HEAD" ] || [ "$restored_head" != "$EXPECTED_HEAD" ]; then
echo "Restored Alembic head '$restored_head' differs from backup head '$EXPECTED_HEAD'." >&2
return 1
fi
while IFS=$'\t' read -r table expected; do
[[ "$table" =~ ^[a-z_]+$ ]] || {
echo "Unsafe table name in retained counts: $table" >&2
return 1
}
actual="$(docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
-U "$GEOINTEL_POSTGRES_USER" -d "$database_name" -Atqc \
"SELECT count(*) FROM public.${table};")"
if [ "$actual" != "$expected" ]; then
echo "Restored count mismatch for $table: expected $expected, got $actual." >&2
return 1
fi
done < "$BACKUP_DIR/table-counts.tsv"
}
validate_restored_database "$RESTORE_PROOF_DB"
echo "Isolated predeploy restore proof passed: ${RESTORE_PROOF_DB}"
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IN ('${GEOINTEL_POSTGRES_DB}', '${RESTORE_PROOF_DB}') AND pid <> pg_backend_pid();" \
>/dev/null
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
"ALTER DATABASE ${GEOINTEL_POSTGRES_DB} RENAME TO ${RECOVERY_DB};"
if ! docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
"ALTER DATABASE ${RESTORE_PROOF_DB} RENAME TO ${GEOINTEL_POSTGRES_DB};"; then
echo "Restored database cutover failed; restoring the untouched production database name." >&2
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
"ALTER DATABASE ${RECOVERY_DB} RENAME TO ${GEOINTEL_POSTGRES_DB};"
exit 4
fi
SWAP_COMPLETE="true"
if ! validate_restored_database "$GEOINTEL_POSTGRES_DB"; then
echo "Post-cutover validation failed; restoring the retained pre-restore database." >&2
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${GEOINTEL_POSTGRES_DB}' AND pid <> pg_backend_pid();" \
>/dev/null
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
"ALTER DATABASE ${GEOINTEL_POSTGRES_DB} RENAME TO ${FAILED_RESTORE_DB};"
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
"ALTER DATABASE ${RECOVERY_DB} RENAME TO ${GEOINTEL_POSTGRES_DB};"
SWAP_COMPLETE="false"
echo "Original production database was restored; failed restore retained as ${FAILED_RESTORE_DB}." >&2
exit 4
fi
if [ -z "$RECOVERY_DB" ]; then
echo "Recovery database identity was not retained." >&2
exit 4
fi
while IFS=$'\t' read -r table expected; do
[[ "$table" =~ ^[a-z_]+$ ]] || {
echo "Unsafe table name in retained counts: $table" >&2
exit 4
}
done < "$BACKUP_DIR/table-counts.tsv"
rm -f -- "$RESTORED_LIST"
cleanup_recovery
trap - EXIT
echo "Production database restored and verified from: $BACKUP_DIR"
echo "Pre-restore production database retained for operator recovery as: $RECOVERY_DB"
+68 -4
View File
@@ -4,14 +4,78 @@ set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$ROOT" cd "$ROOT"
GEOINTEL_ROLLBACK_IMAGE="${GEOINTEL_ROLLBACK_IMAGE:-geointel-all-in-one:previous}" GEOINTEL_DEPLOY_LOCK_FILE="${GEOINTEL_DEPLOY_LOCK_FILE:-/tmp/geointel-release-deploy.lock}"
if [ "${GEOINTEL_DEPLOY_LOCK_HELD:-false}" != "true" ]; then
command -v flock >/dev/null 2>&1 || {
echo "GeoIntel rollback requires flock to prevent concurrent deployment." >&2
exit 2
}
exec 9>"$GEOINTEL_DEPLOY_LOCK_FILE"
if ! flock -n 9; then
echo "Another GeoIntel deployment or rollback is already running." >&2
exit 3
fi
GEOINTEL_DEPLOY_LOCK_HELD="true"
export GEOINTEL_DEPLOY_LOCK_HELD
fi
if ! docker image inspect "$GEOINTEL_ROLLBACK_IMAGE" >/dev/null 2>&1; then GEOINTEL_ROLLBACK_IMAGE="${GEOINTEL_ROLLBACK_IMAGE:-}"
echo "Rollback image does not exist: ${GEOINTEL_ROLLBACK_IMAGE}" >&2 BACKUP_DIR="${GEOINTEL_ROLLBACK_BACKUP_DIR:-}"
CONFIRM_RESTORE="false"
usage() {
cat <<'EOF'
Usage: bash deploy/unraid/rollback-dockerman-container.sh \
--backup-dir PATH --confirm-production-database-restore
Restores the verified pre-deploy PostgreSQL dump first and only then starts the
retained previous image. Image-only rollback against an unknown migrated
schema is deliberately not supported.
EOF
}
while [ "$#" -gt 0 ]; do
case "$1" in
--backup-dir) BACKUP_DIR="$2"; shift 2 ;;
--confirm-production-database-restore) CONFIRM_RESTORE="true"; shift ;;
--help|-h) usage; exit 0 ;;
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
esac
done
if [ -z "$BACKUP_DIR" ] || [ "$CONFIRM_RESTORE" != "true" ]; then
echo "Rollback requires a verified pre-deploy backup and explicit database-restore confirmation." >&2
usage >&2
exit 2 exit 2
fi fi
echo "Starting rollback image ${GEOINTEL_ROLLBACK_IMAGE} without changing persistent volumes..." restore_image_args=()
if [ -n "$GEOINTEL_ROLLBACK_IMAGE" ]; then
restore_image_args=(--image "$GEOINTEL_ROLLBACK_IMAGE")
fi
echo "Restoring the pre-deploy database before starting its checksum-bound image..."
bash deploy/unraid/restore-predeploy-database.sh \
--backup-dir "$BACKUP_DIR" \
"${restore_image_args[@]}" \
--confirm-production-database-restore
if [ -z "$GEOINTEL_ROLLBACK_IMAGE" ]; then
GEOINTEL_ROLLBACK_IMAGE="$(python3 - "$BACKUP_DIR/manifest.json" <<'PY'
import json
import pathlib
import re
import sys
image_id = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")).get("image_id", "")
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
raise SystemExit("Backup manifest lacks an immutable rollback image ID")
print(image_id)
PY
)"
fi
docker image inspect "$GEOINTEL_ROLLBACK_IMAGE" >/dev/null
echo "Starting rollback image ${GEOINTEL_ROLLBACK_IMAGE} with the restored persistent database..."
GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE" bash deploy/unraid/run-dockerman-container.sh GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE" bash deploy/unraid/run-dockerman-container.sh
for attempt in $(seq 1 90); do for attempt in $(seq 1 90); do
+53
View File
@@ -33,6 +33,7 @@ GEOINTEL_POSTGRES_USER="${GEOINTEL_POSTGRES_USER:-geointel}"
GEOINTEL_POSTGRES_PASSWORD="${GEOINTEL_POSTGRES_PASSWORD:-}" GEOINTEL_POSTGRES_PASSWORD="${GEOINTEL_POSTGRES_PASSWORD:-}"
GEOINTEL_CORS_ORIGINS="${GEOINTEL_CORS_ORIGINS:-http://localhost:${GEOINTEL_FRONTEND_PORT},http://127.0.0.1:${GEOINTEL_FRONTEND_PORT},http://192.168.10.150:${GEOINTEL_FRONTEND_PORT}}" GEOINTEL_CORS_ORIGINS="${GEOINTEL_CORS_ORIGINS:-http://localhost:${GEOINTEL_FRONTEND_PORT},http://127.0.0.1:${GEOINTEL_FRONTEND_PORT},http://192.168.10.150:${GEOINTEL_FRONTEND_PORT}}"
GEOINTEL_MAX_UPLOAD_MB="${GEOINTEL_MAX_UPLOAD_MB:-500}" GEOINTEL_MAX_UPLOAD_MB="${GEOINTEL_MAX_UPLOAD_MB:-500}"
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}"
GEOINTEL_AOI_WORKER_ENABLED="${GEOINTEL_AOI_WORKER_ENABLED:-true}" GEOINTEL_AOI_WORKER_ENABLED="${GEOINTEL_AOI_WORKER_ENABLED:-true}"
GEOINTEL_AOI_WORKER_POLL_SECONDS="${GEOINTEL_AOI_WORKER_POLL_SECONDS:-2}" GEOINTEL_AOI_WORKER_POLL_SECONDS="${GEOINTEL_AOI_WORKER_POLL_SECONDS:-2}"
GEOINTEL_AUTH_ENABLED="${GEOINTEL_AUTH_ENABLED:-false}" GEOINTEL_AUTH_ENABLED="${GEOINTEL_AUTH_ENABLED:-false}"
@@ -40,6 +41,11 @@ GEOINTEL_AUTH_USERNAME="${GEOINTEL_AUTH_USERNAME:-}"
GEOINTEL_AUTH_PASSWORD_HASH="${GEOINTEL_AUTH_PASSWORD_HASH:-}" GEOINTEL_AUTH_PASSWORD_HASH="${GEOINTEL_AUTH_PASSWORD_HASH:-}"
GEOINTEL_AUTH_SESSION_SECRET="${GEOINTEL_AUTH_SESSION_SECRET:-}" GEOINTEL_AUTH_SESSION_SECRET="${GEOINTEL_AUTH_SESSION_SECRET:-}"
GEOINTEL_AUTH_SESSION_TTL_SECONDS="${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200}" GEOINTEL_AUTH_SESSION_TTL_SECONDS="${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200}"
GEOINTEL_PUBLIC_BASE_URL="${GEOINTEL_PUBLIC_BASE_URL:-http://localhost:${GEOINTEL_FRONTEND_PORT}}"
GEOINTEL_AUTHENTIK_ISSUER="${GEOINTEL_AUTHENTIK_ISSUER:-}"
GEOINTEL_AUTHENTIK_CLIENT_ID="${GEOINTEL_AUTHENTIK_CLIENT_ID:-}"
GEOINTEL_AUTHENTIK_CLIENT_SECRET="${GEOINTEL_AUTHENTIK_CLIENT_SECRET:-}"
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL="${GEOINTEL_AUTHENTIK_ALLOWED_EMAIL:-}"
GEOINTEL_GUEST_ACCESS_ENABLED="${GEOINTEL_GUEST_ACCESS_ENABLED:-true}" GEOINTEL_GUEST_ACCESS_ENABLED="${GEOINTEL_GUEST_ACCESS_ENABLED:-true}"
GEOINTEL_GUEST_DISPLAY_NAME="${GEOINTEL_GUEST_DISPLAY_NAME:-Gast}" GEOINTEL_GUEST_DISPLAY_NAME="${GEOINTEL_GUEST_DISPLAY_NAME:-Gast}"
GEOINTEL_GUEST_SESSION_TTL_SECONDS="${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200}" GEOINTEL_GUEST_SESSION_TTL_SECONDS="${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200}"
@@ -189,6 +195,17 @@ validate_runtime_config() {
return 2 return 2
fi fi
case "$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" in
''|*[!0-9]*)
echo "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be a whole number." >&2
return 2
;;
esac
if [ "$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" -lt 1 ] || [ "$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" -gt 256 ]; then
echo "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be between 1 and 256." >&2
return 2
fi
case "$GEOINTEL_AUTH_ENABLED" in case "$GEOINTEL_AUTH_ENABLED" in
true|false) ;; true|false) ;;
*) *)
@@ -212,6 +229,36 @@ validate_runtime_config() {
esac esac
fi fi
local authentik_count=0
local authentik_value
for authentik_value in \
"$GEOINTEL_AUTHENTIK_ISSUER" \
"$GEOINTEL_AUTHENTIK_CLIENT_ID" \
"$GEOINTEL_AUTHENTIK_CLIENT_SECRET" \
"$GEOINTEL_AUTHENTIK_ALLOWED_EMAIL"; do
if [ -n "$authentik_value" ]; then
authentik_count=$((authentik_count + 1))
fi
done
if [ "$authentik_count" -ne 0 ] && [ "$authentik_count" -ne 4 ]; then
echo "All GEOINTEL_AUTHENTIK_* values must be configured together." >&2
return 2
fi
if [ "$authentik_count" -eq 4 ]; then
if [ "$GEOINTEL_AUTH_ENABLED" != "true" ]; then
echo "GEOINTEL_AUTH_ENABLED must be true when Authentik is configured." >&2
return 2
fi
case "$GEOINTEL_AUTHENTIK_ISSUER" in
https://*) ;;
*) echo "GEOINTEL_AUTHENTIK_ISSUER must use HTTPS." >&2; return 2 ;;
esac
case "$GEOINTEL_PUBLIC_BASE_URL" in
https://*) ;;
*) echo "GEOINTEL_PUBLIC_BASE_URL must use HTTPS for Authentik." >&2; return 2 ;;
esac
fi
case "$GEOINTEL_GUEST_ACCESS_ENABLED" in case "$GEOINTEL_GUEST_ACCESS_ENABLED" in
true|false) ;; true|false) ;;
*) *)
@@ -314,6 +361,7 @@ docker run -d \
-e GEOINTEL_STORAGE_ROOT=/app/storage \ -e GEOINTEL_STORAGE_ROOT=/app/storage \
-e GEOINTEL_CORS_ORIGINS="$GEOINTEL_CORS_ORIGINS" \ -e GEOINTEL_CORS_ORIGINS="$GEOINTEL_CORS_ORIGINS" \
-e GEOINTEL_MAX_UPLOAD_MB="$GEOINTEL_MAX_UPLOAD_MB" \ -e GEOINTEL_MAX_UPLOAD_MB="$GEOINTEL_MAX_UPLOAD_MB" \
-e GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" \
-e GEOINTEL_AOI_WORKER_ENABLED="$GEOINTEL_AOI_WORKER_ENABLED" \ -e GEOINTEL_AOI_WORKER_ENABLED="$GEOINTEL_AOI_WORKER_ENABLED" \
-e GEOINTEL_AOI_WORKER_POLL_SECONDS="$GEOINTEL_AOI_WORKER_POLL_SECONDS" \ -e GEOINTEL_AOI_WORKER_POLL_SECONDS="$GEOINTEL_AOI_WORKER_POLL_SECONDS" \
-e GEOINTEL_AUTH_ENABLED="$GEOINTEL_AUTH_ENABLED" \ -e GEOINTEL_AUTH_ENABLED="$GEOINTEL_AUTH_ENABLED" \
@@ -321,6 +369,11 @@ docker run -d \
-e GEOINTEL_AUTH_PASSWORD_HASH="$GEOINTEL_AUTH_PASSWORD_HASH" \ -e GEOINTEL_AUTH_PASSWORD_HASH="$GEOINTEL_AUTH_PASSWORD_HASH" \
-e GEOINTEL_AUTH_SESSION_SECRET="$GEOINTEL_AUTH_SESSION_SECRET" \ -e GEOINTEL_AUTH_SESSION_SECRET="$GEOINTEL_AUTH_SESSION_SECRET" \
-e GEOINTEL_AUTH_SESSION_TTL_SECONDS="$GEOINTEL_AUTH_SESSION_TTL_SECONDS" \ -e GEOINTEL_AUTH_SESSION_TTL_SECONDS="$GEOINTEL_AUTH_SESSION_TTL_SECONDS" \
-e GEOINTEL_PUBLIC_BASE_URL="$GEOINTEL_PUBLIC_BASE_URL" \
-e GEOINTEL_AUTHENTIK_ISSUER="$GEOINTEL_AUTHENTIK_ISSUER" \
-e GEOINTEL_AUTHENTIK_CLIENT_ID="$GEOINTEL_AUTHENTIK_CLIENT_ID" \
-e GEOINTEL_AUTHENTIK_CLIENT_SECRET="$GEOINTEL_AUTHENTIK_CLIENT_SECRET" \
-e GEOINTEL_AUTHENTIK_ALLOWED_EMAIL="$GEOINTEL_AUTHENTIK_ALLOWED_EMAIL" \
-e GEOINTEL_GUEST_ACCESS_ENABLED="$GEOINTEL_GUEST_ACCESS_ENABLED" \ -e GEOINTEL_GUEST_ACCESS_ENABLED="$GEOINTEL_GUEST_ACCESS_ENABLED" \
-e GEOINTEL_GUEST_DISPLAY_NAME="$GEOINTEL_GUEST_DISPLAY_NAME" \ -e GEOINTEL_GUEST_DISPLAY_NAME="$GEOINTEL_GUEST_DISPLAY_NAME" \
-e GEOINTEL_GUEST_SESSION_TTL_SECONDS="$GEOINTEL_GUEST_SESSION_TTL_SECONDS" \ -e GEOINTEL_GUEST_SESSION_TTL_SECONDS="$GEOINTEL_GUEST_SESSION_TTL_SECONDS" \
+6
View File
@@ -21,11 +21,17 @@ services:
GEOINTEL_AOI_WORKER_POLL_SECONDS: ${GEOINTEL_AOI_WORKER_POLL_SECONDS:-2} GEOINTEL_AOI_WORKER_POLL_SECONDS: ${GEOINTEL_AOI_WORKER_POLL_SECONDS:-2}
GEOINTEL_CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202} GEOINTEL_CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202}
GEOINTEL_MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500} GEOINTEL_MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500}
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB: ${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}
GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-false} GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-false}
GEOINTEL_AUTH_USERNAME: ${GEOINTEL_AUTH_USERNAME:-} GEOINTEL_AUTH_USERNAME: ${GEOINTEL_AUTH_USERNAME:-}
GEOINTEL_AUTH_PASSWORD_HASH: ${GEOINTEL_AUTH_PASSWORD_HASH:-} GEOINTEL_AUTH_PASSWORD_HASH: ${GEOINTEL_AUTH_PASSWORD_HASH:-}
GEOINTEL_AUTH_SESSION_SECRET: ${GEOINTEL_AUTH_SESSION_SECRET:-} GEOINTEL_AUTH_SESSION_SECRET: ${GEOINTEL_AUTH_SESSION_SECRET:-}
GEOINTEL_AUTH_SESSION_TTL_SECONDS: ${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200} GEOINTEL_AUTH_SESSION_TTL_SECONDS: ${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200}
GEOINTEL_PUBLIC_BASE_URL: ${GEOINTEL_PUBLIC_BASE_URL:-http://localhost:1202}
GEOINTEL_AUTHENTIK_ISSUER: ${GEOINTEL_AUTHENTIK_ISSUER:-}
GEOINTEL_AUTHENTIK_CLIENT_ID: ${GEOINTEL_AUTHENTIK_CLIENT_ID:-}
GEOINTEL_AUTHENTIK_CLIENT_SECRET: ${GEOINTEL_AUTHENTIK_CLIENT_SECRET:-}
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL: ${GEOINTEL_AUTHENTIK_ALLOWED_EMAIL:-}
GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-true} GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-true}
GEOINTEL_GUEST_DISPLAY_NAME: ${GEOINTEL_GUEST_DISPLAY_NAME:-Gast} GEOINTEL_GUEST_DISPLAY_NAME: ${GEOINTEL_GUEST_DISPLAY_NAME:-Gast}
GEOINTEL_GUEST_SESSION_TTL_SECONDS: ${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200} GEOINTEL_GUEST_SESSION_TTL_SECONDS: ${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200}
+6
View File
@@ -23,11 +23,17 @@ services:
STORAGE_ROOT: /app/storage STORAGE_ROOT: /app/storage
CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202} CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202}
MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500} MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500}
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB: ${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}
GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-false} GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-false}
GEOINTEL_AUTH_USERNAME: ${GEOINTEL_AUTH_USERNAME:-} GEOINTEL_AUTH_USERNAME: ${GEOINTEL_AUTH_USERNAME:-}
GEOINTEL_AUTH_PASSWORD_HASH: ${GEOINTEL_AUTH_PASSWORD_HASH:-} GEOINTEL_AUTH_PASSWORD_HASH: ${GEOINTEL_AUTH_PASSWORD_HASH:-}
GEOINTEL_AUTH_SESSION_SECRET: ${GEOINTEL_AUTH_SESSION_SECRET:-} GEOINTEL_AUTH_SESSION_SECRET: ${GEOINTEL_AUTH_SESSION_SECRET:-}
GEOINTEL_AUTH_SESSION_TTL_SECONDS: ${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200} GEOINTEL_AUTH_SESSION_TTL_SECONDS: ${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200}
GEOINTEL_PUBLIC_BASE_URL: ${GEOINTEL_PUBLIC_BASE_URL:-http://localhost:1202}
GEOINTEL_AUTHENTIK_ISSUER: ${GEOINTEL_AUTHENTIK_ISSUER:-}
GEOINTEL_AUTHENTIK_CLIENT_ID: ${GEOINTEL_AUTHENTIK_CLIENT_ID:-}
GEOINTEL_AUTHENTIK_CLIENT_SECRET: ${GEOINTEL_AUTHENTIK_CLIENT_SECRET:-}
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL: ${GEOINTEL_AUTHENTIK_ALLOWED_EMAIL:-}
GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-true} GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-true}
GEOINTEL_GUEST_DISPLAY_NAME: ${GEOINTEL_GUEST_DISPLAY_NAME:-Gast} GEOINTEL_GUEST_DISPLAY_NAME: ${GEOINTEL_GUEST_DISPLAY_NAME:-Gast}
GEOINTEL_GUEST_SESSION_TTL_SECONDS: ${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200} GEOINTEL_GUEST_SESSION_TTL_SECONDS: ${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200}
+36 -20
View File
@@ -5,8 +5,10 @@ GeoIntel uses the same release gates in Gitea Actions and GitHub Actions:
- `.gitea/workflows/release-gates.yml` - `.gitea/workflows/release-gates.yml`
- `.github/workflows/release-gates.yml` - `.github/workflows/release-gates.yml`
Gitea is the operational source-control platform. The GitHub workflow is kept Gitea is the operational source-control platform. Its release workflow builds
equivalent so a mirror or external review does not receive a weaker gate. the production AI variant and is the only workflow that can automatically
deploy. The GitHub mirror builds and scans the same AI variant for external
review but is not a production deployment authority.
## Runner requirements ## Runner requirements
@@ -16,11 +18,11 @@ The `ubuntu-latest` runner must provide:
- Python 3.11 and Node 20 through the official setup actions; - Python 3.11 and Node 20 through the official setup actions;
- Bash and Docker with Compose v2; - Bash and Docker with Compose v2;
- permission to build images and mount `/var/run/docker.sock`; - permission to build images and mount `/var/run/docker.sock`;
- sufficient disk for the all-in-one GIS image and scanner databases. - sufficient disk for the all-in-one AI image and scanner databases.
The container job builds the GIS release variant only. PyTorch and Both container jobs build the explicit AI image, including the
Ultralytics remain in the optional `ai` extra and in the explicit AI image PyTorch/Ultralytics layers used on Tower. AI packages remain outside the
variant; CI does not silently make them base dependencies. standard backend lock and are pinned by Docker build arguments.
## Quality gate ## Quality gate
@@ -33,11 +35,25 @@ cd frontend && npm ci
``` ```
It then validates the lock policy and runs the complete readiness script. The It then validates the lock policy and runs the complete readiness script. The
readiness script covers backend compile/tests, contract audits, Alembic readiness script covers Ruff, repository-layout validation, backend
single-head, frontend typecheck/build and release-script syntax. CI also compile/tests, contract audits, Alembic single-head, frontend typecheck/build
and release-script syntax. CI also
renders offline migration SQL and resolved Compose configuration as retained renders offline migration SQL and resolved Compose configuration as retained
evidence. evidence.
Pull requests run managed validation against the real `backend/` and
`frontend/` projects plus the complete release gates. On a `main` push, the
Unraid deploy job has explicit `needs` dependencies on quality, dependency and
AI-container jobs. There is no separate deployment workflow: manual validation
uses `workflow_dispatch` on this same release-gates workflow and cannot skip
quality, dependency or AI-container jobs.
The deploy host requires a full controller commit SHA, builds the AI variant,
records its Docker image/config digest, generates SBOM and Trivy evidence for
that exact local image, and starts the immutable image ID. Deployment fails if
the running container ID, revision label or AI label differs from the retained
attestation.
## Reproducible Python lock ## Reproducible Python lock
`backend/requirements-runtime.lock` and `backend/requirements-ci.lock` are `backend/requirements-runtime.lock` and `backend/requirements-ci.lock` are
@@ -73,12 +89,12 @@ The dependency job:
- publishes both unfiltered and policy-filtered Python JSON reports plus the - publishes both unfiltered and policy-filtered Python JSON reports plus the
npm JSON report, including on failure. npm JSON report, including on failure.
The only current Python/container exceptions are the Starlette 2026 advisories recorded There are currently no Python or container vulnerability exceptions.
in `security/pip-audit-exceptions.json`. FastAPI 0.139.2 still constrains `security/pip-audit-exceptions.json` remains as a strict, machine-readable
Starlette below 0.53 while patched releases begin at 1.x. GeoIntel applies registry: every future exception must identify one advisory, package, specific
request-target, form-content, route-class and Linux-runtime compensating reason and expiry date. Readiness and CI fail on malformed or expired entries;
controls. The exception file has a mandatory review date; readiness and CI new advisories are never auto-ignored. GeoIntel requires Starlette 1.3.1 or
fail automatically after it expires. New advisories are never auto-ignored. newer and therefore no longer suppresses the five 2026 Starlette advisories.
The all-in-one image replaces the Go-based base-image `gosu` helper with a The all-in-one image replaces the Go-based base-image `gosu` helper with a
small `setpriv` exec wrapper and upgrades packaged setuptools/wheel metadata; small `setpriv` exec wrapper and upgrades packaged setuptools/wheel metadata;
the final runtime filesystem no longer exposes the vulnerable Go executable. the final runtime filesystem no longer exposes the vulnerable Go executable.
@@ -89,8 +105,8 @@ verified to contain the audited shell wrapper. This is not a vulnerability
exception: the raw evidence remains published and the runtime wrapper is exception: the raw evidence remains published and the runtime wrapper is
exercised during live release validation. exercised during live release validation.
The container job builds a non-AI all-in-one image and uses digest-pinned The operational container job builds the production AI all-in-one image and
scanner images: uses digest-pinned scanner images:
- Syft 1.44.0 generates an SPDX JSON SBOM; - Syft 1.44.0 generates an SPDX JSON SBOM;
- Trivy 0.70.0 generates a complete JSON vulnerability report; - Trivy 0.70.0 generates a complete JSON vulnerability report;
@@ -103,12 +119,12 @@ Run these controls on a Docker-enabled workstation:
```bash ```bash
docker build \ docker build \
-f deploy/unraid/Dockerfile.all-in-one \ -f deploy/unraid/Dockerfile.all-in-one \
--build-arg GEOINTEL_INSTALL_AI=false \ --build-arg GEOINTEL_INSTALL_AI=true \
--build-arg GEOINTEL_BUILD_SHA=local \ --build-arg GEOINTEL_BUILD_SHA=local \
--build-arg GEOINTEL_BUILD_TIME=local \ --build-arg GEOINTEL_BUILD_TIME=local \
-t geointel-ci:local . -t geointel-ci:local-ai .
bash scripts/generate_container_sbom.sh geointel-ci:local bash scripts/generate_container_sbom.sh geointel-ci:local-ai
bash scripts/scan_container_image.sh geointel-ci:local bash scripts/scan_container_image.sh geointel-ci:local-ai
``` ```
Outputs are written below ignored `artifacts/`; scanner cache is written below Outputs are written below ignored `artifacts/`; scanner cache is written below
+28 -7
View File
@@ -89,9 +89,9 @@ The normal project lifecycle archive path does not need destructive
confirmation because it changes only `status=archived` and preserves all confirmation because it changes only `status=archived` and preserves all
data. data.
## Destructive apply gate ## Recoverable quarantine gate
First create a fresh backup with a SHA-256 storage inventory on the host: First create a fresh backup with a byte-complete SHA-256 storage snapshot on the host:
```bash ```bash
bash scripts/backup_release_state.sh \ bash scripts/backup_release_state.sh \
@@ -112,7 +112,8 @@ bash scripts/verify_release_backup.sh \
``` ```
The Unraid runtime mounts `GEOINTEL_BACKUPS_PATH` read-only at `/app/backups`. The Unraid runtime mounts `GEOINTEL_BACKUPS_PATH` read-only at `/app/backups`.
Only after reviewing the dry run may an operator execute: Only after reviewing the dry run may an operator move the exact candidates to
a protected quarantine:
```bash ```bash
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
@@ -120,14 +121,34 @@ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
--max-delete <reviewed-candidate-count> \ --max-delete <reviewed-candidate-count> \
--backup-dir /app/backups/<release-id> \ --backup-dir /app/backups/<release-id> \
--backup-max-age-hours 24 \ --backup-max-age-hours 24 \
--confirm DELETE_STORAGE_ARTIFACTS \ --confirm QUARANTINE_STORAGE_ARTIFACTS \
--apply --apply
``` ```
The command re-runs the audit immediately before deletion. It refuses the The command re-runs the audit immediately before quarantine. It refuses the
operation when the exact token is absent, the candidate count exceeds the operation when the exact token is absent, the candidate count exceeds the
operator limit, the backup is stale/incomplete, checksums differ, the storage operator limit, the backup is stale/incomplete, snapshot checksums differ, or
inventory is not SHA-256, or the path is outside the cleanup allowlist. the path is outside the cleanup allowlist.
Every moved byte is hashed and retained below
`operator-evidence/cleanup-quarantine/<operation-id>/files/`; an atomic
manifest records its original path, quarantine path, size, checksum and
backup identity. Cleanup uses a backup-to-quarantine hard-link state machine:
the manifest records `planned`, `linked` and `quarantined` transitions so each
crash window can be reconciled without losing the retained inode.
Restore a reviewed quarantine without overwriting any path:
```bash
docker exec geointel python /app/scripts/restore_storage_quarantine.py \
--storage-root /app/storage \
--manifest /app/storage/operator-evidence/cleanup-quarantine/<operation-id>/manifest.json \
--confirm RESTORE_QUARANTINED_ARTIFACTS
```
Restore validates every retained checksum and never replaces an existing
destination. It uses an exclusive hard link and reconciles both-file and
one-file interruption states before updating the manifest, so an interrupted
restore can be resumed and verified.
The older demo-export cleanup has the same gate and uses confirmation token The older demo-export cleanup has the same gate and uses confirmation token
`DELETE_DEMO_EXPORTS`. `DELETE_DEMO_EXPORTS`.
+46 -7
View File
@@ -15,7 +15,7 @@ The repository version is stored in `VERSION`. The current release is
- clean `main` worktree at the commit being released; - clean `main` worktree at the commit being released;
- secure non-default PostGIS password in the Tower `.env`; - secure non-default PostGIS password in the Tower `.env`;
- existing local AI model only when the AI image is enabled; - existing local AI model only when the AI image is enabled;
- recent checksum-verified backup with SHA-256 storage inventory; - recent checksum-verified database dump and byte-complete storage/model snapshot;
- Docker, `ssh-keygen`, Python 3.11, Node 20 and Bash available; - Docker, `ssh-keygen`, Python 3.11, Node 20 and Bash available;
- one Alembic head and no unsupported metric represented as successful. - one Alembic head and no unsupported metric represented as successful.
@@ -39,6 +39,24 @@ docker compose config
## Immutable deployment ## Immutable deployment
`deploy-release.sh` first builds/reuses the candidate while the current release
stays available. Immediately before replacement it quiesces backend writes and
creates and checksum-verifies a database dump plus byte-complete SHA-256
storage/model snapshots under `/mnt/user/appdata/geointel/backups`. The first
snapshot copies every byte (using CoW reflinks when supported); later snapshots
hard-link only checksum-identical bytes from a verified older backup, never
from live storage. A conservative full-copy/free-space preflight runs before
backend quiescence. Only then may the
candidate start or run Alembic. If backup fails, the unchanged release is
restarted; deployment stops if existing PostGIS state cannot be backed up
consistently.
The backup manifest separates provenance intentionally:
`backup_tool_revision` is the new candidate source running the backup tool,
while `running_image_revision` is the OCI label of the old release whose data
is being captured. The retained Docker `image_id`, not either descriptive
revision field, is authoritative for rollback.
On the Codex workstation: On the Codex workstation:
```powershell ```powershell
@@ -58,8 +76,9 @@ bash scripts/live_migration_smoke.sh
## Backup and recovery proof ## Backup and recovery proof
Create an immutable backup. The SHA-256 inventory can take several minutes on Create an immutable byte-complete backup. Initial storage/model copy and
large storage: verification can be I/O-heavy; subsequent backups deduplicate unchanged bytes
against the newest verified prior snapshot:
```bash ```bash
bash scripts/backup_release_state.sh \ bash scripts/backup_release_state.sh \
@@ -124,17 +143,37 @@ gate fails on reachable fixed HIGH/CRITICAL findings.
## Rollback proof ## Rollback proof
The rollback command reuses persistent paths and never downgrades Alembic: Use the exact backup printed by the deployment. Rollback first stops the
candidate, restores and verifies the pre-deploy PostgreSQL dump in a temporary
proof database, then swaps database names while retaining the pre-restore
database as a recovery point. Only then does it start the exact image ID bound
into that backup; it never relies on a mutable global `previous` tag. It never runs an
Alembic downgrade or starts an old image against an unknown newer schema:
```bash ```bash
bash deploy/unraid/rollback-dockerman-container.sh bash deploy/unraid/rollback-dockerman-container.sh \
--backup-dir /mnt/user/appdata/geointel/backups/<predeploy-release-id> \
--confirm-production-database-restore
curl -fsS http://127.0.0.1:1202/health/ready curl -fsS http://127.0.0.1:1202/health/ready
bash deploy/unraid/deploy-release.sh bash deploy/unraid/deploy-release.sh
curl -fsS http://127.0.0.1:1202/health/ready curl -fsS http://127.0.0.1:1202/health/ready
``` ```
For a future backward-incompatible migration, restore the verified pre-release Database rollback restores persisted rows and schema. Files newly written by a
backup instead of running an older image against a newer schema. failed candidate remain in storage as unreferenced evidence; the protected,
recoverable quarantine flow in `DATA_OPERATIONS_RUNBOOK.md` handles those
files without deleting source data.
After the rollback has remained healthy and its retained evidence has been
reviewed, list the recovery database printed by the script. Remove it only by
an explicit, separately approved `dropdb` maintenance command; deployment and
rollback never auto-delete recovery databases or backup directories. Retain at
least the current successful predeploy backup and its predecessor. Before
removing an older backup, run `verify_release_backup.sh` on the backups that
remain and confirm no newer snapshot hard-links depend on operator policy for
retention (hard-linked bytes remain allocated while any retained backup names
them). Remove its `rollback-predeploy-*` image tag only in the same explicitly
reviewed retention operation.
## Tag and signed package ## Tag and signed package
+5 -4
View File
@@ -28,10 +28,11 @@ The restore smoke may only create databases whose name starts with
use `pg_restore --clean` and drops the temporary database unless an operator use `pg_restore --clean` and drops the temporary database unless an operator
explicitly asks to retain it. explicitly asks to retain it.
Storage and model files are inventoried rather than copied into the database Storage and model files are retained as byte-complete, SHA-256 verified
dump. Release backups must therefore be paired with the persistent storage snapshots alongside the database dump. Snapshot creation rejects symlinks and
volume backup policy. Use `--inventory-mode sha256` for final release special files. The first snapshot uses CoW reflinks when supported and falls
evidence. back to full copies; later snapshots hard-link checksum-identical files only
from a verified older backup, never from live storage.
An old persistent volume can also retain glibc collation metadata for the An old persistent volume can also retain glibc collation metadata for the
empty `postgres` and `template1` system databases. If `createdb` fails for that empty `postgres` and `template1` system databases. If `createdb` fails for that
+2
View File
@@ -5,3 +5,5 @@ __pycache__
.pytest_cache .pytest_cache
.vite .vite
.env .env
.env.*
!.env.example
+31 -2
View File
@@ -1,3 +1,16 @@
geo $geointel_trusted_forwarder {
default 0;
127.0.0.0/8 1;
::1/128 1;
172.16.0.0/12 1;
}
map "$geointel_trusted_forwarder:$http_x_forwarded_proto" $geointel_forwarded_proto {
default $scheme;
"1:https" https;
"1:http" http;
}
server { server {
listen 80; listen 80;
server_name _; server_name _;
@@ -5,16 +18,32 @@ server {
proxy_read_timeout 600s; proxy_read_timeout 600s;
proxy_send_timeout 600s; proxy_send_timeout 600s;
add_header Content-Security-Policy "frame-ancestors 'none'" always;
add_header X-Frame-Options "DENY" always;
add_header X-Content-Type-Options "nosniff" always;
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
root /usr/share/nginx/html; root /usr/share/nginx/html;
index index.html; index index.html;
location = /index.html { location = /index.html {
add_header Cache-Control "no-cache"; add_header Cache-Control "no-cache";
add_header Content-Security-Policy "frame-ancestors 'none'" always;
add_header X-Frame-Options "DENY" always;
add_header X-Content-Type-Options "nosniff" always;
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
try_files /index.html =404; try_files /index.html =404;
} }
location /assets/ { location /assets/ {
add_header Cache-Control "no-cache"; add_header Cache-Control "no-cache";
add_header Content-Security-Policy "frame-ancestors 'none'" always;
add_header X-Frame-Options "DENY" always;
add_header X-Content-Type-Options "nosniff" always;
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
try_files $uri =404; try_files $uri =404;
} }
@@ -24,7 +53,7 @@ server {
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;
} }
location = /health { location = /health {
@@ -33,7 +62,7 @@ server {
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;
} }
location = /health/live { location = /health/live {
+19 -9
View File
@@ -2142,11 +2142,17 @@ bash scripts/backup_release_state.sh \
``` ```
The backup is written atomically and contains a PostgreSQL custom-format dump, The backup is written atomically and contains a PostgreSQL custom-format dump,
archive listing, Alembic/PostGIS metadata, critical table counts, optional archive listing, Alembic/PostGIS metadata, critical table counts and
storage/model inventories and SHA-256 checksums. An empty or known-default byte-complete SHA-256-verified storage/model snapshots. The first snapshot is a
database password leaves the release gate failed. For an emergency backup full copy; a later deployment may hard-link only checksum-identical files from
before rotating that password, add `--allow-insecure-password`; the manifest another completed, fully verified backup with `--link-dest-backup`. It never
still records the insecure state. hard-links a live source file and never deletes an older backup. An empty or
known-default database password leaves the release gate failed. For an
emergency backup before rotating that password, add
`--allow-insecure-password`; the manifest still records the insecure state.
`backup_tool_revision` identifies the candidate source that executed the
backup; `running_image_revision` identifies the currently running old image.
Rollback is always bound to the retained immutable Docker `image_id`.
Verify without changing any database: Verify without changing any database:
@@ -2190,7 +2196,7 @@ docker exec geointel python /app/scripts/audit_data_operations.py \
--output /app/storage/release-evidence/rc-current/data-operations.json --output /app/storage/release-evidence/rc-current/data-operations.json
``` ```
Preview old unreferenced derived/cache/export candidates without deletion: Preview old unreferenced derived/cache/export candidates without mutation:
```bash ```bash
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
@@ -2199,10 +2205,14 @@ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
``` ```
Apply requires a reviewed candidate count, the exact Apply requires a reviewed candidate count, the exact
`DELETE_STORAGE_ARTIFACTS` token and a backup no older than 24 hours with a `QUARANTINE_STORAGE_ARTIFACTS` token and a backup no older than 24 hours with a
checksum-verified database dump and SHA-256 storage inventory. The host backup checksum-verified database dump and byte-complete storage snapshot. The host backup
root is mounted read-only at `/app/backups`. See root is mounted read-only at `/app/backups`. See
`docs/DATA_OPERATIONS_RUNBOOK.md`. No cleanup is scheduled by GeoIntel. `docs/DATA_OPERATIONS_RUNBOOK.md`. Candidates enter protected
`operator-evidence/cleanup-quarantine` storage through an interruption-safe
hard-link/unlink state machine. `restore_storage_quarantine.py` reverses that
move with the exact `RESTORE_QUARANTINED_ARTIFACTS` token and refuses to
overwrite an existing original path. No cleanup is scheduled by GeoIntel.
## RC-8 Belgium/North Sea release journeys ## RC-8 Belgium/North Sea release journeys
+14 -5
View File
@@ -14,17 +14,26 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
from typing import Any, Iterable from typing import Any, Iterable
from sqlalchemy import func
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
BACKEND_ROOT = ROOT / "backend" if (ROOT / "backend" / "app").is_dir() else ROOT BACKEND_ROOT = ROOT / "backend" if (ROOT / "backend" / "app").is_dir() else ROOT
if str(BACKEND_ROOT) not in sys.path: if str(BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(BACKEND_ROOT)) sys.path.insert(0, str(BACKEND_ROOT))
from app.core.config import get_settings from app.core.config import get_settings # noqa: E402 - imported after backend path bootstrap
from app.db.session import SessionLocal from app.db.session import SessionLocal # noqa: E402 - imported after backend path bootstrap
from sqlalchemy import func from app.models import ( # noqa: E402 - imported after backend path bootstrap
AnalysisRun,
from app.models import AnalysisRun, Dataset, DatasetVersion, Detection, Export, Job, Project, Segmentation Dataset,
DatasetVersion,
Detection,
Export,
Job,
Project,
Segmentation,
)
NATIONAL_PROJECT_NAME = "Belgium and North Sea Workbench" NATIONAL_PROJECT_NAME = "Belgium and North Sea Workbench"
+163 -48
View File
@@ -3,12 +3,14 @@ set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
CONTAINER="geointel" CONTAINER="geointel"
OUTPUT_ROOT="backups" OUTPUT_ROOT="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)" RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)"
STORAGE_PATH="" STORAGE_PATH=""
MODELS_PATH="" MODELS_PATH=""
INVENTORY_MODE="metadata" INVENTORY_MODE="metadata"
ALLOW_INSECURE_PASSWORD="false" ALLOW_INSECURE_PASSWORD="false"
LINK_DEST_BACKUP=""
ROLLBACK_IMAGE_TAG=""
usage() { usage() {
cat <<'EOF' cat <<'EOF'
@@ -19,11 +21,16 @@ GeoIntel container. It never deletes or restores application data.
Options: Options:
--container NAME Docker container (default: geointel) --container NAME Docker container (default: geointel)
--output-root PATH Host backup root (default: backups) --output-root PATH Host backup root (default:
/mnt/user/appdata/geointel/backups)
--release-id ID Safe backup directory name --release-id ID Safe backup directory name
--storage-path PATH Optional host storage path to inventory --storage-path PATH Host storage path to snapshot byte-for-byte
--models-path PATH Optional host model path to inventory --models-path PATH Host model path to snapshot byte-for-byte
--inventory-mode metadata|sha256 Hash all inventoried files only with sha256 --inventory-mode metadata|sha256 Retained manifest compatibility setting
--link-dest-backup PATH Verified older backup used only to hard-link
checksum-identical backup-to-backup files
--rollback-image-tag TAG Immutable backup-specific tag bound to the
running image ID
--allow-insecure-password Complete emergency backup despite an --allow-insecure-password Complete emergency backup despite an
empty/default production DB password empty/default production DB password
EOF EOF
@@ -37,6 +44,8 @@ while [ "$#" -gt 0 ]; do
--storage-path) STORAGE_PATH="$2"; shift 2 ;; --storage-path) STORAGE_PATH="$2"; shift 2 ;;
--models-path) MODELS_PATH="$2"; shift 2 ;; --models-path) MODELS_PATH="$2"; shift 2 ;;
--inventory-mode) INVENTORY_MODE="$2"; shift 2 ;; --inventory-mode) INVENTORY_MODE="$2"; shift 2 ;;
--link-dest-backup) LINK_DEST_BACKUP="$2"; shift 2 ;;
--rollback-image-tag) ROLLBACK_IMAGE_TAG="$2"; shift 2 ;;
--allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;; --allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;;
--help|-h) usage; exit 0 ;; --help|-h) usage; exit 0 ;;
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;; *) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
@@ -51,12 +60,93 @@ if [ "$INVENTORY_MODE" != "metadata" ] && [ "$INVENTORY_MODE" != "sha256" ]; the
echo "--inventory-mode must be metadata or sha256" >&2 echo "--inventory-mode must be metadata or sha256" >&2
exit 2 exit 2
fi fi
for required in docker python3 sha256sum git; do for required in docker python3 sha256sum; do
if ! command -v "$required" >/dev/null 2>&1; then if ! command -v "$required" >/dev/null 2>&1; then
echo "Missing required command: $required" >&2 echo "Missing required command: $required" >&2
exit 2 exit 2
fi fi
done done
resolve_source_revision() {
local controller_sha="" explicit_sha="${GEOINTEL_BUILD_SHA:-}"
local gitea_sha="${GITEA_COMMIT_SHA:-}" github_sha="${GITHUB_SHA:-}"
local git_head="" git_dirty="false" source=""
if [ -n "$gitea_sha" ]; then
if ! [[ "$gitea_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
echo "GITEA_COMMIT_SHA must contain one full 40-character Git commit SHA." >&2
return 2
fi
controller_sha="${gitea_sha,,}"
source="GITEA_COMMIT_SHA"
fi
if [ -n "$github_sha" ]; then
if ! [[ "$github_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2
return 2
fi
github_sha="${github_sha,,}"
if [ -n "$controller_sha" ] && [ "$controller_sha" != "$github_sha" ]; then
echo "Controller commit variables disagree." >&2
return 2
fi
controller_sha="$github_sha"
source="${source:-GITHUB_SHA}"
fi
if [ -n "$explicit_sha" ]; then
if ! [[ "$explicit_sha" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]]; then
echo "GEOINTEL_BUILD_SHA contains an unsafe release revision." >&2
return 2
fi
explicit_sha="${explicit_sha,,}"
fi
if [ -n "$controller_sha" ]; then
if [ -n "$explicit_sha" ] && [ "$explicit_sha" != "$controller_sha" ]; then
echo "GEOINTEL_BUILD_SHA differs from the controller revision." >&2
return 2
fi
explicit_sha="$controller_sha"
elif [ -n "${GITEA_REPOSITORY:-}${GITHUB_REPOSITORY:-}" ]; then
if ! [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]]; then
echo "Automated backup requires a full controller or GEOINTEL_BUILD_SHA revision." >&2
return 2
fi
source="GEOINTEL_BUILD_SHA"
fi
if command -v git >/dev/null 2>&1 && git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1; then
git_head="$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)"
git_head="${git_head,,}"
if ! [[ "$git_head" =~ ^[0-9a-f]{40}$ ]]; then
echo "Could not resolve a full Git revision from the source checkout." >&2
return 2
fi
if [ -n "$explicit_sha" ] && [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]] && [ "$git_head" != "$explicit_sha" ]; then
echo "Source checkout does not match the supplied release revision." >&2
return 2
fi
if [ -n "$(git -C "$ROOT" status --porcelain=v1 2>/dev/null)" ]; then
git_dirty="true"
fi
if [ -z "$explicit_sha" ]; then
explicit_sha="$git_head"
source="git"
fi
fi
if [ -z "$explicit_sha" ]; then
echo "Cannot bind backup to a source revision; provide GEOINTEL_BUILD_SHA or a controller SHA." >&2
return 2
fi
SOURCE_REVISION="$explicit_sha"
SOURCE_REVISION_SOURCE="${source:-GEOINTEL_BUILD_SHA}"
SOURCE_GIT_DIRTY="$git_dirty"
}
SOURCE_REVISION=""
SOURCE_REVISION_SOURCE=""
SOURCE_GIT_DIRTY="false"
resolve_source_revision
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
echo "Container '$CONTAINER' is not running." >&2 echo "Container '$CONTAINER' is not running." >&2
exit 3 exit 3
@@ -72,6 +162,27 @@ if [ -e "$PARTIAL" ] || [ -e "$FINAL" ]; then
fi fi
mkdir -p "$PARTIAL" mkdir -p "$PARTIAL"
if [ -n "$LINK_DEST_BACKUP" ]; then
LINK_DEST_BACKUP="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve(strict=True))' "$LINK_DEST_BACKUP")"
python3 - "$OUTPUT_ROOT" "$LINK_DEST_BACKUP" <<'PY'
import pathlib
import sys
root = pathlib.Path(sys.argv[1])
candidate = pathlib.Path(sys.argv[2])
try:
candidate.relative_to(root)
except ValueError as exc:
raise SystemExit(f"Link-dest backup must remain below {root}") from exc
if candidate == root or candidate.name.startswith("."):
raise SystemExit("Link-dest backup must identify one completed immutable backup")
PY
(
cd "$LINK_DEST_BACKUP"
sha256sum -c CHECKSUMS.sha256 >/dev/null
)
fi
cleanup_partial() { cleanup_partial() {
if [ -d "$PARTIAL" ]; then if [ -d "$PARTIAL" ]; then
rm -rf -- "$PARTIAL" rm -rf -- "$PARTIAL"
@@ -113,10 +224,21 @@ test -s "$PARTIAL/database.list"
IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")" IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")"
IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")" IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")"
GIT_COMMIT="$(git -C "$ROOT" rev-parse HEAD)" RUNNING_IMAGE_REVISION="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CONTAINER")"
GIT_DIRTY="false" RUNNING_IMAGE_REVISION_IS_FULL_SHA="false"
if [ -n "$(git -C "$ROOT" status --porcelain=v1)" ]; then if [[ "$RUNNING_IMAGE_REVISION" =~ ^[0-9A-Fa-f]{40}$ ]]; then
GIT_DIRTY="true" RUNNING_IMAGE_REVISION="${RUNNING_IMAGE_REVISION,,}"
RUNNING_IMAGE_REVISION_IS_FULL_SHA="true"
elif ! [[ "$RUNNING_IMAGE_REVISION" =~ ^[A-Za-z0-9._-]{1,128}$ ]]; then
echo "Running image has an unsafe or missing OCI revision label." >&2
exit 3
fi
if [ -n "$ROLLBACK_IMAGE_TAG" ]; then
TAGGED_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$ROLLBACK_IMAGE_TAG" 2>/dev/null || true)"
if [ "$TAGGED_IMAGE_ID" != "$IMAGE_ID" ]; then
echo "Backup-specific rollback tag does not resolve to the running image ID." >&2
exit 3
fi
fi fi
docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \ docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \
@@ -132,48 +254,32 @@ for table in projects areas datasets dataset_versions vector_features jobs analy
printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv" printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv"
done done
inventory_path() { snapshot_path() {
local source_path="$1" local source_path="$1"
local output_path="$2" local label="$2"
local manifest_path="$PARTIAL/${label}-manifest.tsv"
local snapshot_path="$PARTIAL/${label}-snapshot"
local link_args=()
if [ -z "$source_path" ]; then if [ -z "$source_path" ]; then
printf 'not_requested\n' > "$output_path" printf 'not_requested\n' > "$manifest_path"
return return
fi fi
python3 - "$source_path" "$output_path" "$INVENTORY_MODE" <<'PY' if [ -n "$LINK_DEST_BACKUP" ]; then
import hashlib link_args=(
import os --link-dest-snapshot "$LINK_DEST_BACKUP/${label}-snapshot"
import pathlib --link-dest-manifest "$LINK_DEST_BACKUP/${label}-manifest.tsv"
import sys )
fi
root = pathlib.Path(sys.argv[1]).expanduser().resolve() python3 "$ROOT/scripts/release_backup_snapshot.py" create \
output = pathlib.Path(sys.argv[2]) --source "$source_path" \
mode = sys.argv[3] --snapshot "$snapshot_path" \
if not root.is_dir(): --manifest "$manifest_path" \
raise SystemExit(f"Inventory root is not a directory: {root}") --label "$label" \
"${link_args[@]}"
def digest(path: pathlib.Path) -> str:
value = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
value.update(chunk)
return value.hexdigest()
with output.open("w", encoding="utf-8", newline="\n") as handle:
handle.write("relative_path\tsize_bytes\tmtime_ns\tsha256\n")
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()):
if path.is_symlink() or not path.is_file():
continue
stat = path.stat()
checksum = digest(path) if mode == "sha256" else ""
relative = path.relative_to(root).as_posix()
if "\t" in relative or "\n" in relative:
raise SystemExit(f"Unsupported inventory path: {relative!r}")
handle.write(f"{relative}\t{stat.st_size}\t{stat.st_mtime_ns}\t{checksum}\n")
PY
} }
inventory_path "$STORAGE_PATH" "$PARTIAL/storage-manifest.tsv" snapshot_path "$STORAGE_PATH" storage
inventory_path "$MODELS_PATH" "$PARTIAL/models-manifest.tsv" snapshot_path "$MODELS_PATH" models
python3 - "$PARTIAL/manifest.json" <<PY python3 - "$PARTIAL/manifest.json" <<PY
import json import json
@@ -190,16 +296,25 @@ payload = {
"database_password_secure": ${PASSWORD_SECURE@Q} == "true", "database_password_secure": ${PASSWORD_SECURE@Q} == "true",
"image_id": ${IMAGE_ID@Q}, "image_id": ${IMAGE_ID@Q},
"image_name": ${IMAGE_NAME@Q}, "image_name": ${IMAGE_NAME@Q},
"git_commit": ${GIT_COMMIT@Q}, "rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None,
"git_dirty": ${GIT_DIRTY@Q} == "true", "backup_tool_revision": ${SOURCE_REVISION@Q},
"backup_tool_revision_source": ${SOURCE_REVISION_SOURCE@Q},
"backup_tool_git_dirty": ${SOURCE_GIT_DIRTY@Q} == "true",
"running_image_revision": ${RUNNING_IMAGE_REVISION@Q},
"running_image_revision_is_full_sha": ${RUNNING_IMAGE_REVISION_IS_FULL_SHA@Q} == "true",
"inventory_mode": ${INVENTORY_MODE@Q}, "inventory_mode": ${INVENTORY_MODE@Q},
"storage_inventory_requested": bool(${STORAGE_PATH@Q}), "storage_inventory_requested": bool(${STORAGE_PATH@Q}),
"models_inventory_requested": bool(${MODELS_PATH@Q}), "models_inventory_requested": bool(${MODELS_PATH@Q}),
"storage_snapshot_requested": bool(${STORAGE_PATH@Q}),
"models_snapshot_requested": bool(${MODELS_PATH@Q}),
"link_dest_backup": ${LINK_DEST_BACKUP@Q} or None,
} }
path = pathlib.Path(__import__("sys").argv[1]) path = pathlib.Path(__import__("sys").argv[1])
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
PY PY
python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$PARTIAL"
( (
cd "$PARTIAL" cd "$PARTIAL"
find . -maxdepth 1 -type f ! -name CHECKSUMS.sha256 -printf '%f\n' \ find . -maxdepth 1 -type f ! -name CHECKSUMS.sha256 -printf '%f\n' \
+105 -7
View File
@@ -1,11 +1,15 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Dry-run-first cleanup for old unreferenced derived/cache artifacts.""" """Dry-run-first quarantine for old unreferenced derived/cache artifacts."""
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import hashlib
import json import json
import os
from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from uuid import uuid4
from audit_data_operations import build_report from audit_data_operations import build_report
from release_backup_guard import require_confirmation, verify_current_backup from release_backup_guard import require_confirmation, verify_current_backup
@@ -14,7 +18,21 @@ from app.core.config import get_settings
from app.db.session import SessionLocal from app.db.session import SessionLocal
CONFIRMATION = "DELETE_STORAGE_ARTIFACTS" CONFIRMATION = "QUARANTINE_STORAGE_ARTIFACTS"
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def write_manifest(path: Path, payload: dict[str, object]) -> None:
temporary = path.with_suffix(".json.partial")
temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
temporary.replace(path)
def parse_args() -> argparse.Namespace: def parse_args() -> argparse.Namespace:
@@ -26,6 +44,11 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--confirm") parser.add_argument("--confirm")
parser.add_argument("--backup-dir", type=Path) parser.add_argument("--backup-dir", type=Path)
parser.add_argument("--backup-max-age-hours", type=float, default=24.0) parser.add_argument("--backup-max-age-hours", type=float, default=24.0)
parser.add_argument(
"--quarantine-root",
type=Path,
help="Protected destination below the storage root (default: operator-evidence/cleanup-quarantine)",
)
return parser.parse_args() return parser.parse_args()
@@ -44,7 +67,8 @@ def main() -> int:
blocked_reason = None blocked_reason = None
backup = None backup = None
deleted: list[str] = [] quarantined: list[dict[str, object]] = []
quarantine_manifest: Path | None = None
if args.apply: if args.apply:
require_confirmation(args.confirm, CONFIRMATION) require_confirmation(args.confirm, CONFIRMATION)
if args.backup_dir is None: if args.backup_dir is None:
@@ -59,9 +83,80 @@ def main() -> int:
"review the dry run and raise the explicit limit" "review the dry run and raise the explicit limit"
) )
else: else:
quarantine_root = (
args.quarantine_root
or storage_root / "operator-evidence" / "cleanup-quarantine"
).resolve()
protected_quarantine_root = (
storage_root / "operator-evidence" / "cleanup-quarantine"
).resolve()
try:
quarantine_root.relative_to(protected_quarantine_root)
except ValueError as exc:
raise RuntimeError(
"--quarantine-root must remain below "
"operator-evidence/cleanup-quarantine in --storage-root"
) from exc
operation_id = f"cleanup-{datetime.now(timezone.utc):%Y%m%dT%H%M%SZ}-{uuid4().hex[:12]}"
operation_root = quarantine_root / operation_id
operation_root.mkdir(parents=True, exist_ok=False)
quarantine_manifest = operation_root / "manifest.json"
entries: list[dict[str, object]] = []
for candidate in candidates: for candidate in candidates:
if candidate.path.is_symlink():
raise RuntimeError(f"Cleanup candidate became a symlink: {candidate.relative_path}")
try:
candidate.path.resolve().relative_to(storage_root)
except ValueError as exc:
raise RuntimeError(
f"Cleanup candidate escaped storage: {candidate.relative_path}"
) from exc
destination = operation_root / "files" / candidate.relative_path
current_size = candidate.path.stat().st_size
if current_size != candidate.size_bytes:
raise RuntimeError(f"Cleanup candidate changed size: {candidate.relative_path}")
entries.append(
{
"relative_path": candidate.relative_path,
"size_bytes": current_size,
"sha256": sha256(candidate.path),
"status": "planned",
"quarantine_relative_path": destination.relative_to(storage_root).as_posix(),
}
)
manifest: dict[str, object] = {
"schema_version": 1,
"operation_id": operation_id,
"created_at": datetime.now(timezone.utc).isoformat(),
"state": "in_progress",
"storage_root": str(storage_root),
"backup_release_id": backup.release_id,
"entries": entries,
}
write_manifest(quarantine_manifest, manifest)
for candidate, entry in zip(candidates, entries, strict=True):
destination = storage_root / str(entry["quarantine_relative_path"])
destination.parent.mkdir(parents=True, exist_ok=True)
try:
os.link(candidate.path, destination, follow_symlinks=False)
except FileExistsError as exc:
raise RuntimeError(f"Quarantine destination already exists: {destination}") from exc
if (
not destination.is_file()
or destination.stat().st_size != entry["size_bytes"]
or sha256(destination) != entry["sha256"]
):
destination.unlink(missing_ok=True)
raise RuntimeError(f"Quarantine link verification failed: {candidate.relative_path}")
entry["status"] = "linked"
write_manifest(quarantine_manifest, manifest)
candidate.path.unlink() candidate.path.unlink()
deleted.append(candidate.relative_path) entry["status"] = "quarantined"
quarantined.append(dict(entry))
write_manifest(quarantine_manifest, manifest)
manifest["state"] = "complete"
manifest["completed_at"] = datetime.now(timezone.utc).isoformat()
write_manifest(quarantine_manifest, manifest)
payload = { payload = {
"schema_version": 1, "schema_version": 1,
@@ -72,8 +167,11 @@ def main() -> int:
"candidate_count": len(candidates), "candidate_count": len(candidates),
"candidate_bytes": sum(item.size_bytes for item in candidates), "candidate_bytes": sum(item.size_bytes for item in candidates),
"candidates": [item.relative_path for item in candidates], "candidates": [item.relative_path for item in candidates],
"deleted_count": len(deleted), "deleted_count": 0,
"deleted": deleted, "deleted": [],
"quarantined_count": len(quarantined),
"quarantined": quarantined,
"quarantine_manifest": str(quarantine_manifest) if quarantine_manifest else None,
"blocked_reason": blocked_reason, "blocked_reason": blocked_reason,
"protected_prefixes": report["cleanup"]["protected_prefixes"], "protected_prefixes": report["cleanup"]["protected_prefixes"],
"backup": ( "backup": (
@@ -81,7 +179,7 @@ def main() -> int:
"release_id": backup.release_id, "release_id": backup.release_id,
"created_at": backup.created_at.isoformat(), "created_at": backup.created_at.isoformat(),
"age_hours": round(backup.age_hours, 3), "age_hours": round(backup.age_hours, 3),
"git_commit": backup.git_commit, "backup_tool_revision": backup.backup_tool_revision,
} }
if backup if backup
else None else None
+12 -5
View File
@@ -9,6 +9,8 @@ from dataclasses import dataclass
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from release_backup_snapshot import verify_backup as verify_byte_snapshots
@dataclass(frozen=True) @dataclass(frozen=True)
class VerifiedBackup: class VerifiedBackup:
@@ -16,7 +18,7 @@ class VerifiedBackup:
release_id: str release_id: str
created_at: datetime created_at: datetime
age_hours: float age_hours: float
git_commit: str backup_tool_revision: str
def _sha256(path: Path) -> str: def _sha256(path: Path) -> str:
@@ -62,6 +64,8 @@ def verify_current_backup(
missing = sorted(name for name in required if not (root / name).is_file()) missing = sorted(name for name in required if not (root / name).is_file())
if missing: if missing:
raise RuntimeError(f"Backup is incomplete; missing: {', '.join(missing)}") raise RuntimeError(f"Backup is incomplete; missing: {', '.join(missing)}")
if not (root / "storage-snapshot").is_dir():
raise RuntimeError("Backup is incomplete; missing: storage-snapshot")
checksum_lines = (root / "CHECKSUMS.sha256").read_text(encoding="utf-8").splitlines() checksum_lines = (root / "CHECKSUMS.sha256").read_text(encoding="utf-8").splitlines()
checked: set[str] = set() checked: set[str] = set()
@@ -93,6 +97,9 @@ def verify_current_backup(
raise RuntimeError("Backup was made from an insecure database configuration") raise RuntimeError("Backup was made from an insecure database configuration")
if manifest.get("inventory_mode") != "sha256" or manifest.get("storage_inventory_requested") is not True: if manifest.get("inventory_mode") != "sha256" or manifest.get("storage_inventory_requested") is not True:
raise RuntimeError("Destructive maintenance requires a SHA-256 storage inventory backup") raise RuntimeError("Destructive maintenance requires a SHA-256 storage inventory backup")
if manifest.get("storage_snapshot_requested") is not True:
raise RuntimeError("Destructive maintenance requires a byte-complete storage snapshot")
verify_byte_snapshots(root)
created = _created_at(manifest.get("created_at")) created = _created_at(manifest.get("created_at"))
current = now or datetime.now(timezone.utc) current = now or datetime.now(timezone.utc)
@@ -107,17 +114,17 @@ def verify_current_backup(
) )
release_id = manifest.get("release_id") release_id = manifest.get("release_id")
git_commit = manifest.get("git_commit") backup_tool_revision = manifest.get("backup_tool_revision", manifest.get("git_commit"))
if not isinstance(release_id, str) or not release_id: if not isinstance(release_id, str) or not release_id:
raise RuntimeError("Backup release id is missing") raise RuntimeError("Backup release id is missing")
if not isinstance(git_commit, str) or len(git_commit) < 7: if not isinstance(backup_tool_revision, str) or len(backup_tool_revision) < 7:
raise RuntimeError("Backup Git commit is missing") raise RuntimeError("Backup tool revision is missing")
return VerifiedBackup( return VerifiedBackup(
backup_dir=root, backup_dir=root,
release_id=release_id, release_id=release_id,
created_at=created, created_at=created,
age_hours=age_hours, age_hours=age_hours,
git_commit=git_commit, backup_tool_revision=backup_tool_revision,
) )
+376
View File
@@ -0,0 +1,376 @@
#!/usr/bin/env python3
"""Create and verify byte-complete, symlink-safe release backup snapshots."""
from __future__ import annotations
import argparse
import errno
import hashlib
import json
import os
import re
import stat
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
MANIFEST_HEADER = "relative_path\tsize_bytes\tmtime_ns\tsha256"
SAFE_LABEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
FICLONE = 0x40049409
FICLONE_FALLBACK_ERRORS = {
errno.EXDEV,
errno.EOPNOTSUPP,
errno.ENOTTY,
errno.EINVAL,
errno.ENOSYS,
}
@dataclass(frozen=True)
class SourceEntry:
path: Path
relative_path: str
stat_result: os.stat_result
is_directory: bool
@dataclass(frozen=True)
class ManifestEntry:
relative_path: str
size_bytes: int
mtime_ns: int
sha256: str
def _safe_relative(value: str) -> str:
if not value or "\t" in value or "\n" in value or "\r" in value:
raise RuntimeError(f"Unsupported snapshot path: {value!r}")
candidate = PurePosixPath(value)
if candidate.is_absolute() or any(part in {"", ".", ".."} for part in candidate.parts):
raise RuntimeError(f"Unsafe snapshot path: {value!r}")
return candidate.as_posix()
def _collect(root: Path) -> list[SourceEntry]:
entries: list[SourceEntry] = []
for current, directory_names, file_names in os.walk(root, topdown=True, followlinks=False):
directory_names.sort()
file_names.sort()
current_path = Path(current)
for name, is_directory in [
*((name, True) for name in directory_names),
*((name, False) for name in file_names),
]:
path = current_path / name
details = path.lstat()
relative = _safe_relative(path.relative_to(root).as_posix())
if stat.S_ISLNK(details.st_mode):
raise RuntimeError(f"Release snapshot refuses symlinked content: {relative}")
if is_directory and not stat.S_ISDIR(details.st_mode):
raise RuntimeError(f"Snapshot directory changed during inventory: {relative}")
if not is_directory and not stat.S_ISREG(details.st_mode):
raise RuntimeError(f"Release snapshot refuses non-regular content: {relative}")
entries.append(SourceEntry(path, relative, details, is_directory))
return entries
def _same_file_state(before: os.stat_result, after: os.stat_result) -> bool:
return (
before.st_dev,
before.st_ino,
before.st_size,
before.st_mtime_ns,
) == (
after.st_dev,
after.st_ino,
after.st_size,
after.st_mtime_ns,
)
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _copy_all(source_descriptor: int, destination_descriptor: int) -> None:
while True:
value = os.read(source_descriptor, 1024 * 1024)
if not value:
return
view = memoryview(value)
while view:
written = os.write(destination_descriptor, view)
if written <= 0:
raise RuntimeError("Snapshot copy stopped before writing all bytes")
view = view[written:]
def _clone_or_copy(entry: SourceEntry, destination: Path) -> ManifestEntry:
source_flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
destination_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0)
source_descriptor = os.open(entry.path, source_flags)
destination_descriptor = -1
try:
opened = os.fstat(source_descriptor)
if not stat.S_ISREG(opened.st_mode) or not _same_file_state(entry.stat_result, opened):
raise RuntimeError(f"Snapshot file changed before copying: {entry.relative_path}")
destination.parent.mkdir(parents=True, exist_ok=True)
destination_descriptor = os.open(destination, destination_flags, stat.S_IMODE(opened.st_mode))
cloned = False
if os.name == "posix":
try:
import fcntl
fcntl.ioctl(destination_descriptor, FICLONE, source_descriptor)
cloned = True
except OSError as exc:
if exc.errno not in FICLONE_FALLBACK_ERRORS:
raise
if not cloned:
os.lseek(source_descriptor, 0, os.SEEK_SET)
os.ftruncate(destination_descriptor, 0)
_copy_all(source_descriptor, destination_descriptor)
os.fsync(destination_descriptor)
after = os.fstat(source_descriptor)
if not _same_file_state(opened, after):
raise RuntimeError(f"Snapshot file changed while copying: {entry.relative_path}")
except BaseException:
if destination_descriptor >= 0:
os.close(destination_descriptor)
destination_descriptor = -1
destination.unlink(missing_ok=True)
raise
finally:
if destination_descriptor >= 0:
os.close(destination_descriptor)
os.close(source_descriptor)
os.chmod(destination, stat.S_IMODE(entry.stat_result.st_mode) & ~0o222, follow_symlinks=False)
retained_times = (entry.stat_result.st_atime_ns, entry.stat_result.st_mtime_ns)
try:
os.utime(destination, ns=retained_times, follow_symlinks=False)
except NotImplementedError:
# Windows does not expose no-follow utime. The destination was created
# exclusively above; recheck it before using the portable call.
if destination.is_symlink():
destination.unlink(missing_ok=True)
raise RuntimeError(f"Snapshot destination became a symlink: {entry.relative_path}")
os.utime(destination, ns=retained_times)
source_checksum = _sha256(entry.path)
snapshot_checksum = _sha256(destination)
final_source = entry.path.lstat()
if not _same_file_state(entry.stat_result, final_source):
raise RuntimeError(f"Snapshot file changed during checksum verification: {entry.relative_path}")
if source_checksum != snapshot_checksum:
raise RuntimeError(f"Snapshot checksum differs from source: {entry.relative_path}")
return ManifestEntry(
relative_path=entry.relative_path,
size_bytes=entry.stat_result.st_size,
mtime_ns=entry.stat_result.st_mtime_ns,
sha256=snapshot_checksum,
)
def _link_verified_prior(
entry: SourceEntry,
destination: Path,
prior_root: Path,
prior_manifest: dict[str, ManifestEntry],
) -> ManifestEntry | None:
retained = prior_manifest.get(entry.relative_path)
if retained is None or retained.size_bytes != entry.stat_result.st_size:
return None
source_checksum = _sha256(entry.path)
final_source = entry.path.lstat()
if not _same_file_state(entry.stat_result, final_source):
raise RuntimeError(f"Snapshot file changed during prior comparison: {entry.relative_path}")
if source_checksum != retained.sha256:
return None
prior_path = prior_root / entry.relative_path
try:
prior_details = prior_path.lstat()
except FileNotFoundError:
return None
if not stat.S_ISREG(prior_details.st_mode) or prior_details.st_size != retained.size_bytes:
raise RuntimeError(f"Prior snapshot file is not reusable: {entry.relative_path}")
if _sha256(prior_path) != retained.sha256:
raise RuntimeError(f"Prior snapshot checksum changed: {entry.relative_path}")
destination.parent.mkdir(parents=True, exist_ok=True)
os.link(prior_path, destination, follow_symlinks=False)
if destination.stat().st_size != retained.size_bytes or _sha256(destination) != retained.sha256:
destination.unlink(missing_ok=True)
raise RuntimeError(f"Hard-linked snapshot verification failed: {entry.relative_path}")
return ManifestEntry(
relative_path=entry.relative_path,
size_bytes=entry.stat_result.st_size,
mtime_ns=entry.stat_result.st_mtime_ns,
sha256=source_checksum,
)
def read_manifest(path: Path) -> dict[str, ManifestEntry]:
lines = path.read_text(encoding="utf-8").splitlines()
if not lines or lines[0] != MANIFEST_HEADER:
raise RuntimeError(f"Snapshot inventory has an invalid header: {path}")
entries: dict[str, ManifestEntry] = {}
for line in lines[1:]:
fields = line.split("\t")
if len(fields) != 4:
raise RuntimeError(f"Snapshot inventory has an invalid row: {line!r}")
relative_path, size_text, mtime_text, checksum = fields
relative_path = _safe_relative(relative_path)
if relative_path in entries:
raise RuntimeError(f"Snapshot inventory repeats a path: {relative_path}")
try:
size_bytes = int(size_text)
mtime_ns = int(mtime_text)
except ValueError as exc:
raise RuntimeError(f"Snapshot inventory has invalid metadata: {relative_path}") from exc
if size_bytes < 0 or mtime_ns < 0 or not re.fullmatch(r"[0-9a-f]{64}", checksum):
raise RuntimeError(f"Snapshot inventory has invalid retained state: {relative_path}")
entries[relative_path] = ManifestEntry(relative_path, size_bytes, mtime_ns, checksum)
return entries
def verify_snapshot(snapshot_path: Path, manifest_path: Path) -> None:
root = snapshot_path.expanduser().resolve(strict=True)
if not root.is_dir():
raise RuntimeError(f"Snapshot path is not a directory: {root}")
expected = read_manifest(manifest_path)
observed_entries = _collect(root)
observed_files = {item.relative_path: item for item in observed_entries if not item.is_directory}
extra = sorted(set(observed_files) - set(expected))
missing = sorted(set(expected) - set(observed_files))
if extra:
raise RuntimeError(f"Snapshot contains unmanifested files: {', '.join(extra[:10])}")
if missing:
raise RuntimeError(f"Snapshot omits manifested files: {', '.join(missing[:10])}")
for relative, retained in expected.items():
current = observed_files[relative]
if current.stat_result.st_size != retained.size_bytes:
raise RuntimeError(f"Snapshot size differs for: {relative}")
if _sha256(current.path) != retained.sha256:
raise RuntimeError(f"Snapshot checksum differs for: {relative}")
def create_snapshot(
source: Path,
snapshot_path: Path,
manifest_path: Path,
*,
label: str,
link_dest_snapshot: Path | None = None,
link_dest_manifest: Path | None = None,
) -> None:
if not SAFE_LABEL.fullmatch(label):
raise RuntimeError(f"Unsafe snapshot label: {label!r}")
root = source.expanduser()
if root.is_symlink():
raise RuntimeError(f"Release snapshot refuses a symlinked root: {root}")
root = root.resolve(strict=True)
if not root.is_dir():
raise RuntimeError(f"Snapshot source is not a directory: {root}")
snapshot_path = snapshot_path.expanduser().resolve()
manifest_path = manifest_path.expanduser().resolve()
for output in (snapshot_path, manifest_path):
try:
output.relative_to(root)
except ValueError:
pass
else:
raise RuntimeError("Release snapshot output must not be inside its source tree")
if snapshot_path.exists():
raise RuntimeError(f"Snapshot destination already exists: {snapshot_path}")
snapshot_path.mkdir(parents=True, exist_ok=False)
manifest_path.parent.mkdir(parents=True, exist_ok=True)
prior_root: Path | None = None
prior_manifest: dict[str, ManifestEntry] = {}
if (link_dest_snapshot is None) != (link_dest_manifest is None):
raise RuntimeError("Prior snapshot and manifest must be supplied together")
if link_dest_snapshot is not None and link_dest_manifest is not None:
prior_root = link_dest_snapshot.expanduser().resolve(strict=True)
prior_manifest = read_manifest(link_dest_manifest.expanduser().resolve(strict=True))
initial = _collect(root)
retained: list[ManifestEntry] = []
for entry in initial:
destination = snapshot_path / entry.relative_path
if entry.is_directory:
destination.mkdir(parents=True, exist_ok=False)
os.chmod(destination, stat.S_IMODE(entry.stat_result.st_mode), follow_symlinks=False)
continue
linked = (
_link_verified_prior(entry, destination, prior_root, prior_manifest)
if prior_root is not None
else None
)
retained.append(linked or _clone_or_copy(entry, destination))
final = _collect(root)
if [(item.relative_path, item.is_directory) for item in initial] != [
(item.relative_path, item.is_directory) for item in final
]:
raise RuntimeError(f"Snapshot source contents changed while backup was running: {root}")
with manifest_path.open("w", encoding="utf-8", newline="\n") as handle:
handle.write(f"{MANIFEST_HEADER}\n")
for entry in retained:
handle.write(
f"{entry.relative_path}\t{entry.size_bytes}\t{entry.mtime_ns}\t{entry.sha256}\n"
)
verify_snapshot(snapshot_path, manifest_path)
def verify_backup(backup_dir: Path) -> None:
root = backup_dir.expanduser().resolve(strict=True)
payload = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
for label in ("storage", "models"):
requested = payload.get(f"{label}_inventory_requested") is True
snapshotted = payload.get(f"{label}_snapshot_requested") is True
if requested != snapshotted:
raise RuntimeError(f"Backup manifest does not bind the {label} inventory to a snapshot")
if requested:
verify_snapshot(root / f"{label}-snapshot", root / f"{label}-manifest.tsv")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="command", required=True)
create = subparsers.add_parser("create")
create.add_argument("--source", type=Path, required=True)
create.add_argument("--snapshot", type=Path, required=True)
create.add_argument("--manifest", type=Path, required=True)
create.add_argument("--label", required=True)
create.add_argument("--link-dest-snapshot", type=Path)
create.add_argument("--link-dest-manifest", type=Path)
verify = subparsers.add_parser("verify")
verify.add_argument("--snapshot", type=Path, required=True)
verify.add_argument("--manifest", type=Path, required=True)
verify_backup_parser = subparsers.add_parser("verify-backup")
verify_backup_parser.add_argument("--backup-dir", type=Path, required=True)
return parser.parse_args()
def main() -> int:
args = parse_args()
if args.command == "create":
create_snapshot(
args.source,
args.snapshot,
args.manifest,
label=args.label,
link_dest_snapshot=args.link_dest_snapshot,
link_dest_manifest=args.link_dest_manifest,
)
elif args.command == "verify":
verify_snapshot(args.snapshot, args.manifest)
else:
verify_backup(args.backup_dir)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+160
View File
@@ -0,0 +1,160 @@
#!/usr/bin/env python3
"""Restore a traceable GeoIntel cleanup quarantine without overwriting data."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from datetime import datetime, timezone
from pathlib import Path
CONFIRMATION = "RESTORE_QUARANTINED_ARTIFACTS"
CLEANUP_PREFIXES = ("exports", "previews", "tiles", "masks", "derived", "rasters/derived")
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def write_manifest(path: Path, payload: dict[str, object]) -> None:
temporary = path.with_suffix(".json.partial")
temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
temporary.replace(path)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--storage-root", type=Path, required=True)
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--confirm", required=True)
return parser.parse_args()
def _within(path: Path, root: Path, *, label: str) -> Path:
resolved = path.resolve()
try:
resolved.relative_to(root)
except ValueError as exc:
raise RuntimeError(f"{label} escapes the storage root") from exc
return resolved
def main() -> int:
args = parse_args()
if args.confirm != CONFIRMATION:
raise RuntimeError(f"Refusing restore; pass --confirm {CONFIRMATION}")
storage_root = args.storage_root.expanduser().resolve()
manifest_path = _within(args.manifest.expanduser(), storage_root, label="Manifest")
protected_quarantine_root = storage_root / "operator-evidence" / "cleanup-quarantine"
try:
manifest_path.relative_to(protected_quarantine_root.resolve())
except ValueError as exc:
raise RuntimeError("Manifest is outside the protected cleanup quarantine") from exc
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
if payload.get("schema_version") != 1:
raise RuntimeError("Unsupported quarantine manifest")
if payload.get("state") not in {"complete", "in_progress", "restore_in_progress", "restored"}:
raise RuntimeError("Quarantine manifest is not in a restorable state")
raw_entries = payload.get("entries")
if not isinstance(raw_entries, list):
raise RuntimeError("Quarantine manifest entries are invalid")
plans: list[tuple[str, dict[str, object], Path, Path]] = []
for raw_entry in raw_entries:
if not isinstance(raw_entry, dict):
raise RuntimeError("Quarantine manifest entry is invalid")
status = raw_entry.get("status")
if status not in {"planned", "linked", "quarantined", "restore_linked", "restored"}:
raise RuntimeError(f"Quarantine manifest entry has an invalid status: {status!r}")
relative_path = raw_entry.get("relative_path")
quarantine_relative_path = raw_entry.get("quarantine_relative_path")
expected_hash = raw_entry.get("sha256")
expected_size = raw_entry.get("size_bytes")
if (
not isinstance(relative_path, str)
or not isinstance(expected_hash, str)
or not isinstance(expected_size, int)
):
raise RuntimeError("Quarantine manifest entry lacks recovery metadata")
if not any(
relative_path == prefix or relative_path.startswith(f"{prefix}/")
for prefix in CLEANUP_PREFIXES
):
raise RuntimeError(f"Original path is outside the cleanup allowlist: {relative_path}")
original = _within(storage_root / relative_path, storage_root, label="Original path")
if not isinstance(quarantine_relative_path, str):
if status != "planned":
raise RuntimeError("Quarantine manifest entry lacks its retained path")
quarantine_relative_path = (
manifest_path.parent / "files" / relative_path
).relative_to(storage_root).as_posix()
raw_entry["quarantine_relative_path"] = quarantine_relative_path
quarantined = _within(
storage_root / quarantine_relative_path,
storage_root,
label="Quarantine path",
)
try:
quarantined.relative_to(manifest_path.parent.resolve())
except ValueError as exc:
raise RuntimeError("Quarantine entry escapes its operation directory") from exc
original_exists = original.exists()
quarantined_exists = quarantined.exists()
if original_exists:
if not original.is_file() or original.stat().st_size != expected_size or sha256(original) != expected_hash:
raise RuntimeError(f"Restore destination already exists with different bytes: {relative_path}")
if quarantined_exists:
if (
not quarantined.is_file()
or quarantined.stat().st_size != expected_size
or sha256(quarantined) != expected_hash
):
raise RuntimeError(f"Quarantined artifact checksum mismatch: {quarantine_relative_path}")
if original_exists and quarantined_exists:
if not os.path.samefile(original, quarantined):
raise RuntimeError(f"Restore destination already exists: {relative_path}")
plans.append(("remove_duplicate_link", raw_entry, quarantined, original))
elif original_exists:
plans.append(("mark_restored", raw_entry, quarantined, original))
elif quarantined_exists:
plans.append(("restore", raw_entry, quarantined, original))
else:
raise RuntimeError(f"Both original and quarantined artifacts are missing: {relative_path}")
payload["state"] = "restore_in_progress"
write_manifest(manifest_path, payload)
for action, entry, quarantined, original in plans:
if action == "restore":
original.parent.mkdir(parents=True, exist_ok=True)
_within(original, storage_root, label="Original path")
try:
os.link(quarantined, original, follow_symlinks=False)
except FileExistsError as exc:
raise RuntimeError(f"Restore destination was created concurrently: {original}") from exc
if not os.path.samefile(quarantined, original):
original.unlink(missing_ok=True)
raise RuntimeError(f"Restore link verification failed: {original}")
entry["status"] = "restore_linked"
write_manifest(manifest_path, payload)
quarantined.unlink()
elif action == "remove_duplicate_link":
quarantined.unlink()
entry["status"] = "restored"
entry["restored_at"] = datetime.now(timezone.utc).isoformat()
write_manifest(manifest_path, payload)
payload["state"] = "restored"
payload["restored_at"] = datetime.now(timezone.utc).isoformat()
write_manifest(manifest_path, payload)
print(json.dumps({"state": "restored", "restored_count": len(plans)}, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+6
View File
@@ -43,6 +43,8 @@ echo "== GeoIntel run readiness check =="
"$PYTHON_BIN" -m py_compile scripts/build_release_package.py "$PYTHON_BIN" -m py_compile scripts/build_release_package.py
"$PYTHON_BIN" -m py_compile scripts/verify_python_lock.py "$PYTHON_BIN" -m py_compile scripts/verify_python_lock.py
"$PYTHON_BIN" scripts/verify_python_lock.py "$PYTHON_BIN" scripts/verify_python_lock.py
"$PYTHON_BIN" -m py_compile scripts/verify_repository_layout.py
"$PYTHON_BIN" scripts/verify_repository_layout.py
"$PYTHON_BIN" -m py_compile scripts/verify_security_exceptions.py "$PYTHON_BIN" -m py_compile scripts/verify_security_exceptions.py
"$PYTHON_BIN" scripts/verify_security_exceptions.py "$PYTHON_BIN" scripts/verify_security_exceptions.py
bash -n scripts/backup_release_state.sh bash -n scripts/backup_release_state.sh
@@ -55,6 +57,7 @@ bash -n scripts/scan_container_image.sh
bash -n scripts/audit_python_dependencies.sh bash -n scripts/audit_python_dependencies.sh
echo "Using Python: ${PYTHON_BIN}" echo "Using Python: ${PYTHON_BIN}"
bash scripts/check_repo_structure.sh bash scripts/check_repo_structure.sh
"$PYTHON_BIN" -m ruff check backend scripts tests
${PYTHON_BIN} scripts/smoke_docs.py ${PYTHON_BIN} scripts/smoke_docs.py
${PYTHON_BIN} scripts/validate_fixtures.py ${PYTHON_BIN} scripts/validate_fixtures.py
${PYTHON_BIN} scripts/smoke_contracts.py ${PYTHON_BIN} scripts/smoke_contracts.py
@@ -129,8 +132,10 @@ ${PYTHON_BIN} -m py_compile scripts/migrate_runtime_model_provenance.py
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
${PYTHON_BIN} -m py_compile scripts/archive_technical_projects.py ${PYTHON_BIN} -m py_compile scripts/archive_technical_projects.py
${PYTHON_BIN} -m py_compile scripts/release_backup_guard.py ${PYTHON_BIN} -m py_compile scripts/release_backup_guard.py
${PYTHON_BIN} -m py_compile scripts/release_backup_snapshot.py
${PYTHON_BIN} -m py_compile scripts/audit_data_operations.py ${PYTHON_BIN} -m py_compile scripts/audit_data_operations.py
${PYTHON_BIN} -m py_compile scripts/cleanup_storage_artifacts.py ${PYTHON_BIN} -m py_compile scripts/cleanup_storage_artifacts.py
${PYTHON_BIN} -m py_compile scripts/restore_storage_quarantine.py
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
${PYTHON_BIN} -m compileall backend/app ${PYTHON_BIN} -m compileall backend/app
(cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning) (cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning)
@@ -149,6 +154,7 @@ bash -n deploy/unraid/gosu-setpriv
bash -n deploy/unraid/run-dockerman-container.sh bash -n deploy/unraid/run-dockerman-container.sh
bash -n deploy/unraid/deploy-release.sh bash -n deploy/unraid/deploy-release.sh
bash -n deploy/unraid/rollback-dockerman-container.sh bash -n deploy/unraid/rollback-dockerman-container.sh
bash -n deploy/unraid/restore-predeploy-database.sh
bash -n scripts/verify_browser_runtime.sh bash -n scripts/verify_browser_runtime.sh
bash -n scripts/verify_demo_export_workflow.sh bash -n scripts/verify_demo_export_workflow.sh
bash -n scripts/verify_demo_raster_workflow.sh bash -n scripts/verify_demo_raster_workflow.sh
+13 -4
View File
@@ -21,8 +21,17 @@ esac
docker image inspect "$TARGET_IMAGE" >/dev/null docker image inspect "$TARGET_IMAGE" >/dev/null
mkdir -p "$ROOT/$(dirname "$OUTPUT")" "$CACHE_DIR" mkdir -p "$ROOT/$(dirname "$OUTPUT")" "$CACHE_DIR"
"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" "$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py"
"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" \ mapfile -t ignored_container_ids < <(
--print-container-ids > "$IGNORE_FILE" "$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" \
--print-container-ids | tr -d '\r'
)
ignore_args=()
trivy_ignore_args=()
if [ "${#ignored_container_ids[@]}" -gt 0 ]; then
printf '%s\n' "${ignored_container_ids[@]}" > "$IGNORE_FILE"
ignore_args=(-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro")
trivy_ignore_args=(--ignorefile "$CONTAINER_IGNORE_FILE")
fi
# Keep the complete report, including vulnerabilities without an available fix. # Keep the complete report, including vulnerabilities without an available fix.
docker run --rm \ docker run --rm \
@@ -45,14 +54,14 @@ docker run --rm \
docker run --rm \ docker run --rm \
-v /var/run/docker.sock:/var/run/docker.sock \ -v /var/run/docker.sock:/var/run/docker.sock \
-v "$CACHE_DIR:/root/.cache/trivy" \ -v "$CACHE_DIR:/root/.cache/trivy" \
-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro" \ "${ignore_args[@]}" \
"$TRIVY_IMAGE" \ "$TRIVY_IMAGE" \
image \ image \
--scanners vuln \ --scanners vuln \
--timeout 20m \ --timeout 20m \
--skip-version-check \ --skip-version-check \
--ignore-unfixed \ --ignore-unfixed \
--ignorefile "$CONTAINER_IGNORE_FILE" \ "${trivy_ignore_args[@]}" \
--skip-files /usr/local/bin/gosu \ --skip-files /usr/local/bin/gosu \
--severity HIGH,CRITICAL \ --severity HIGH,CRITICAL \
--exit-code 1 \ --exit-code 1 \
+9 -1
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
CONTAINER="geointel" CONTAINER="geointel"
BACKUP_DIR="" BACKUP_DIR=""
@@ -59,15 +60,22 @@ required = {
"database_name", "database_name",
"database_user", "database_user",
"image_id", "image_id",
"git_commit",
} }
missing = sorted(required - payload.keys()) missing = sorted(required - payload.keys())
if missing: if missing:
raise SystemExit(f"Backup manifest misses: {', '.join(missing)}") raise SystemExit(f"Backup manifest misses: {', '.join(missing)}")
if payload["schema_version"] != 1 or payload["read_only_source"] is not True: if payload["schema_version"] != 1 or payload["read_only_source"] is not True:
raise SystemExit("Unsupported or unsafe backup manifest") raise SystemExit("Unsupported or unsafe backup manifest")
tool_revision = payload.get("backup_tool_revision", payload.get("git_commit"))
if not isinstance(tool_revision, str) or len(tool_revision) < 7:
raise SystemExit("Backup manifest lacks its backup-tool revision")
running_revision = payload.get("running_image_revision")
if running_revision is not None and not isinstance(running_revision, str):
raise SystemExit("Backup manifest has an invalid running-image revision")
PY PY
python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$BACKUP_DIR"
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
echo "Container '$CONTAINER' is required to run pg_restore --list." >&2 echo "Container '$CONTAINER' is required to run pg_restore --list." >&2
exit 3 exit 3
+27 -18
View File
@@ -17,29 +17,39 @@ EXCEPTIONS_PATH = ROOT / "security" / "pip-audit-exceptions.json"
def load_and_validate() -> tuple[dict[str, object], list[str]]: def load_and_validate() -> tuple[dict[str, object], list[str]]:
payload = json.loads(EXCEPTIONS_PATH.read_text(encoding="utf-8")) payload = json.loads(EXCEPTIONS_PATH.read_text(encoding="utf-8"))
errors: list[str] = [] errors: list[str] = []
try: if set(payload) != {"schema_version", "advisories"}:
review_by = dt.date.fromisoformat(str(payload["review_by"])) errors.append("exception policy must contain only schema_version and advisories")
except (KeyError, ValueError): if payload.get("schema_version") != 1:
errors.append("review_by must be an ISO date") errors.append("schema_version must be 1")
review_by = dt.date.min
if review_by < dt.date.today():
errors.append(f"dependency exception review expired on {review_by.isoformat()}")
if payload.get("package") != "starlette":
errors.append("only the documented Starlette compatibility exception is allowed")
controls = payload.get("compensating_controls")
if not isinstance(controls, list) or len(controls) < 3:
errors.append("at least three compensating controls are required")
advisories = payload.get("advisories") advisories = payload.get("advisories")
if not isinstance(advisories, list) or not advisories: if not isinstance(advisories, list):
errors.append("at least one advisory exception is required") errors.append("advisories must be a list")
else: else:
ids = [str(item.get("id", "")) for item in advisories if isinstance(item, dict)] ids = [str(item.get("id", "")) for item in advisories if isinstance(item, dict)]
if len(ids) != len(set(ids)) or any(not item.startswith("PYSEC-") for item in ids): if len(ids) != len(set(ids)) or any(not item.startswith("PYSEC-") for item in ids):
errors.append("advisory IDs must be unique PYSEC identifiers") errors.append("advisory IDs must be unique PYSEC identifiers")
for item in advisories: for item in advisories:
if not isinstance(item, dict) or len(str(item.get("reason", ""))) < 30: if not isinstance(item, dict):
errors.append("every advisory must be an object")
continue
required = {"id", "package", "review_by", "reason"}
allowed = required | {"aliases"}
if not required.issubset(item) or not set(item).issubset(allowed):
errors.append("every advisory must match the documented exception schema")
if not str(item.get("package", "")).strip():
errors.append("every advisory requires a package")
if len(str(item.get("reason", ""))) < 30:
errors.append("every advisory requires a specific reason") errors.append("every advisory requires a specific reason")
break try:
review_by = dt.date.fromisoformat(str(item["review_by"]))
except (KeyError, ValueError):
errors.append("every advisory review_by must be an ISO date")
else:
if review_by < dt.date.today():
errors.append(
f"dependency exception {item.get('id', '')} expired on "
f"{review_by.isoformat()}"
)
aliases = [ aliases = [
str(alias) str(alias)
for item in advisories for item in advisories
@@ -72,8 +82,7 @@ def main() -> int:
print(alias) print(alias)
else: else:
print( print(
"Dependency exceptions valid through " f"Dependency exception policy valid; {len(payload['advisories'])} active exception(s)."
f"{payload['review_by']} with documented compensating controls."
) )
return 0 return 0
+2 -33
View File
@@ -1,35 +1,4 @@
{ {
"review_by": "2026-08-31", "schema_version": 1,
"package": "starlette", "advisories": []
"constraint": "FastAPI 0.139.2 requires Starlette below 0.53; patched Starlette releases start at 1.0.1 or later.",
"compensating_controls": [
"GeoIntel rejects missing or ambiguous Host headers and request targets before request.url is accessed.",
"Application code logs the raw ASGI path and does not use request.url for authorization or routing.",
"application/x-www-form-urlencoded is rejected; supported uploads use bounded multipart requests behind nginx.",
"The release image is Linux, no HTTPEndpoint route class is used, and the Windows StaticFiles advisory is not applicable."
],
"advisories": [
{
"id": "PYSEC-2026-161",
"reason": "Host/path ambiguity is rejected at the outer request middleware and request.url is not a security boundary."
},
{
"id": "PYSEC-2026-248",
"reason": "Non-slash request paths and ambiguous Host values are rejected before URL reconstruction."
},
{
"id": "PYSEC-2026-249",
"aliases": ["CVE-2026-54283"],
"reason": "GeoIntel rejects application/x-www-form-urlencoded before Starlette form parsing."
},
{
"id": "PYSEC-2026-2280",
"reason": "GeoIntel registers FastAPI APIRouter functions and has no Starlette HTTPEndpoint routes."
},
{
"id": "PYSEC-2026-2281",
"aliases": ["CVE-2026-48818"],
"reason": "The supported all-in-one production runtime is Linux; the advisory affects Windows StaticFiles."
}
]
} }