fix(release): make deployment backup and rollback immutable
This commit is contained in:
@@ -4,6 +4,7 @@ GEOINTEL_API_PREFIX=/api/v1
|
|||||||
DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1
|
DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1
|
||||||
STORAGE_ROOT=./storage
|
STORAGE_ROOT=./storage
|
||||||
MAX_UPLOAD_MB=500
|
MAX_UPLOAD_MB=500
|
||||||
|
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB=64
|
||||||
CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202
|
CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202
|
||||||
|
|
||||||
# Optional single-operator access gate. Store only a PBKDF2-SHA256 hash and
|
# Optional single-operator access gate. Store only a PBKDF2-SHA256 hash and
|
||||||
@@ -14,6 +15,11 @@ GEOINTEL_AUTH_USERNAME=
|
|||||||
GEOINTEL_AUTH_PASSWORD_HASH=
|
GEOINTEL_AUTH_PASSWORD_HASH=
|
||||||
GEOINTEL_AUTH_SESSION_SECRET=
|
GEOINTEL_AUTH_SESSION_SECRET=
|
||||||
GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200
|
GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200
|
||||||
|
GEOINTEL_PUBLIC_BASE_URL=http://localhost:1202
|
||||||
|
GEOINTEL_AUTHENTIK_ISSUER=
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_ID=
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_SECRET=
|
||||||
|
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL=
|
||||||
GEOINTEL_GUEST_ACCESS_ENABLED=true
|
GEOINTEL_GUEST_ACCESS_ENABLED=true
|
||||||
GEOINTEL_GUEST_DISPLAY_NAME=Gast
|
GEOINTEL_GUEST_DISPLAY_NAME=Gast
|
||||||
GEOINTEL_GUEST_SESSION_TTL_SECONDS=7200
|
GEOINTEL_GUEST_SESSION_TTL_SECONDS=7200
|
||||||
|
|||||||
@@ -20,12 +20,22 @@ concurrency:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
full:
|
full:
|
||||||
name: full
|
name: ${{ inputs.profile || 'full' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 30
|
timeout-minutes: 60
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- name: Validate repository with a bounded profile
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||||
|
with:
|
||||||
|
python-version: "3.11"
|
||||||
|
cache: pip
|
||||||
|
cache-dependency-path: backend/requirements-ci.lock
|
||||||
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||||
|
with:
|
||||||
|
node-version: "20"
|
||||||
|
cache: npm
|
||||||
|
cache-dependency-path: frontend/package-lock.json
|
||||||
|
- name: Validate the requested profile against the real projects
|
||||||
shell: bash
|
shell: bash
|
||||||
env:
|
env:
|
||||||
REQUESTED_PROFILE: ${{ inputs.profile }}
|
REQUESTED_PROFILE: ${{ inputs.profile }}
|
||||||
@@ -42,77 +52,34 @@ jobs:
|
|||||||
echo "Unresolved merge markers detected" >&2
|
echo "Unresolved merge markers detected" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
python scripts/verify_repository_layout.py
|
||||||
|
|
||||||
if [[ -f pyproject.toml || -f requirements.txt ]]; then
|
python -m pip install --disable-pip-version-check --require-hashes -r backend/requirements-ci.lock
|
||||||
# Compile only tracked Python sources. Running compileall after a
|
python -m pip install --disable-pip-version-check --no-deps -e backend
|
||||||
# Node install would otherwise traverse node_modules and turn a
|
(cd frontend && npm ci)
|
||||||
# lightweight baseline into a large runner workload.
|
|
||||||
git ls-files -z '*.py' | xargs -0 -r python -m py_compile
|
|
||||||
if [[ -f uv.lock ]]; then
|
|
||||||
python -m venv "${RUNNER_TEMP}/managed-uv"
|
|
||||||
uv_python="${RUNNER_TEMP}/managed-uv/bin/python"
|
|
||||||
"${uv_python}" -m pip install --disable-pip-version-check uv==0.10.0
|
|
||||||
managed_uv="${RUNNER_TEMP}/managed-uv/bin/uv"
|
|
||||||
export UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/managed-project-venv"
|
|
||||||
"${managed_uv}" sync --locked
|
|
||||||
export PATH="${UV_PROJECT_ENVIRONMENT}/bin:${PATH}"
|
|
||||||
if [[ "${profile}" == test || "${profile}" == full ]]; then
|
|
||||||
if "${managed_uv}" run python -c 'import pytest' 2>/dev/null; then
|
|
||||||
"${managed_uv}" run python -m pytest
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
if [[ "${profile}" == lint || "${profile}" == full ]]; then
|
|
||||||
if "${managed_uv}" run python -c 'import ruff' 2>/dev/null; then
|
|
||||||
"${managed_uv}" run python -m ruff check .
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
elif [[ -f requirements.txt ]]; then
|
|
||||||
python -m venv "${RUNNER_TEMP}/managed-python"
|
|
||||||
managed_python="${RUNNER_TEMP}/managed-python/bin/python"
|
|
||||||
"${managed_python}" -m pip install --disable-pip-version-check -r requirements.txt
|
|
||||||
export PATH="${RUNNER_TEMP}/managed-python/bin:${PATH}"
|
|
||||||
if [[ "${profile}" == test || "${profile}" == full ]]; then
|
|
||||||
if "${managed_python}" -c 'import pytest' 2>/dev/null; then
|
|
||||||
"${managed_python}" -m pytest
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Prepare Python before invoking Node scripts. Polyglot repositories
|
case "${profile}" in
|
||||||
# commonly delegate their test script to Python and need the managed
|
test)
|
||||||
# virtual environment to be active first.
|
(cd backend && python -m pytest -W error::DeprecationWarning)
|
||||||
if [[ -f package.json ]]; then
|
(cd frontend && npm run test:unit)
|
||||||
corepack enable
|
;;
|
||||||
if [[ -f pnpm-lock.yaml ]]; then
|
lint)
|
||||||
pnpm install --frozen-lockfile
|
python -m ruff check backend scripts tests
|
||||||
[[ "${profile}" == test || "${profile}" == full ]] && pnpm --if-present test
|
(cd frontend && npm run lint --if-present)
|
||||||
[[ "${profile}" == lint || "${profile}" == full ]] && pnpm --if-present lint
|
;;
|
||||||
[[ "${profile}" == typecheck || "${profile}" == full ]] && pnpm --if-present typecheck
|
typecheck)
|
||||||
[[ "${profile}" == build || "${profile}" == full ]] && pnpm --if-present build
|
(cd frontend && npm run typecheck)
|
||||||
elif [[ -f package-lock.json ]]; then
|
;;
|
||||||
npm ci
|
build)
|
||||||
[[ "${profile}" == test || "${profile}" == full ]] && npm run --if-present test
|
python -m compileall backend/app
|
||||||
[[ "${profile}" == lint || "${profile}" == full ]] && npm run --if-present lint
|
(cd frontend && npm run build)
|
||||||
if [[ "${profile}" == typecheck || "${profile}" == full ]]; then
|
;;
|
||||||
npm run --if-present typecheck
|
security)
|
||||||
fi
|
python -m pip install --disable-pip-version-check pip-audit==2.10.1
|
||||||
[[ "${profile}" == build || "${profile}" == full ]] && npm run --if-present build
|
bash scripts/audit_python_dependencies.sh
|
||||||
fi
|
(cd frontend && npm audit --audit-level=high)
|
||||||
fi
|
;;
|
||||||
|
full)
|
||||||
if [[ -f go.mod ]]; then
|
PYTHON_BIN=python bash scripts/run_readiness_check.sh
|
||||||
if [[ "${profile}" == test || "${profile}" == build || "${profile}" == full ]]; then
|
;;
|
||||||
go test ./...
|
esac
|
||||||
fi
|
|
||||||
fi
|
|
||||||
if [[ -f Cargo.toml ]]; then
|
|
||||||
if [[ "${profile}" == test || "${profile}" == build || "${profile}" == full ]]; then
|
|
||||||
cargo test --locked
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
if compgen -G '*.sln' >/dev/null; then
|
|
||||||
if [[ "${profile}" == test || "${profile}" == build || "${profile}" == full ]]; then
|
|
||||||
dotnet test --configuration Release
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
name: GeoIntel release gates
|
name: GeoIntel release gates
|
||||||
|
|
||||||
on:
|
on:
|
||||||
|
pull_request:
|
||||||
push:
|
push:
|
||||||
branches: [main, develop]
|
branches: [main, develop]
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
@@ -16,21 +17,21 @@ jobs:
|
|||||||
quality:
|
quality:
|
||||||
name: Compile, test, contracts and builds
|
name: Compile, test, contracts and builds
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 45
|
timeout-minutes: 60
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- name: Secret scan
|
- name: Secret scan
|
||||||
run: >-
|
run: >-
|
||||||
docker run --rm
|
docker run --rm
|
||||||
--volume "$PWD:/repo:ro"
|
--volume "$PWD:/repo:ro"
|
||||||
trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3
|
trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3
|
||||||
filesystem /repo --only-verified --no-update
|
filesystem /repo --only-verified --no-update
|
||||||
- uses: actions/setup-python@v5
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
cache: pip
|
cache: pip
|
||||||
cache-dependency-path: backend/requirements-ci.lock
|
cache-dependency-path: backend/requirements-ci.lock
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "20"
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -57,7 +58,7 @@ jobs:
|
|||||||
docker compose config > artifacts/docker-compose.resolved.yml
|
docker compose config > artifacts/docker-compose.resolved.yml
|
||||||
- name: Publish quality evidence
|
- name: Publish quality evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: quality-evidence
|
name: quality-evidence
|
||||||
path: |
|
path: |
|
||||||
@@ -71,13 +72,13 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 20
|
timeout-minutes: 20
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- uses: actions/setup-python@v5
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
cache: pip
|
cache: pip
|
||||||
cache-dependency-path: backend/requirements-ci.lock
|
cache-dependency-path: backend/requirements-ci.lock
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "20"
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -94,7 +95,7 @@ jobs:
|
|||||||
npm audit --audit-level=high --json > ../artifacts/npm-audit.json
|
npm audit --audit-level=high --json > ../artifacts/npm-audit.json
|
||||||
- name: Publish dependency evidence
|
- name: Publish dependency evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: dependency-audits
|
name: dependency-audits
|
||||||
path: |
|
path: |
|
||||||
@@ -105,41 +106,67 @@ jobs:
|
|||||||
retention-days: 30
|
retention-days: 30
|
||||||
|
|
||||||
container:
|
container:
|
||||||
name: GIS image, SBOM and container scan
|
name: Production AI image, SBOM and container scan
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 60
|
timeout-minutes: 120
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- name: Build non-AI release image
|
- name: Build production AI release image
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ gitea.sha }}
|
RELEASE_SHA: ${{ gitea.sha }}
|
||||||
run: |
|
run: |
|
||||||
mkdir -p artifacts
|
mkdir -p artifacts
|
||||||
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
APP_VERSION="$(tr -d '[:space:]' < VERSION)"
|
||||||
docker build \
|
docker build \
|
||||||
-f deploy/unraid/Dockerfile.all-in-one \
|
-f deploy/unraid/Dockerfile.all-in-one \
|
||||||
--build-arg GEOINTEL_INSTALL_AI=false \
|
--build-arg GEOINTEL_INSTALL_AI=true \
|
||||||
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
|
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
|
||||||
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
|
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
|
||||||
-t "geointel-ci:$RELEASE_SHA-gis" \
|
--build-arg GEOINTEL_APP_VERSION="$APP_VERSION" \
|
||||||
|
-t "geointel-ci:$RELEASE_SHA-ai" \
|
||||||
.
|
.
|
||||||
docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json
|
IMAGE_ID="$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")"
|
||||||
|
printf '%s\n' "$IMAGE_ID" > artifacts/image-id.txt
|
||||||
|
docker image inspect "$IMAGE_ID" > artifacts/image-inspect.json
|
||||||
- name: Generate SPDX SBOM
|
- name: Generate SPDX SBOM
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ gitea.sha }}
|
RELEASE_SHA: ${{ gitea.sha }}
|
||||||
run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis"
|
run: |
|
||||||
|
IMAGE_ID="$(cat artifacts/image-id.txt)"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
|
bash scripts/generate_container_sbom.sh "$IMAGE_ID"
|
||||||
- name: Enforce container vulnerability policy
|
- name: Enforce container vulnerability policy
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ gitea.sha }}
|
RELEASE_SHA: ${{ gitea.sha }}
|
||||||
run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis"
|
run: |
|
||||||
|
IMAGE_ID="$(cat artifacts/image-id.txt)"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
|
bash scripts/scan_container_image.sh "$IMAGE_ID"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
- name: Publish container evidence
|
- name: Publish container evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: container-evidence
|
name: container-evidence
|
||||||
path: |
|
path: |
|
||||||
artifacts/image-inspect.json
|
artifacts/image-inspect.json
|
||||||
|
artifacts/image-id.txt
|
||||||
artifacts/geointel-sbom.spdx.json
|
artifacts/geointel-sbom.spdx.json
|
||||||
artifacts/geointel-container-vulnerabilities.json
|
artifacts/geointel-container-vulnerabilities.json
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
retention-days: 30
|
retention-days: 30
|
||||||
|
|
||||||
|
deploy:
|
||||||
|
name: Deploy exact gated revision to Unraid
|
||||||
|
needs: [quality, dependency-audit, container]
|
||||||
|
if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }}
|
||||||
|
runs-on: unraid-deploy
|
||||||
|
timeout-minutes: 180
|
||||||
|
steps:
|
||||||
|
- name: Deploy only after every release gate is green
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
docker exec gitea-deploy-control \
|
||||||
|
/opt/gitea-deploy/deploy.py deploy \
|
||||||
|
"${{ gitea.repository }}" "${{ gitea.sha }}"
|
||||||
|
|||||||
@@ -1,31 +0,0 @@
|
|||||||
name: Unraid autoredeploy
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
paths-ignore:
|
|
||||||
- ".gitea/**"
|
|
||||||
- "docs/**"
|
|
||||||
- "**/*.md"
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: unraid-production-geointel
|
|
||||||
cancel-in-progress: false
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
deploy:
|
|
||||||
name: Deploy geointel
|
|
||||||
runs-on: unraid-deploy
|
|
||||||
timeout-minutes: 180
|
|
||||||
steps:
|
|
||||||
- name: Deploy exact Gitea revision
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
docker exec gitea-deploy-control \
|
|
||||||
/opt/gitea-deploy/deploy.py deploy \
|
|
||||||
"$GITHUB_REPOSITORY" "$GITHUB_SHA"
|
|
||||||
|
|
||||||
@@ -20,19 +20,19 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 45
|
timeout-minutes: 45
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- name: Secret scan
|
- name: Secret scan
|
||||||
run: >-
|
run: >-
|
||||||
docker run --rm
|
docker run --rm
|
||||||
--volume "$PWD:/repo:ro"
|
--volume "$PWD:/repo:ro"
|
||||||
trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3
|
trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3
|
||||||
filesystem /repo --only-verified --no-update
|
filesystem /repo --only-verified --no-update
|
||||||
- uses: actions/setup-python@v5
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
cache: pip
|
cache: pip
|
||||||
cache-dependency-path: backend/requirements-ci.lock
|
cache-dependency-path: backend/requirements-ci.lock
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "20"
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -59,7 +59,7 @@ jobs:
|
|||||||
docker compose config > artifacts/docker-compose.resolved.yml
|
docker compose config > artifacts/docker-compose.resolved.yml
|
||||||
- name: Publish quality evidence
|
- name: Publish quality evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: quality-evidence
|
name: quality-evidence
|
||||||
path: |
|
path: |
|
||||||
@@ -73,13 +73,13 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 20
|
timeout-minutes: 20
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- uses: actions/setup-python@v5
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
cache: pip
|
cache: pip
|
||||||
cache-dependency-path: backend/requirements-ci.lock
|
cache-dependency-path: backend/requirements-ci.lock
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "20"
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -96,7 +96,7 @@ jobs:
|
|||||||
npm audit --audit-level=high --json > ../artifacts/npm-audit.json
|
npm audit --audit-level=high --json > ../artifacts/npm-audit.json
|
||||||
- name: Publish dependency evidence
|
- name: Publish dependency evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: dependency-audits
|
name: dependency-audits
|
||||||
path: |
|
path: |
|
||||||
@@ -107,40 +107,52 @@ jobs:
|
|||||||
retention-days: 30
|
retention-days: 30
|
||||||
|
|
||||||
container:
|
container:
|
||||||
name: GIS image, SBOM and container scan
|
name: Production AI image, SBOM and container scan
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 60
|
timeout-minutes: 120
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- name: Build non-AI release image
|
- name: Build production AI release image
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ github.sha }}
|
RELEASE_SHA: ${{ github.sha }}
|
||||||
run: |
|
run: |
|
||||||
mkdir -p artifacts
|
mkdir -p artifacts
|
||||||
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
APP_VERSION="$(tr -d '[:space:]' < VERSION)"
|
||||||
docker build \
|
docker build \
|
||||||
-f deploy/unraid/Dockerfile.all-in-one \
|
-f deploy/unraid/Dockerfile.all-in-one \
|
||||||
--build-arg GEOINTEL_INSTALL_AI=false \
|
--build-arg GEOINTEL_INSTALL_AI=true \
|
||||||
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
|
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
|
||||||
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
|
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
|
||||||
-t "geointel-ci:$RELEASE_SHA-gis" \
|
--build-arg GEOINTEL_APP_VERSION="$APP_VERSION" \
|
||||||
|
-t "geointel-ci:$RELEASE_SHA-ai" \
|
||||||
.
|
.
|
||||||
docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json
|
IMAGE_ID="$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")"
|
||||||
|
printf '%s\n' "$IMAGE_ID" > artifacts/image-id.txt
|
||||||
|
docker image inspect "$IMAGE_ID" > artifacts/image-inspect.json
|
||||||
- name: Generate SPDX SBOM
|
- name: Generate SPDX SBOM
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ github.sha }}
|
RELEASE_SHA: ${{ github.sha }}
|
||||||
run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis"
|
run: |
|
||||||
|
IMAGE_ID="$(cat artifacts/image-id.txt)"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
|
bash scripts/generate_container_sbom.sh "$IMAGE_ID"
|
||||||
- name: Enforce container vulnerability policy
|
- name: Enforce container vulnerability policy
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ github.sha }}
|
RELEASE_SHA: ${{ github.sha }}
|
||||||
run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis"
|
run: |
|
||||||
|
IMAGE_ID="$(cat artifacts/image-id.txt)"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
|
bash scripts/scan_container_image.sh "$IMAGE_ID"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
- name: Publish container evidence
|
- name: Publish container evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: container-evidence
|
name: container-evidence
|
||||||
path: |
|
path: |
|
||||||
artifacts/image-inspect.json
|
artifacts/image-inspect.json
|
||||||
|
artifacts/image-id.txt
|
||||||
artifacts/geointel-sbom.spdx.json
|
artifacts/geointel-sbom.spdx.json
|
||||||
artifacts/geointel-container-vulnerabilities.json
|
artifacts/geointel-container-vulnerabilities.json
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
|
|||||||
@@ -8,3 +8,5 @@ storage
|
|||||||
dist
|
dist
|
||||||
node_modules
|
node_modules
|
||||||
.env
|
.env
|
||||||
|
.env.*
|
||||||
|
!.env.example
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ readme = "README.md"
|
|||||||
requires-python = ">=3.11"
|
requires-python = ">=3.11"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"fastapi>=0.112.0",
|
"fastapi>=0.112.0",
|
||||||
"starlette>=0.46.0,<1.0.0",
|
"starlette>=1.3.1,<2.0.0",
|
||||||
"uvicorn[standard]>=0.30.6",
|
"uvicorn[standard]>=0.30.6",
|
||||||
"SQLAlchemy>=2.0.34",
|
"SQLAlchemy>=2.0.34",
|
||||||
"psycopg[binary]>=3.2.1",
|
"psycopg[binary]>=3.2.1",
|
||||||
@@ -16,6 +16,8 @@ dependencies = [
|
|||||||
"shapely>=2.0.4",
|
"shapely>=2.0.4",
|
||||||
"pyproj>=3.6.1",
|
"pyproj>=3.6.1",
|
||||||
"python-multipart>=0.0.9",
|
"python-multipart>=0.0.9",
|
||||||
|
"itsdangerous>=2.2.0",
|
||||||
|
"PyJWT[crypto]>=2.10.1",
|
||||||
"rdflib>=7.1,<8",
|
"rdflib>=7.1,<8",
|
||||||
"alembic>=1.13.2",
|
"alembic>=1.13.2",
|
||||||
]
|
]
|
||||||
@@ -37,7 +39,7 @@ ai = [
|
|||||||
"ultralytics>=8.3,<9",
|
"ultralytics>=8.3,<9",
|
||||||
"torch>=2.4",
|
"torch>=2.4",
|
||||||
]
|
]
|
||||||
dev = ["pytest>=8.3.2", "httpx>=0.27.0", "ruff>=0.6.9"]
|
dev = ["pytest>=8.3.2", "httpx>=0.27.0", "httpx2>=2.0.0", "ruff>=0.6.9"]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
geointel-backend = "app.main:main"
|
geointel-backend = "app.main:main"
|
||||||
|
|||||||
@@ -2,9 +2,9 @@
|
|||||||
# This file is autogenerated by pip-compile with Python 3.11
|
# This file is autogenerated by pip-compile with Python 3.11
|
||||||
# by the following command:
|
# by the following command:
|
||||||
#
|
#
|
||||||
# pip-compile --extra=dev --extra=gis --generate-hashes --output-file=requirements-ci.lock --strip-extras pyproject.toml
|
# pip-compile --extra=dev --extra=gis --generate-hashes --no-index --output-file=requirements-ci.lock --strip-extras pyproject.toml
|
||||||
#
|
#
|
||||||
# geointel-input-sha256: 03c20efedd96474cbe62591b7b70cdad2681688b618bdd76731bd4cfaf85b3d4
|
# geointel-input-sha256: e1dd11f5b30f4c8c902f33476282da48da386d36093cab68212edc69e75df8e3
|
||||||
affine==2.4.0 \
|
affine==2.4.0 \
|
||||||
--hash=sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92 \
|
--hash=sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92 \
|
||||||
--hash=sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea
|
--hash=sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea
|
||||||
@@ -26,6 +26,7 @@ anyio==4.14.2 \
|
|||||||
--hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f
|
--hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f
|
||||||
# via
|
# via
|
||||||
# httpx
|
# httpx
|
||||||
|
# httpx2
|
||||||
# starlette
|
# starlette
|
||||||
# watchfiles
|
# watchfiles
|
||||||
attrs==26.1.0 \
|
attrs==26.1.0 \
|
||||||
@@ -41,6 +42,108 @@ certifi==2026.6.17 \
|
|||||||
# pyogrio
|
# pyogrio
|
||||||
# pyproj
|
# pyproj
|
||||||
# rasterio
|
# rasterio
|
||||||
|
cffi==2.1.1 \
|
||||||
|
--hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \
|
||||||
|
--hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \
|
||||||
|
--hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \
|
||||||
|
--hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \
|
||||||
|
--hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \
|
||||||
|
--hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \
|
||||||
|
--hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \
|
||||||
|
--hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \
|
||||||
|
--hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \
|
||||||
|
--hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \
|
||||||
|
--hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \
|
||||||
|
--hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \
|
||||||
|
--hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \
|
||||||
|
--hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \
|
||||||
|
--hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \
|
||||||
|
--hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \
|
||||||
|
--hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \
|
||||||
|
--hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \
|
||||||
|
--hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \
|
||||||
|
--hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \
|
||||||
|
--hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \
|
||||||
|
--hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \
|
||||||
|
--hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \
|
||||||
|
--hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \
|
||||||
|
--hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \
|
||||||
|
--hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \
|
||||||
|
--hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \
|
||||||
|
--hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \
|
||||||
|
--hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \
|
||||||
|
--hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \
|
||||||
|
--hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \
|
||||||
|
--hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \
|
||||||
|
--hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \
|
||||||
|
--hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \
|
||||||
|
--hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \
|
||||||
|
--hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \
|
||||||
|
--hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \
|
||||||
|
--hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \
|
||||||
|
--hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \
|
||||||
|
--hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \
|
||||||
|
--hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \
|
||||||
|
--hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \
|
||||||
|
--hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \
|
||||||
|
--hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \
|
||||||
|
--hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \
|
||||||
|
--hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \
|
||||||
|
--hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \
|
||||||
|
--hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \
|
||||||
|
--hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \
|
||||||
|
--hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \
|
||||||
|
--hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \
|
||||||
|
--hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \
|
||||||
|
--hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \
|
||||||
|
--hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \
|
||||||
|
--hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \
|
||||||
|
--hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \
|
||||||
|
--hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \
|
||||||
|
--hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \
|
||||||
|
--hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \
|
||||||
|
--hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \
|
||||||
|
--hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \
|
||||||
|
--hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \
|
||||||
|
--hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \
|
||||||
|
--hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \
|
||||||
|
--hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \
|
||||||
|
--hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \
|
||||||
|
--hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \
|
||||||
|
--hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \
|
||||||
|
--hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \
|
||||||
|
--hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \
|
||||||
|
--hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \
|
||||||
|
--hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \
|
||||||
|
--hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \
|
||||||
|
--hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \
|
||||||
|
--hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \
|
||||||
|
--hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \
|
||||||
|
--hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \
|
||||||
|
--hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \
|
||||||
|
--hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \
|
||||||
|
--hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \
|
||||||
|
--hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \
|
||||||
|
--hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \
|
||||||
|
--hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \
|
||||||
|
--hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \
|
||||||
|
--hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \
|
||||||
|
--hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \
|
||||||
|
--hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \
|
||||||
|
--hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \
|
||||||
|
--hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \
|
||||||
|
--hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \
|
||||||
|
--hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \
|
||||||
|
--hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \
|
||||||
|
--hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \
|
||||||
|
--hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \
|
||||||
|
--hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \
|
||||||
|
--hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \
|
||||||
|
--hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \
|
||||||
|
--hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \
|
||||||
|
--hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \
|
||||||
|
--hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264
|
||||||
|
# via cryptography
|
||||||
click==8.4.2 \
|
click==8.4.2 \
|
||||||
--hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \
|
--hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \
|
||||||
--hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76
|
--hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76
|
||||||
@@ -57,6 +160,54 @@ cligj==0.7.2 \
|
|||||||
--hash=sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27 \
|
--hash=sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27 \
|
||||||
--hash=sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df
|
--hash=sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df
|
||||||
# via rasterio
|
# via rasterio
|
||||||
|
cryptography==50.0.1 \
|
||||||
|
--hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \
|
||||||
|
--hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \
|
||||||
|
--hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \
|
||||||
|
--hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \
|
||||||
|
--hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \
|
||||||
|
--hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \
|
||||||
|
--hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \
|
||||||
|
--hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \
|
||||||
|
--hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \
|
||||||
|
--hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \
|
||||||
|
--hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \
|
||||||
|
--hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \
|
||||||
|
--hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \
|
||||||
|
--hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \
|
||||||
|
--hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \
|
||||||
|
--hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \
|
||||||
|
--hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \
|
||||||
|
--hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \
|
||||||
|
--hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \
|
||||||
|
--hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \
|
||||||
|
--hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \
|
||||||
|
--hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \
|
||||||
|
--hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \
|
||||||
|
--hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \
|
||||||
|
--hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \
|
||||||
|
--hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \
|
||||||
|
--hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \
|
||||||
|
--hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \
|
||||||
|
--hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \
|
||||||
|
--hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \
|
||||||
|
--hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \
|
||||||
|
--hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \
|
||||||
|
--hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \
|
||||||
|
--hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \
|
||||||
|
--hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \
|
||||||
|
--hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \
|
||||||
|
--hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \
|
||||||
|
--hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \
|
||||||
|
--hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \
|
||||||
|
--hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \
|
||||||
|
--hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \
|
||||||
|
--hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \
|
||||||
|
--hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \
|
||||||
|
--hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \
|
||||||
|
--hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \
|
||||||
|
--hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef
|
||||||
|
# via pyjwt
|
||||||
fastapi==0.139.2 \
|
fastapi==0.139.2 \
|
||||||
--hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \
|
--hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \
|
||||||
--hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c
|
--hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c
|
||||||
@@ -155,11 +306,16 @@ h11==0.16.0 \
|
|||||||
--hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
|
--hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
|
||||||
# via
|
# via
|
||||||
# httpcore
|
# httpcore
|
||||||
|
# httpcore2
|
||||||
# uvicorn
|
# uvicorn
|
||||||
httpcore==1.0.9 \
|
httpcore==1.0.9 \
|
||||||
--hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
|
--hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
|
||||||
--hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
|
--hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
|
||||||
# via httpx
|
# via httpx
|
||||||
|
httpcore2==2.12.0 \
|
||||||
|
--hash=sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb \
|
||||||
|
--hash=sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648
|
||||||
|
# via httpx2
|
||||||
httptools==0.8.0 \
|
httptools==0.8.0 \
|
||||||
--hash=sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683 \
|
--hash=sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683 \
|
||||||
--hash=sha256:0ea897f0c729581ebf72131a438a7932d9b14efef72d75ada966700cac3caaeb \
|
--hash=sha256:0ea897f0c729581ebf72131a438a7932d9b14efef72d75ada966700cac3caaeb \
|
||||||
@@ -216,16 +372,25 @@ httpx==0.28.1 \
|
|||||||
--hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
|
--hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
|
||||||
--hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
|
--hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
|
||||||
# via geointel-backend (pyproject.toml)
|
# via geointel-backend (pyproject.toml)
|
||||||
|
httpx2==2.12.0 \
|
||||||
|
--hash=sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf \
|
||||||
|
--hash=sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36
|
||||||
|
# via geointel-backend (pyproject.toml)
|
||||||
idna==3.18 \
|
idna==3.18 \
|
||||||
--hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \
|
--hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \
|
||||||
--hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848
|
--hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848
|
||||||
# via
|
# via
|
||||||
# anyio
|
# anyio
|
||||||
# httpx
|
# httpx
|
||||||
|
# httpx2
|
||||||
iniconfig==2.3.0 \
|
iniconfig==2.3.0 \
|
||||||
--hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \
|
--hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \
|
||||||
--hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12
|
--hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12
|
||||||
# via pytest
|
# via pytest
|
||||||
|
itsdangerous==2.2.0 \
|
||||||
|
--hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \
|
||||||
|
--hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173
|
||||||
|
# via geointel-backend (pyproject.toml)
|
||||||
mako==1.3.12 \
|
mako==1.3.12 \
|
||||||
--hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \
|
--hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \
|
||||||
--hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a
|
--hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a
|
||||||
@@ -613,6 +778,10 @@ psycopg-binary==3.3.4 \
|
|||||||
--hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \
|
--hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \
|
||||||
--hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7
|
--hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7
|
||||||
# via psycopg
|
# via psycopg
|
||||||
|
pycparser==3.0 \
|
||||||
|
--hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \
|
||||||
|
--hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992
|
||||||
|
# via cffi
|
||||||
pydantic==2.13.4 \
|
pydantic==2.13.4 \
|
||||||
--hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
|
--hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
|
||||||
--hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
|
--hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
|
||||||
@@ -750,6 +919,12 @@ pygments==2.20.0 \
|
|||||||
--hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \
|
--hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \
|
||||||
--hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176
|
--hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176
|
||||||
# via pytest
|
# via pytest
|
||||||
|
pyjwt==2.13.0 \
|
||||||
|
--hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \
|
||||||
|
--hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728
|
||||||
|
# via
|
||||||
|
# geointel-backend (pyproject.toml)
|
||||||
|
# pyjwt
|
||||||
pyogrio==0.13.0 \
|
pyogrio==0.13.0 \
|
||||||
--hash=sha256:1b91f6d6e6757a6ea84b9459d24f479dcb52bbf4ebcdb16baf39e49d2836a1cf \
|
--hash=sha256:1b91f6d6e6757a6ea84b9459d24f479dcb52bbf4ebcdb16baf39e49d2836a1cf \
|
||||||
--hash=sha256:220a988ce2a26591d6db5c775b07289d4f54cabdf274cc048f0e17a0b9d5be14 \
|
--hash=sha256:220a988ce2a26591d6db5c775b07289d4f54cabdf274cc048f0e17a0b9d5be14 \
|
||||||
@@ -1127,12 +1302,18 @@ sqlalchemy==2.0.51 \
|
|||||||
# alembic
|
# alembic
|
||||||
# geoalchemy2
|
# geoalchemy2
|
||||||
# geointel-backend (pyproject.toml)
|
# geointel-backend (pyproject.toml)
|
||||||
starlette==0.52.1 \
|
starlette==1.6.0 \
|
||||||
--hash=sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74 \
|
--hash=sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c \
|
||||||
--hash=sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933
|
--hash=sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b
|
||||||
# via
|
# via
|
||||||
# fastapi
|
# fastapi
|
||||||
# geointel-backend (pyproject.toml)
|
# geointel-backend (pyproject.toml)
|
||||||
|
truststore==0.10.4 \
|
||||||
|
--hash=sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301 \
|
||||||
|
--hash=sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981
|
||||||
|
# via
|
||||||
|
# httpcore2
|
||||||
|
# httpx2
|
||||||
typing-extensions==4.16.0 \
|
typing-extensions==4.16.0 \
|
||||||
--hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
|
--hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
|
||||||
--hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
|
--hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
|
||||||
@@ -1140,6 +1321,7 @@ typing-extensions==4.16.0 \
|
|||||||
# alembic
|
# alembic
|
||||||
# anyio
|
# anyio
|
||||||
# fastapi
|
# fastapi
|
||||||
|
# httpx2
|
||||||
# psycopg
|
# psycopg
|
||||||
# pydantic
|
# pydantic
|
||||||
# pydantic-core
|
# pydantic-core
|
||||||
|
|||||||
@@ -2,9 +2,9 @@
|
|||||||
# This file is autogenerated by pip-compile with Python 3.11
|
# This file is autogenerated by pip-compile with Python 3.11
|
||||||
# by the following command:
|
# by the following command:
|
||||||
#
|
#
|
||||||
# pip-compile --extra=gis --generate-hashes --output-file=requirements-runtime.lock --strip-extras pyproject.toml
|
# pip-compile --extra=gis --generate-hashes --no-index --output-file=requirements-runtime.lock --strip-extras pyproject.toml
|
||||||
#
|
#
|
||||||
# geointel-input-sha256: 0d0d2cdceb01da58354610f03b5ce244523130cf9fd29e3f8988dbe684838e8b
|
# geointel-input-sha256: 30318170074aad9b91f570cccc574eaaad0eded5814a464d1207a94d2e1d3317
|
||||||
affine==2.4.0 \
|
affine==2.4.0 \
|
||||||
--hash=sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92 \
|
--hash=sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92 \
|
||||||
--hash=sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea
|
--hash=sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea
|
||||||
@@ -38,6 +38,108 @@ certifi==2026.6.17 \
|
|||||||
# pyogrio
|
# pyogrio
|
||||||
# pyproj
|
# pyproj
|
||||||
# rasterio
|
# rasterio
|
||||||
|
cffi==2.1.1 \
|
||||||
|
--hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \
|
||||||
|
--hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \
|
||||||
|
--hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \
|
||||||
|
--hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \
|
||||||
|
--hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \
|
||||||
|
--hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \
|
||||||
|
--hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \
|
||||||
|
--hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \
|
||||||
|
--hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \
|
||||||
|
--hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \
|
||||||
|
--hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \
|
||||||
|
--hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \
|
||||||
|
--hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \
|
||||||
|
--hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \
|
||||||
|
--hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \
|
||||||
|
--hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \
|
||||||
|
--hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \
|
||||||
|
--hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \
|
||||||
|
--hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \
|
||||||
|
--hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \
|
||||||
|
--hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \
|
||||||
|
--hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \
|
||||||
|
--hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \
|
||||||
|
--hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \
|
||||||
|
--hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \
|
||||||
|
--hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \
|
||||||
|
--hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \
|
||||||
|
--hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \
|
||||||
|
--hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \
|
||||||
|
--hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \
|
||||||
|
--hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \
|
||||||
|
--hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \
|
||||||
|
--hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \
|
||||||
|
--hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \
|
||||||
|
--hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \
|
||||||
|
--hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \
|
||||||
|
--hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \
|
||||||
|
--hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \
|
||||||
|
--hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \
|
||||||
|
--hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \
|
||||||
|
--hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \
|
||||||
|
--hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \
|
||||||
|
--hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \
|
||||||
|
--hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \
|
||||||
|
--hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \
|
||||||
|
--hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \
|
||||||
|
--hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \
|
||||||
|
--hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \
|
||||||
|
--hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \
|
||||||
|
--hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \
|
||||||
|
--hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \
|
||||||
|
--hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \
|
||||||
|
--hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \
|
||||||
|
--hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \
|
||||||
|
--hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \
|
||||||
|
--hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \
|
||||||
|
--hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \
|
||||||
|
--hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \
|
||||||
|
--hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \
|
||||||
|
--hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \
|
||||||
|
--hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \
|
||||||
|
--hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \
|
||||||
|
--hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \
|
||||||
|
--hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \
|
||||||
|
--hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \
|
||||||
|
--hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \
|
||||||
|
--hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \
|
||||||
|
--hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \
|
||||||
|
--hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \
|
||||||
|
--hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \
|
||||||
|
--hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \
|
||||||
|
--hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \
|
||||||
|
--hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \
|
||||||
|
--hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \
|
||||||
|
--hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \
|
||||||
|
--hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \
|
||||||
|
--hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \
|
||||||
|
--hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \
|
||||||
|
--hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \
|
||||||
|
--hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \
|
||||||
|
--hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \
|
||||||
|
--hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \
|
||||||
|
--hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \
|
||||||
|
--hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \
|
||||||
|
--hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \
|
||||||
|
--hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \
|
||||||
|
--hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \
|
||||||
|
--hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \
|
||||||
|
--hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \
|
||||||
|
--hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \
|
||||||
|
--hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \
|
||||||
|
--hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \
|
||||||
|
--hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \
|
||||||
|
--hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \
|
||||||
|
--hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \
|
||||||
|
--hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \
|
||||||
|
--hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \
|
||||||
|
--hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \
|
||||||
|
--hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \
|
||||||
|
--hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264
|
||||||
|
# via cryptography
|
||||||
click==8.4.2 \
|
click==8.4.2 \
|
||||||
--hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \
|
--hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \
|
||||||
--hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76
|
--hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76
|
||||||
@@ -54,6 +156,54 @@ cligj==0.7.2 \
|
|||||||
--hash=sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27 \
|
--hash=sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27 \
|
||||||
--hash=sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df
|
--hash=sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df
|
||||||
# via rasterio
|
# via rasterio
|
||||||
|
cryptography==50.0.1 \
|
||||||
|
--hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \
|
||||||
|
--hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \
|
||||||
|
--hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \
|
||||||
|
--hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \
|
||||||
|
--hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \
|
||||||
|
--hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \
|
||||||
|
--hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \
|
||||||
|
--hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \
|
||||||
|
--hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \
|
||||||
|
--hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \
|
||||||
|
--hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \
|
||||||
|
--hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \
|
||||||
|
--hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \
|
||||||
|
--hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \
|
||||||
|
--hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \
|
||||||
|
--hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \
|
||||||
|
--hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \
|
||||||
|
--hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \
|
||||||
|
--hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \
|
||||||
|
--hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \
|
||||||
|
--hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \
|
||||||
|
--hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \
|
||||||
|
--hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \
|
||||||
|
--hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \
|
||||||
|
--hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \
|
||||||
|
--hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \
|
||||||
|
--hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \
|
||||||
|
--hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \
|
||||||
|
--hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \
|
||||||
|
--hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \
|
||||||
|
--hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \
|
||||||
|
--hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \
|
||||||
|
--hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \
|
||||||
|
--hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \
|
||||||
|
--hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \
|
||||||
|
--hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \
|
||||||
|
--hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \
|
||||||
|
--hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \
|
||||||
|
--hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \
|
||||||
|
--hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \
|
||||||
|
--hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \
|
||||||
|
--hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \
|
||||||
|
--hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \
|
||||||
|
--hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \
|
||||||
|
--hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \
|
||||||
|
--hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef
|
||||||
|
# via pyjwt
|
||||||
fastapi==0.139.2 \
|
fastapi==0.139.2 \
|
||||||
--hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \
|
--hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \
|
||||||
--hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c
|
--hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c
|
||||||
@@ -207,6 +357,10 @@ idna==3.18 \
|
|||||||
--hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \
|
--hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \
|
||||||
--hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848
|
--hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848
|
||||||
# via anyio
|
# via anyio
|
||||||
|
itsdangerous==2.2.0 \
|
||||||
|
--hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \
|
||||||
|
--hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173
|
||||||
|
# via geointel-backend (pyproject.toml)
|
||||||
mako==1.3.12 \
|
mako==1.3.12 \
|
||||||
--hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \
|
--hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \
|
||||||
--hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a
|
--hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a
|
||||||
@@ -589,6 +743,10 @@ psycopg-binary==3.3.4 \
|
|||||||
--hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \
|
--hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \
|
||||||
--hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7
|
--hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7
|
||||||
# via psycopg
|
# via psycopg
|
||||||
|
pycparser==3.0 \
|
||||||
|
--hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \
|
||||||
|
--hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992
|
||||||
|
# via cffi
|
||||||
pydantic==2.13.4 \
|
pydantic==2.13.4 \
|
||||||
--hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
|
--hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
|
||||||
--hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
|
--hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
|
||||||
@@ -722,6 +880,12 @@ pydantic-settings==2.14.2 \
|
|||||||
--hash=sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440 \
|
--hash=sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440 \
|
||||||
--hash=sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f
|
--hash=sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f
|
||||||
# via geointel-backend (pyproject.toml)
|
# via geointel-backend (pyproject.toml)
|
||||||
|
pyjwt==2.13.0 \
|
||||||
|
--hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \
|
||||||
|
--hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728
|
||||||
|
# via
|
||||||
|
# geointel-backend (pyproject.toml)
|
||||||
|
# pyjwt
|
||||||
pyogrio==0.13.0 \
|
pyogrio==0.13.0 \
|
||||||
--hash=sha256:1b91f6d6e6757a6ea84b9459d24f479dcb52bbf4ebcdb16baf39e49d2836a1cf \
|
--hash=sha256:1b91f6d6e6757a6ea84b9459d24f479dcb52bbf4ebcdb16baf39e49d2836a1cf \
|
||||||
--hash=sha256:220a988ce2a26591d6db5c775b07289d4f54cabdf274cc048f0e17a0b9d5be14 \
|
--hash=sha256:220a988ce2a26591d6db5c775b07289d4f54cabdf274cc048f0e17a0b9d5be14 \
|
||||||
@@ -1075,9 +1239,9 @@ sqlalchemy==2.0.51 \
|
|||||||
# alembic
|
# alembic
|
||||||
# geoalchemy2
|
# geoalchemy2
|
||||||
# geointel-backend (pyproject.toml)
|
# geointel-backend (pyproject.toml)
|
||||||
starlette==0.52.1 \
|
starlette==1.6.0 \
|
||||||
--hash=sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74 \
|
--hash=sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c \
|
||||||
--hash=sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933
|
--hash=sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b
|
||||||
# via
|
# via
|
||||||
# fastapi
|
# fastapi
|
||||||
# geointel-backend (pyproject.toml)
|
# geointel-backend (pyproject.toml)
|
||||||
|
|||||||
@@ -172,6 +172,25 @@ def test_walloon_runtime_settings_are_editable_in_compose_and_unraid() -> None:
|
|||||||
assert "WALOUS_MAX_PIXELS" in content
|
assert "WALOUS_MAX_PIXELS" in content
|
||||||
|
|
||||||
|
|
||||||
|
def test_in_memory_vector_limit_is_propagated_and_validated_in_every_runtime() -> None:
|
||||||
|
expected = "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB"
|
||||||
|
for path in (
|
||||||
|
ROOT / ".env.example",
|
||||||
|
ROOT / "docker-compose.yml",
|
||||||
|
ROOT / "docker-compose.unraid.yml",
|
||||||
|
ROOT / "deploy" / "unraid" / "geointel.env.example",
|
||||||
|
ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml",
|
||||||
|
):
|
||||||
|
assert expected in path.read_text(encoding="utf-8"), path
|
||||||
|
|
||||||
|
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
assert 'GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}"' in run_script
|
||||||
|
assert "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be between 1 and 256." in run_script
|
||||||
|
assert '-e GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB"' in run_script
|
||||||
|
|
||||||
|
|
||||||
def test_frontend_uses_same_origin_api_proxy_by_default() -> None:
|
def test_frontend_uses_same_origin_api_proxy_by_default() -> None:
|
||||||
api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
|
api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
|
||||||
nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
|
nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
|
||||||
@@ -208,6 +227,45 @@ def test_nginx_runtime_allows_long_ai_and_qa_requests() -> None:
|
|||||||
assert "proxy_send_timeout 600s;" in config
|
assert "proxy_send_timeout 600s;" in config
|
||||||
|
|
||||||
|
|
||||||
|
def test_nginx_preserves_outer_https_scheme_for_secure_session_cookies() -> None:
|
||||||
|
configs = (
|
||||||
|
(ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"),
|
||||||
|
(ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8"),
|
||||||
|
)
|
||||||
|
for config in configs:
|
||||||
|
assert "geo $geointel_trusted_forwarder" in config
|
||||||
|
assert "default 0;" in config
|
||||||
|
assert "172.16.0.0/12 1;" in config
|
||||||
|
assert 'map "$geointel_trusted_forwarder:$http_x_forwarded_proto"' in config
|
||||||
|
assert '"1:https" https;' in config
|
||||||
|
assert "proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;" in config
|
||||||
|
assert "proxy_set_header X-Forwarded-Proto $scheme;" not in config
|
||||||
|
|
||||||
|
|
||||||
|
def test_nginx_runtime_sets_security_headers_on_all_cached_locations() -> None:
|
||||||
|
configs = (
|
||||||
|
(ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"),
|
||||||
|
(ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
required = (
|
||||||
|
'Content-Security-Policy "frame-ancestors \'none\'" always;',
|
||||||
|
'X-Frame-Options "DENY" always;',
|
||||||
|
'X-Content-Type-Options "nosniff" always;',
|
||||||
|
'Referrer-Policy "strict-origin-when-cross-origin" always;',
|
||||||
|
'Permissions-Policy "camera=(), microphone=(), geolocation=()" always;',
|
||||||
|
)
|
||||||
|
|
||||||
|
for config in configs:
|
||||||
|
cached_locations = config.count("add_header Cache-Control")
|
||||||
|
assert cached_locations >= 2
|
||||||
|
for header in required:
|
||||||
|
# Nginx 1.27 locations with Cache-Control do not inherit server-level
|
||||||
|
# add_header directives, so every cached location repeats the policy.
|
||||||
|
assert config.count(f"add_header {header}") == cached_locations + 1
|
||||||
|
|
||||||
|
|
||||||
def test_compose_does_not_publish_postgis_on_default_host_port() -> None:
|
def test_compose_does_not_publish_postgis_on_default_host_port() -> None:
|
||||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
|||||||
import hashlib
|
import hashlib
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -18,6 +20,8 @@ SCRIPTS = ROOT / "scripts"
|
|||||||
|
|
||||||
def load_script(name: str):
|
def load_script(name: str):
|
||||||
path = SCRIPTS / name
|
path = SCRIPTS / name
|
||||||
|
if str(SCRIPTS) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPTS))
|
||||||
spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
|
spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
|
||||||
assert spec is not None and spec.loader is not None
|
assert spec is not None and spec.loader is not None
|
||||||
module = importlib.util.module_from_spec(spec)
|
module = importlib.util.module_from_spec(spec)
|
||||||
@@ -36,6 +40,9 @@ def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha
|
|||||||
"database_password_secure": True,
|
"database_password_secure": True,
|
||||||
"inventory_mode": inventory_mode,
|
"inventory_mode": inventory_mode,
|
||||||
"storage_inventory_requested": True,
|
"storage_inventory_requested": True,
|
||||||
|
"storage_snapshot_requested": True,
|
||||||
|
"models_inventory_requested": False,
|
||||||
|
"models_snapshot_requested": False,
|
||||||
"git_commit": "0123456789abcdef",
|
"git_commit": "0123456789abcdef",
|
||||||
}
|
}
|
||||||
files = {
|
files = {
|
||||||
@@ -48,6 +55,7 @@ def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha
|
|||||||
}
|
}
|
||||||
for name, content in files.items():
|
for name, content in files.items():
|
||||||
(root / name).write_text(content, encoding="utf-8")
|
(root / name).write_text(content, encoding="utf-8")
|
||||||
|
(root / "storage-snapshot").mkdir()
|
||||||
checksums = []
|
checksums = []
|
||||||
for name in sorted(files):
|
for name in sorted(files):
|
||||||
digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
|
digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
|
||||||
@@ -337,8 +345,11 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
|
|||||||
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
|
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||||
live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
|
live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert "DELETE_STORAGE_ARTIFACTS" in generic
|
assert "QUARANTINE_STORAGE_ARTIFACTS" in generic
|
||||||
assert "verify_current_backup" in generic
|
assert "verify_current_backup" in generic
|
||||||
|
assert "os.link" in generic
|
||||||
|
assert 'entry["status"] = "linked"' in generic
|
||||||
|
assert "cleanup-quarantine" in generic
|
||||||
assert "DELETE_DEMO_EXPORTS" in demo
|
assert "DELETE_DEMO_EXPORTS" in demo
|
||||||
assert "verify_current_backup" in demo
|
assert "verify_current_backup" in demo
|
||||||
assert "/app/backups:ro" in compose
|
assert "/app/backups:ro" in compose
|
||||||
@@ -347,6 +358,8 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
|
|||||||
"release_backup_guard.py",
|
"release_backup_guard.py",
|
||||||
"audit_data_operations.py",
|
"audit_data_operations.py",
|
||||||
"cleanup_storage_artifacts.py",
|
"cleanup_storage_artifacts.py",
|
||||||
|
"restore_storage_quarantine.py",
|
||||||
|
"release_backup_snapshot.py",
|
||||||
):
|
):
|
||||||
assert f"COPY scripts/{name}" in dockerfile
|
assert f"COPY scripts/{name}" in dockerfile
|
||||||
assert f"py_compile scripts/{name}" in readiness
|
assert f"py_compile scripts/{name}" in readiness
|
||||||
@@ -356,3 +369,201 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
|
|||||||
assert "table-counts-after.tsv" in live_audit
|
assert "table-counts-after.tsv" in live_audit
|
||||||
assert "deleted_count" in live_audit
|
assert "deleted_count" in live_audit
|
||||||
assert "missing_manifest_artifact_count" in live_audit
|
assert "missing_manifest_artifact_count" in live_audit
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_apply_moves_bytes_to_protected_traceable_quarantine(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
capsys: pytest.CaptureFixture[str],
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.syspath_prepend(str(SCRIPTS))
|
||||||
|
cleanup = load_script("cleanup_storage_artifacts.py")
|
||||||
|
storage = tmp_path / "storage"
|
||||||
|
source = storage / "derived" / "orphan.bin"
|
||||||
|
source.parent.mkdir(parents=True)
|
||||||
|
source.write_bytes(b"recoverable-derived-artifact")
|
||||||
|
candidate = SimpleNamespace(
|
||||||
|
path=source.resolve(),
|
||||||
|
relative_path="derived/orphan.bin",
|
||||||
|
size_bytes=source.stat().st_size,
|
||||||
|
)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
class SessionContext:
|
||||||
|
def __enter__(self):
|
||||||
|
return SimpleNamespace()
|
||||||
|
|
||||||
|
def __exit__(self, *_args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
cleanup,
|
||||||
|
"parse_args",
|
||||||
|
lambda: SimpleNamespace(
|
||||||
|
storage_root=storage,
|
||||||
|
minimum_age_days=7,
|
||||||
|
max_delete=1,
|
||||||
|
apply=True,
|
||||||
|
confirm="QUARANTINE_STORAGE_ARTIFACTS",
|
||||||
|
backup_dir=tmp_path / "backup",
|
||||||
|
backup_max_age_hours=24.0,
|
||||||
|
quarantine_root=None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(cleanup, "SessionLocal", lambda: SessionContext())
|
||||||
|
monkeypatch.setattr(
|
||||||
|
cleanup,
|
||||||
|
"build_report",
|
||||||
|
lambda *_args, **_kwargs: ({"cleanup": {"protected_prefixes": []}}, [candidate]),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
cleanup,
|
||||||
|
"verify_current_backup",
|
||||||
|
lambda *_args, **_kwargs: SimpleNamespace(
|
||||||
|
release_id="predeploy-test",
|
||||||
|
created_at=now,
|
||||||
|
age_hours=0.1,
|
||||||
|
backup_tool_revision="0123456789abcdef",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert cleanup.main() == 0
|
||||||
|
payload = json.loads(capsys.readouterr().out)
|
||||||
|
manifest_path = Path(payload["quarantine_manifest"])
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
quarantined_path = storage / payload["quarantined"][0]["quarantine_relative_path"]
|
||||||
|
|
||||||
|
assert not source.exists()
|
||||||
|
assert quarantined_path.read_bytes() == b"recoverable-derived-artifact"
|
||||||
|
assert manifest["state"] == "complete"
|
||||||
|
assert manifest["backup_release_id"] == "predeploy-test"
|
||||||
|
assert manifest["entries"][0]["status"] == "quarantined"
|
||||||
|
assert payload["deleted_count"] == 0
|
||||||
|
|
||||||
|
restore = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
str(SCRIPTS / "restore_storage_quarantine.py"),
|
||||||
|
"--storage-root",
|
||||||
|
str(storage),
|
||||||
|
"--manifest",
|
||||||
|
str(manifest_path),
|
||||||
|
"--confirm",
|
||||||
|
"RESTORE_QUARANTINED_ARTIFACTS",
|
||||||
|
],
|
||||||
|
cwd=ROOT,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert restore.returncode == 0, restore.stderr
|
||||||
|
assert source.read_bytes() == b"recoverable-derived-artifact"
|
||||||
|
assert not quarantined_path.exists()
|
||||||
|
restored_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
assert restored_manifest["state"] == "restored"
|
||||||
|
assert restored_manifest["entries"][0]["status"] == "restored"
|
||||||
|
|
||||||
|
|
||||||
|
def _write_interrupted_quarantine(
|
||||||
|
storage: Path,
|
||||||
|
*,
|
||||||
|
original_exists: bool,
|
||||||
|
quarantine_exists: bool,
|
||||||
|
hard_linked: bool = False,
|
||||||
|
) -> tuple[Path, Path, Path]:
|
||||||
|
original = storage / "derived" / "interrupted.bin"
|
||||||
|
operation = storage / "operator-evidence" / "cleanup-quarantine" / "cleanup-interrupted"
|
||||||
|
quarantined = operation / "files" / "derived" / "interrupted.bin"
|
||||||
|
original.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
quarantined.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
retained = b"interrupted-retained-bytes"
|
||||||
|
if original_exists:
|
||||||
|
original.write_bytes(retained)
|
||||||
|
if quarantine_exists:
|
||||||
|
if hard_linked:
|
||||||
|
os.link(original, quarantined)
|
||||||
|
else:
|
||||||
|
quarantined.write_bytes(retained)
|
||||||
|
manifest = operation / "manifest.json"
|
||||||
|
manifest.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"state": "in_progress",
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"relative_path": "derived/interrupted.bin",
|
||||||
|
"quarantine_relative_path": quarantined.relative_to(storage).as_posix(),
|
||||||
|
"size_bytes": len(retained),
|
||||||
|
"sha256": hashlib.sha256(retained).hexdigest(),
|
||||||
|
"status": "linked" if hard_linked else "planned",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return manifest, original, quarantined
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("original_exists", "quarantine_exists", "hard_linked"),
|
||||||
|
((False, True, False), (True, True, True)),
|
||||||
|
)
|
||||||
|
def test_quarantine_restore_recovers_each_interrupted_move_window(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
original_exists: bool,
|
||||||
|
quarantine_exists: bool,
|
||||||
|
hard_linked: bool,
|
||||||
|
) -> None:
|
||||||
|
restore = load_script("restore_storage_quarantine.py")
|
||||||
|
storage = tmp_path / "storage"
|
||||||
|
manifest, original, quarantined = _write_interrupted_quarantine(
|
||||||
|
storage,
|
||||||
|
original_exists=original_exists,
|
||||||
|
quarantine_exists=quarantine_exists,
|
||||||
|
hard_linked=hard_linked,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
restore,
|
||||||
|
"parse_args",
|
||||||
|
lambda: SimpleNamespace(
|
||||||
|
storage_root=storage,
|
||||||
|
manifest=manifest,
|
||||||
|
confirm="RESTORE_QUARANTINED_ARTIFACTS",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert restore.main() == 0
|
||||||
|
assert original.read_bytes() == b"interrupted-retained-bytes"
|
||||||
|
assert not quarantined.exists()
|
||||||
|
assert json.loads(manifest.read_text(encoding="utf-8"))["state"] == "restored"
|
||||||
|
|
||||||
|
|
||||||
|
def test_quarantine_restore_never_clobbers_recreated_destination(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
restore = load_script("restore_storage_quarantine.py")
|
||||||
|
storage = tmp_path / "storage"
|
||||||
|
manifest, original, quarantined = _write_interrupted_quarantine(
|
||||||
|
storage,
|
||||||
|
original_exists=False,
|
||||||
|
quarantine_exists=True,
|
||||||
|
)
|
||||||
|
original.write_bytes(b"new-runtime-bytes")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
restore,
|
||||||
|
"parse_args",
|
||||||
|
lambda: SimpleNamespace(
|
||||||
|
storage_root=storage,
|
||||||
|
manifest=manifest,
|
||||||
|
confirm="RESTORE_QUARANTINED_ARTIFACTS",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="different bytes"):
|
||||||
|
restore.main()
|
||||||
|
assert original.read_bytes() == b"new-runtime-bytes"
|
||||||
|
assert quarantined.read_bytes() == b"interrupted-retained-bytes"
|
||||||
|
|||||||
@@ -18,17 +18,16 @@ def test_build_identity_does_not_invalidate_dependency_layers() -> None:
|
|||||||
assert 'io.geointel.ai.enabled="${GEOINTEL_INSTALL_AI}"' in dockerfile
|
assert 'io.geointel.ai.enabled="${GEOINTEL_INSTALL_AI}"' in dockerfile
|
||||||
|
|
||||||
|
|
||||||
def test_release_deploy_preserves_immutable_and_previous_images() -> None:
|
def test_release_deploy_preserves_immutable_and_backup_specific_rollback_images() -> None:
|
||||||
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert 'GEOINTEL_RELEASE_VARIANT="ai"' in script
|
assert 'GEOINTEL_RELEASE_VARIANT="ai"' in script
|
||||||
assert 'GEOINTEL_RELEASE_VARIANT="gis"' in script
|
assert 'GEOINTEL_RELEASE_VARIANT="gis"' in script
|
||||||
assert 'GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"' in script
|
assert 'GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"' in script
|
||||||
assert 'GEOINTEL_PREVIOUS_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:previous"' in script
|
assert 'GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"' in script
|
||||||
assert 'release_image_id="$(' in script
|
assert 'docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
|
||||||
assert '[ "$current_image_id" != "$release_image_id" ]' in script
|
assert '--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
|
||||||
assert 'docker tag "$current_image_id" "$GEOINTEL_PREVIOUS_IMAGE"' in script
|
assert "GEOINTEL_PREVIOUS_IMAGE" not in script
|
||||||
assert "preserving the existing previous image" in script
|
|
||||||
assert 'if docker image inspect "$GEOINTEL_RELEASE_IMAGE"' in script
|
assert 'if docker image inspect "$GEOINTEL_RELEASE_IMAGE"' in script
|
||||||
assert "Immutable release tag has conflicting metadata" in script
|
assert "Immutable release tag has conflicting metadata" in script
|
||||||
assert "Reusing existing immutable image" in script
|
assert "Reusing existing immutable image" in script
|
||||||
@@ -36,12 +35,62 @@ def test_release_deploy_preserves_immutable_and_previous_images() -> None:
|
|||||||
assert "Deployed immutable image" in script
|
assert "Deployed immutable image" in script
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_creates_verified_backup_before_candidate_migrations() -> None:
|
||||||
|
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
backup_index = script.index("create_predeploy_backup\n")
|
||||||
|
scan_index = script.index("scan_release_image\n")
|
||||||
|
candidate_start_index = script.index('if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"')
|
||||||
|
assert scan_index < backup_index
|
||||||
|
assert backup_index < candidate_start_index
|
||||||
|
assert script.index("docker build") < backup_index
|
||||||
|
assert "/mnt/user/appdata/geointel/backups" in script
|
||||||
|
assert "--inventory-mode sha256" in script
|
||||||
|
assert "scripts/verify_release_backup.sh" in script
|
||||||
|
assert "refusing an unbacked migration" in script
|
||||||
|
assert "Quiescing the current backend" in script
|
||||||
|
assert "restarting the unchanged current release" in script
|
||||||
|
assert "preflight_backup_capacity" in script
|
||||||
|
assert "select_verified_link_dest" in script
|
||||||
|
assert "release_backup_snapshot.py" in script
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_starts_only_the_locally_attested_ai_image() -> None:
|
||||||
|
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"' in script
|
||||||
|
assert "Production release deployment requires the gated AI image" in script
|
||||||
|
assert 'bash scripts/generate_container_sbom.sh "$GEOINTEL_RELEASE_IMAGE"' in script
|
||||||
|
assert 'bash scripts/scan_container_image.sh "$GEOINTEL_RELEASE_IMAGE"' in script
|
||||||
|
assert 'running_image_id="$(docker inspect --format \'{{.Image}}\' geointel)"' in script
|
||||||
|
assert 'if [ "$running_image_id" != "$image" ]' in script
|
||||||
|
assert "deployment-attestation.json" in script
|
||||||
|
assert "artifacts/release-evidence/deploy" in script
|
||||||
|
assert "GITEA_COMMIT_SHA" in script
|
||||||
|
assert "GITHUB_SHA" in script
|
||||||
|
assert "must contain one full 40-character Git commit SHA" in script
|
||||||
|
assert 'running_revision" != "$GEOINTEL_BUILD_SHA"' in script
|
||||||
|
assert 'running_ai" != "true"' in script
|
||||||
|
assert '"revision": revision' in script
|
||||||
|
|
||||||
|
|
||||||
def test_release_and_container_replacement_are_serialized() -> None:
|
def test_release_and_container_replacement_are_serialized() -> None:
|
||||||
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||||
|
rollback_script = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
restore_script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
assert "GEOINTEL_DEPLOY_LOCK_FILE" in release_script
|
assert "GEOINTEL_DEPLOY_LOCK_FILE" in release_script
|
||||||
assert "flock -n 9" in release_script
|
assert "flock -n 9" in release_script
|
||||||
|
assert "GEOINTEL_DEPLOY_LOCK_FILE" in rollback_script
|
||||||
|
assert "flock -n 9" in rollback_script
|
||||||
|
assert "GEOINTEL_DEPLOY_LOCK_FILE" in restore_script
|
||||||
|
assert "flock -n 9" in restore_script
|
||||||
|
assert "GEOINTEL_DEPLOY_LOCK_HELD=true" in release_script
|
||||||
assert "GEOINTEL_CONTAINER_LOCK_FILE" in run_script
|
assert "GEOINTEL_CONTAINER_LOCK_FILE" in run_script
|
||||||
assert "flock -w 300 8" in run_script
|
assert "flock -w 300 8" in run_script
|
||||||
assert "GeoIntel container removal did not complete within 60 seconds" in run_script
|
assert "GeoIntel container removal did not complete within 60 seconds" in run_script
|
||||||
@@ -89,13 +138,35 @@ def test_manual_rollback_reuses_persistent_paths_and_requires_existing_image() -
|
|||||||
rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(encoding="utf-8")
|
rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(encoding="utf-8")
|
||||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert "geointel-all-in-one:previous" in rollback
|
assert "Backup manifest lacks an immutable rollback image ID" in rollback
|
||||||
|
assert 'get("image_id", "")' in rollback
|
||||||
assert 'docker image inspect "$GEOINTEL_ROLLBACK_IMAGE"' in rollback
|
assert 'docker image inspect "$GEOINTEL_ROLLBACK_IMAGE"' in rollback
|
||||||
assert 'GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE"' in rollback
|
assert 'GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE"' in rollback
|
||||||
|
assert "restore-predeploy-database.sh" in rollback
|
||||||
|
assert "--confirm-production-database-restore" in rollback
|
||||||
|
assert "Image-only rollback" in rollback
|
||||||
assert '-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data"' in run_script
|
assert '-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data"' in run_script
|
||||||
assert '-v "${GEOINTEL_STORAGE_PATH}:/app/storage"' in run_script
|
assert '-v "${GEOINTEL_STORAGE_PATH}:/app/storage"' in run_script
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_revision_redeploy_rolls_back_by_backup_bound_image_id() -> None:
|
||||||
|
release = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||||
|
rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
restore = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
backup = (ROOT / "scripts" / "backup_release_state.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "current_image_id" in release
|
||||||
|
assert "release_image_id" not in release
|
||||||
|
assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in backup
|
||||||
|
assert 'if [ -z "$RESTORE_IMAGE" ]; then\n RESTORE_IMAGE="$BACKUP_IMAGE_ID"' in restore
|
||||||
|
assert 'GEOINTEL_ROLLBACK_IMAGE="$(python3 - "$BACKUP_DIR/manifest.json"' in rollback
|
||||||
|
assert "geointel-all-in-one:previous" not in release + rollback + restore
|
||||||
|
|
||||||
|
|
||||||
def test_readiness_checks_all_release_shell_entrypoints() -> None:
|
def test_readiness_checks_all_release_shell_entrypoints() -> None:
|
||||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
@@ -107,6 +178,7 @@ def test_readiness_checks_all_release_shell_entrypoints() -> None:
|
|||||||
"deploy/unraid/run-dockerman-container.sh",
|
"deploy/unraid/run-dockerman-container.sh",
|
||||||
"deploy/unraid/deploy-release.sh",
|
"deploy/unraid/deploy-release.sh",
|
||||||
"deploy/unraid/rollback-dockerman-container.sh",
|
"deploy/unraid/rollback-dockerman-container.sh",
|
||||||
|
"deploy/unraid/restore-predeploy-database.sh",
|
||||||
):
|
):
|
||||||
assert f"bash -n {path}" in readiness
|
assert f"bash -n {path}" in readiness
|
||||||
|
|
||||||
|
|||||||
@@ -48,13 +48,54 @@ def test_ci_runs_complete_release_and_supply_chain_gates() -> None:
|
|||||||
assert "pip-audit==2.10.1" in workflow
|
assert "pip-audit==2.10.1" in workflow
|
||||||
assert "audit_python_dependencies.sh" in workflow
|
assert "audit_python_dependencies.sh" in workflow
|
||||||
assert "npm audit --audit-level=high" in workflow
|
assert "npm audit --audit-level=high" in workflow
|
||||||
assert "GEOINTEL_INSTALL_AI=false" in workflow
|
assert "GEOINTEL_INSTALL_AI=true" in workflow
|
||||||
|
assert "geointel-ci:$RELEASE_SHA-ai" in workflow
|
||||||
|
assert "artifacts/image-id.txt" in workflow
|
||||||
|
assert 'scan_container_image.sh "$IMAGE_ID"' in workflow
|
||||||
assert "generate_container_sbom.sh" in workflow
|
assert "generate_container_sbom.sh" in workflow
|
||||||
assert "scan_container_image.sh" in workflow
|
assert "scan_container_image.sh" in workflow
|
||||||
assert "actions/upload-artifact@v4" in workflow
|
assert "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" in workflow
|
||||||
assert context in workflow
|
assert context in workflow
|
||||||
|
|
||||||
|
|
||||||
|
def test_gitea_production_deploy_depends_on_every_release_gate() -> None:
|
||||||
|
release = read(".gitea/workflows/release-gates.yml")
|
||||||
|
legacy_deploy = ROOT / ".gitea" / "workflows" / "unraid-deploy.yml"
|
||||||
|
|
||||||
|
assert "pull_request:" in release
|
||||||
|
assert "needs: [quality, dependency-audit, container]" in release
|
||||||
|
assert "gitea.event_name == 'push'" in release
|
||||||
|
assert "gitea.ref == 'refs/heads/main'" in release
|
||||||
|
assert "/opt/gitea-deploy/deploy.py deploy" in release
|
||||||
|
assert not legacy_deploy.exists()
|
||||||
|
assert "workflow_dispatch:" in release
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_workflows_pin_third_party_actions_to_reviewed_commits() -> None:
|
||||||
|
expected = (
|
||||||
|
"actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683",
|
||||||
|
"actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065",
|
||||||
|
"actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020",
|
||||||
|
"actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02",
|
||||||
|
)
|
||||||
|
for path in (".gitea/workflows/release-gates.yml", ".github/workflows/release-gates.yml"):
|
||||||
|
workflow = read(path)
|
||||||
|
for action in expected:
|
||||||
|
assert action in workflow
|
||||||
|
|
||||||
|
|
||||||
|
def test_managed_validation_targets_the_actual_backend_and_frontend_projects() -> None:
|
||||||
|
workflow = read(".gitea/workflows/managed-validation.yml")
|
||||||
|
|
||||||
|
assert "backend/requirements-ci.lock" in workflow
|
||||||
|
assert "frontend/package-lock.json" in workflow
|
||||||
|
assert "python -m pytest -W error::DeprecationWarning" in workflow
|
||||||
|
assert "cd frontend && npm run test:unit" in workflow
|
||||||
|
assert "python -m ruff check backend scripts tests" in workflow
|
||||||
|
assert "python scripts/verify_repository_layout.py" in workflow
|
||||||
|
assert "[[ -f pyproject.toml" not in workflow
|
||||||
|
|
||||||
|
|
||||||
def test_scanner_images_are_versioned_and_digest_pinned() -> None:
|
def test_scanner_images_are_versioned_and_digest_pinned() -> None:
|
||||||
sbom = read("scripts/generate_container_sbom.sh")
|
sbom = read("scripts/generate_container_sbom.sh")
|
||||||
scan = read("scripts/scan_container_image.sh")
|
scan = read("scripts/scan_container_image.sh")
|
||||||
@@ -65,8 +106,9 @@ def test_scanner_images_are_versioned_and_digest_pinned() -> None:
|
|||||||
assert "--ignore-unfixed" in scan
|
assert "--ignore-unfixed" in scan
|
||||||
assert "--timeout 20m" in scan
|
assert "--timeout 20m" in scan
|
||||||
assert "--scanners vuln" in scan
|
assert "--scanners vuln" in scan
|
||||||
assert '-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro"' in scan
|
assert 'ignored_container_ids' in scan
|
||||||
assert '--ignorefile "$CONTAINER_IGNORE_FILE"' in scan
|
assert 'ignore_args=(-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro")' in scan
|
||||||
|
assert 'trivy_ignore_args=(--ignorefile "$CONTAINER_IGNORE_FILE")' in scan
|
||||||
assert "--skip-files /usr/local/bin/gosu" in scan
|
assert "--skip-files /usr/local/bin/gosu" in scan
|
||||||
assert "final filesystem replaces it with the audited setpriv shell wrapper" in scan
|
assert "final filesystem replaces it with the audited setpriv shell wrapper" in scan
|
||||||
assert "geointel-container-vulnerabilities.json" in scan
|
assert "geointel-container-vulnerabilities.json" in scan
|
||||||
@@ -86,11 +128,12 @@ def test_readiness_guards_lock_and_supply_chain_entrypoints() -> None:
|
|||||||
assert f"bash -n {path}" in readiness
|
assert f"bash -n {path}" in readiness
|
||||||
|
|
||||||
|
|
||||||
def test_python_audit_exceptions_are_timeboxed_and_full_evidence_is_kept() -> None:
|
def test_python_audit_policy_has_no_active_exceptions_and_keeps_full_evidence() -> None:
|
||||||
policy = read("security/pip-audit-exceptions.json")
|
policy = read("security/pip-audit-exceptions.json")
|
||||||
audit_script = read("scripts/audit_python_dependencies.sh")
|
audit_script = read("scripts/audit_python_dependencies.sh")
|
||||||
|
|
||||||
assert '"review_by": "2026-08-31"' in policy
|
assert '"schema_version": 1' in policy
|
||||||
|
assert '"advisories": []' in policy
|
||||||
assert "pip-audit-full.json" in audit_script
|
assert "pip-audit-full.json" in audit_script
|
||||||
assert "pip-audit-policy.json" in audit_script
|
assert "pip-audit-policy.json" in audit_script
|
||||||
assert "--ignore-vuln" in audit_script
|
assert "--ignore-vuln" in audit_script
|
||||||
|
|||||||
@@ -20,12 +20,35 @@ def test_backup_is_atomic_read_only_and_checksum_bound() -> None:
|
|||||||
assert "--no-owner" in script
|
assert "--no-owner" in script
|
||||||
assert "CHECKSUMS.sha256" in script
|
assert "CHECKSUMS.sha256" in script
|
||||||
assert "database-password" not in script.lower()
|
assert "database-password" not in script.lower()
|
||||||
assert 'git -C "$ROOT" rev-parse HEAD' in script
|
assert 'for required in docker python3 sha256sum; do' in script
|
||||||
assert 'git -C "$ROOT" status --porcelain=v1' in script
|
assert 'for required in docker python3 sha256sum git; do' not in script
|
||||||
|
assert "GITEA_COMMIT_SHA" in script
|
||||||
|
assert "GITHUB_SHA" in script
|
||||||
|
assert "GEOINTEL_BUILD_SHA" in script
|
||||||
|
assert 'if command -v git >/dev/null 2>&1' in script
|
||||||
assert "mv \"$PARTIAL\" \"$FINAL\"" in script
|
assert "mv \"$PARTIAL\" \"$FINAL\"" in script
|
||||||
assert "rm -rf -- \"$PARTIAL\"" in script
|
assert "rm -rf -- \"$PARTIAL\"" in script
|
||||||
assert "DROP DATABASE" not in script
|
assert "DROP DATABASE" not in script
|
||||||
assert "pg_restore --clean" not in script
|
assert "pg_restore --clean" not in script
|
||||||
|
assert "/mnt/user/appdata/geointel/backups" in script
|
||||||
|
assert "release_backup_snapshot.py" in script
|
||||||
|
assert "storage-snapshot" not in script # labels are composed without unsafe path interpolation
|
||||||
|
assert "--link-dest-backup" in script
|
||||||
|
assert "--rollback-image-tag" in script
|
||||||
|
assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in script
|
||||||
|
|
||||||
|
|
||||||
|
def test_backup_binds_prepared_source_without_requiring_dot_git() -> None:
|
||||||
|
script = read("backup_release_state.sh")
|
||||||
|
|
||||||
|
controller_resolution = script.index('local gitea_sha="${GITEA_COMMIT_SHA:-}"')
|
||||||
|
optional_git_fallback = script.index('if command -v git >/dev/null 2>&1')
|
||||||
|
docker_access = script.index("docker inspect -f '{{.State.Running}}'")
|
||||||
|
assert controller_resolution < optional_git_fallback < docker_access
|
||||||
|
assert 'SOURCE_REVISION="$explicit_sha"' in script
|
||||||
|
assert '"backup_tool_revision": ${SOURCE_REVISION@Q}' in script
|
||||||
|
assert '"running_image_revision": ${RUNNING_IMAGE_REVISION@Q}' in script
|
||||||
|
assert "Cannot bind backup to a source revision" in script
|
||||||
|
|
||||||
|
|
||||||
def test_backup_verification_is_read_only() -> None:
|
def test_backup_verification_is_read_only() -> None:
|
||||||
@@ -55,9 +78,11 @@ def test_release_safety_scripts_have_valid_bash_syntax() -> None:
|
|||||||
"backup_release_state.sh",
|
"backup_release_state.sh",
|
||||||
"verify_release_backup.sh",
|
"verify_release_backup.sh",
|
||||||
"restore_release_backup_smoke.sh",
|
"restore_release_backup_smoke.sh",
|
||||||
|
"../deploy/unraid/restore-predeploy-database.sh",
|
||||||
):
|
):
|
||||||
|
script_path = f"scripts/{name}" if not name.startswith("../") else name.removeprefix("../")
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["bash", "-n", f"scripts/{name}"],
|
["bash", "-n", script_path],
|
||||||
cwd=ROOT,
|
cwd=ROOT,
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
@@ -66,6 +91,30 @@ def test_release_safety_scripts_have_valid_bash_syntax() -> None:
|
|||||||
assert result.returncode == 0, f"{name}: {result.stderr}"
|
assert result.returncode == 0, f"{name}: {result.stderr}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_production_restore_is_explicit_bounded_and_verified() -> None:
|
||||||
|
script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "--confirm-production-database-restore" in script
|
||||||
|
assert "/mnt/user/appdata/geointel/backups" in script
|
||||||
|
assert "backup.relative_to(root)" in script
|
||||||
|
assert "sha256sum -c CHECKSUMS.sha256" in script
|
||||||
|
assert '"$RESTORE_PROOF_DB"' in script
|
||||||
|
assert "pg_restore" in script
|
||||||
|
assert "Restored Alembic head" in script
|
||||||
|
assert "Restored count mismatch" in script
|
||||||
|
assert "pg_restore --clean" not in script
|
||||||
|
assert "geointel_restore_proof_" in script
|
||||||
|
assert "Isolated predeploy restore proof passed" in script
|
||||||
|
assert "ALTER DATABASE" in script
|
||||||
|
assert "Pre-restore production database retained" in script
|
||||||
|
drop_start = script.index("dropdb --if-exists --force")
|
||||||
|
drop_command = script[drop_start : script.index("\n fi", drop_start)]
|
||||||
|
assert '"$RESTORE_PROOF_DB"' in drop_command
|
||||||
|
assert '"$GEOINTEL_POSTGRES_DB"' not in drop_command
|
||||||
|
|
||||||
|
|
||||||
def test_readiness_gate_checks_release_safety_scripts() -> None:
|
def test_readiness_gate_checks_release_safety_scripts() -> None:
|
||||||
readiness = read("run_readiness_check.sh")
|
readiness = read("run_readiness_check.sh")
|
||||||
|
|
||||||
|
|||||||
@@ -1,11 +1,12 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
def test_backend_keeps_starlette_on_the_supported_pre_httpx2_line() -> None:
|
def test_backend_uses_patched_starlette_and_explicit_httpx2_test_client() -> None:
|
||||||
pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml"
|
pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml"
|
||||||
content = pyproject.read_text(encoding="utf-8")
|
content = pyproject.read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert '"starlette>=0.46.0,<1.0.0"' in content
|
assert '"starlette>=1.3.1,<2.0.0"' in content
|
||||||
|
assert '"httpx2>=2.0.0"' in content
|
||||||
|
|
||||||
|
|
||||||
def test_readiness_gate_treats_deprecation_warnings_as_errors() -> None:
|
def test_readiness_gate_treats_deprecation_warnings_as_errors() -> None:
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
SCRIPT = ROOT / "scripts" / "release_backup_snapshot.py"
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot_module():
|
||||||
|
spec = importlib.util.spec_from_file_location("release_backup_snapshot_test", SCRIPT)
|
||||||
|
assert spec is not None and spec.loader is not None
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def test_snapshot_is_byte_complete_and_reuses_only_verified_backup_bytes(tmp_path: Path) -> None:
|
||||||
|
snapshot = load_snapshot_module()
|
||||||
|
source = tmp_path / "source"
|
||||||
|
source.mkdir()
|
||||||
|
(source / "same.bin").write_bytes(b"unchanged")
|
||||||
|
(source / "changed.bin").write_bytes(b"before")
|
||||||
|
(source / "empty").mkdir()
|
||||||
|
|
||||||
|
prior = tmp_path / "prior"
|
||||||
|
prior_manifest = tmp_path / "prior.tsv"
|
||||||
|
snapshot.create_snapshot(source, prior, prior_manifest, label="storage")
|
||||||
|
snapshot.verify_snapshot(prior, prior_manifest)
|
||||||
|
|
||||||
|
(source / "changed.bin").write_bytes(b"after")
|
||||||
|
current = tmp_path / "current"
|
||||||
|
current_manifest = tmp_path / "current.tsv"
|
||||||
|
snapshot.create_snapshot(
|
||||||
|
source,
|
||||||
|
current,
|
||||||
|
current_manifest,
|
||||||
|
label="storage",
|
||||||
|
link_dest_snapshot=prior,
|
||||||
|
link_dest_manifest=prior_manifest,
|
||||||
|
)
|
||||||
|
snapshot.verify_snapshot(current, current_manifest)
|
||||||
|
|
||||||
|
assert os.path.samefile(prior / "same.bin", current / "same.bin")
|
||||||
|
assert not os.path.samefile(prior / "changed.bin", current / "changed.bin")
|
||||||
|
assert (current / "changed.bin").read_bytes() == b"after"
|
||||||
|
assert (current / "empty").is_dir()
|
||||||
|
|
||||||
|
|
||||||
|
def test_snapshot_rejects_symlinked_content(tmp_path: Path) -> None:
|
||||||
|
snapshot = load_snapshot_module()
|
||||||
|
source = tmp_path / "source"
|
||||||
|
source.mkdir()
|
||||||
|
target = source / "target.bin"
|
||||||
|
target.write_bytes(b"target")
|
||||||
|
try:
|
||||||
|
(source / "link.bin").symlink_to(target)
|
||||||
|
except OSError:
|
||||||
|
pytest.skip("Symlink creation is unavailable on this host")
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="refuses symlinked content"):
|
||||||
|
snapshot.create_snapshot(source, tmp_path / "snapshot", tmp_path / "manifest.tsv", label="storage")
|
||||||
|
|
||||||
|
|
||||||
|
def test_snapshot_verification_rejects_changed_retained_bytes(tmp_path: Path) -> None:
|
||||||
|
snapshot = load_snapshot_module()
|
||||||
|
source = tmp_path / "source"
|
||||||
|
source.mkdir()
|
||||||
|
(source / "artifact.bin").write_bytes(b"retained")
|
||||||
|
retained = tmp_path / "snapshot"
|
||||||
|
manifest = tmp_path / "manifest.tsv"
|
||||||
|
snapshot.create_snapshot(source, retained, manifest, label="storage")
|
||||||
|
(retained / "artifact.bin").chmod(0o644)
|
||||||
|
(retained / "artifact.bin").write_bytes(b"tampered")
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="checksum differs"):
|
||||||
|
snapshot.verify_snapshot(retained, manifest)
|
||||||
@@ -150,7 +150,7 @@ def test_tower_deploy_uses_single_container_unraid_compose() -> None:
|
|||||||
assert "LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh" in release_script
|
assert "LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh" in release_script
|
||||||
|
|
||||||
|
|
||||||
def test_tower_deploy_build_uses_remote_env_ai_setting_by_default() -> None:
|
def test_tower_deploy_build_requires_the_production_ai_variant_by_default() -> None:
|
||||||
powershell = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8")
|
powershell = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8")
|
||||||
bash = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8")
|
bash = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8")
|
||||||
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
||||||
@@ -161,7 +161,8 @@ def test_tower_deploy_build_uses_remote_env_ai_setting_by_default() -> None:
|
|||||||
|
|
||||||
assert "if [ -f .env ]; then" in release_script
|
assert "if [ -f .env ]; then" in release_script
|
||||||
assert ". ./.env" in release_script
|
assert ". ./.env" in release_script
|
||||||
assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-false}"' in release_script
|
assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"' in release_script
|
||||||
|
assert "Production release deployment requires the gated AI image" in release_script
|
||||||
assert "--build-arg GEOINTEL_INSTALL_AI=" in release_script
|
assert "--build-arg GEOINTEL_INSTALL_AI=" in release_script
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -160,8 +160,10 @@ COPY scripts/migrate_runtime_model_provenance.py /app/scripts/migrate_runtime_mo
|
|||||||
COPY scripts/archive_technical_projects.py /app/scripts/archive_technical_projects.py
|
COPY scripts/archive_technical_projects.py /app/scripts/archive_technical_projects.py
|
||||||
COPY scripts/runtime_state_report.py /app/scripts/runtime_state_report.py
|
COPY scripts/runtime_state_report.py /app/scripts/runtime_state_report.py
|
||||||
COPY scripts/release_backup_guard.py /app/scripts/release_backup_guard.py
|
COPY scripts/release_backup_guard.py /app/scripts/release_backup_guard.py
|
||||||
|
COPY scripts/release_backup_snapshot.py /app/scripts/release_backup_snapshot.py
|
||||||
COPY scripts/audit_data_operations.py /app/scripts/audit_data_operations.py
|
COPY scripts/audit_data_operations.py /app/scripts/audit_data_operations.py
|
||||||
COPY scripts/cleanup_storage_artifacts.py /app/scripts/cleanup_storage_artifacts.py
|
COPY scripts/cleanup_storage_artifacts.py /app/scripts/cleanup_storage_artifacts.py
|
||||||
|
COPY scripts/restore_storage_quarantine.py /app/scripts/restore_storage_quarantine.py
|
||||||
COPY deploy/unraid/nginx-all-in-one.conf /etc/nginx/conf.d/default.conf
|
COPY deploy/unraid/nginx-all-in-one.conf /etc/nginx/conf.d/default.conf
|
||||||
COPY deploy/unraid/all-in-one-start.sh /usr/local/bin/geointel-all-in-one-start
|
COPY deploy/unraid/all-in-one-start.sh /usr/local/bin/geointel-all-in-one-start
|
||||||
COPY --from=frontend-build /frontend/dist/ /usr/share/nginx/html/
|
COPY --from=frontend-build /frontend/dist/ /usr/share/nginx/html/
|
||||||
|
|||||||
@@ -11,9 +11,8 @@ met de naam `geointel`, bereikbaar op `http://192.168.10.150:1202`.
|
|||||||
Kopieer de **volledige** map `C:\Projects\geointel` naar
|
Kopieer de **volledige** map `C:\Projects\geointel` naar
|
||||||
`/mnt/user/appdata/geointel` op de server. Verder niets uitzoeken.
|
`/mnt/user/appdata/geointel` op de server. Verder niets uitzoeken.
|
||||||
|
|
||||||
`.dockerignore` regelt de rest: `node_modules/`, `.git/`, de dubbele
|
`.dockerignore` regelt de rest: `node_modules/`, `.git/`, `docs/`, `artifacts/`
|
||||||
`geointel/`-map, `docs/`, `artifacts/` en testoutput gaan niet mee de
|
en testoutput gaan niet mee de build-context in, ook al staan ze in de map.
|
||||||
build-context in, ook al staan ze in de map.
|
|
||||||
|
|
||||||
Eén waarschuwing bij het overschrijven: laat `storage/`, `postgres-data/`,
|
Eén waarschuwing bij het overschrijven: laat `storage/`, `postgres-data/`,
|
||||||
`backups/` en `models/` op de server **staan**. Dat is je bestaande data, en
|
`backups/` en `models/` op de server **staan**. Dat is je bestaande data, en
|
||||||
@@ -71,7 +70,8 @@ die hostname in `GEOINTEL_CORS_ORIGINS` staan — anders blokkeert de browser de
|
|||||||
API-calls vanaf het publieke adres. De backend doet geen host-validatie, dus
|
API-calls vanaf het publieke adres. De backend doet geen host-validatie, dus
|
||||||
verder is er niets nodig aan applicatiekant.
|
verder is er niets nodig aan applicatiekant.
|
||||||
|
|
||||||
Voor GPU-inferentie (optioneel, kan ook later):
|
De productie-image bevat altijd de gepinde AI-runtime. Inferentie zelf kan
|
||||||
|
uitblijven totdat een lokaal, gevalideerd model beschikbaar is:
|
||||||
|
|
||||||
```env
|
```env
|
||||||
GEOINTEL_INSTALL_AI=true
|
GEOINTEL_INSTALL_AI=true
|
||||||
@@ -81,8 +81,8 @@ YOLO_REQUIRE_CUDA=true
|
|||||||
YOLO_MODEL_PATH=/app/models/<jouw-model>.pt
|
YOLO_MODEL_PATH=/app/models/<jouw-model>.pt
|
||||||
```
|
```
|
||||||
|
|
||||||
Laat `GEOINTEL_INSTALL_AI=false` staan als je eerst gewoon wilt dat de app
|
Laat `YOLO_ENABLED=false` zolang er geen geschikt modelbestand is. Het
|
||||||
draait — dat scheelt een paar GB aan PyTorch-lagen in de build.
|
release-deployscript weigert bewust een GIS-only productie-image.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -134,12 +134,21 @@ bash deploy/unraid/deploy-release.sh
|
|||||||
|
|
||||||
Het script:
|
Het script:
|
||||||
|
|
||||||
1. ruimt een eventueel achtergebleven Compose-stack op (ook de oude
|
1. bindt het exacte huidige image-ID aan een unieke backup-specifieke
|
||||||
|
`rollback-predeploy-*`-tag;
|
||||||
|
2. bouwt of hergebruikt de AI-candidate terwijl de huidige release beschikbaar
|
||||||
|
blijft, legt het exacte lokale image-ID vast en maakt daarop SBOM- en
|
||||||
|
Trivy-evidence;
|
||||||
|
3. controleert vóór het pauzeren de vrije ruimte en maakt vervolgens een
|
||||||
|
byte-complete, SHA-256-geverifieerde database-, storage- en modelsnapshot in
|
||||||
|
`/mnt/user/appdata/geointel/backups`; ongewijzigde bestanden mogen alleen
|
||||||
|
vanuit een oudere geverifieerde backup worden gehardlinkt;
|
||||||
|
4. ruimt een eventueel achtergebleven Compose-stack op (ook de oude
|
||||||
3-container dev-stack die óók poort 1202 pakt);
|
3-container dev-stack die óók poort 1202 pakt);
|
||||||
2. bewaart de huidige image als `geointel-all-in-one:previous`;
|
5. start één container `geointel` met `-p 1202:80` en `--gpus all`;
|
||||||
3. bouwt `deploy/unraid/Dockerfile.all-in-one`;
|
6. bewijst een rollbackdump eerst in een geïsoleerde tijdelijke database,
|
||||||
4. start één container `geointel` met `-p 1202:80` en `--gpus all`;
|
bewaart de oude productiedatabase als herstelpad en start pas daarna
|
||||||
5. rolt automatisch terug naar `:previous` als de healthcheck of smoke faalt.
|
automatisch het image-ID dat cryptografisch in die pre-deploybackup staat.
|
||||||
|
|
||||||
De eerste build duurt lang (PostGIS + GDAL + npm build). Volgende deploys
|
De eerste build duurt lang (PostGIS + GDAL + npm build). Volgende deploys
|
||||||
hergebruiken de Docker-layercache.
|
hergebruiken de Docker-layercache.
|
||||||
@@ -165,24 +174,13 @@ docker logs --tail 200 geointel
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 7. Belangrijk: dubbele projectmap lokaal
|
## 7. Geretireerde dubbele projectmap
|
||||||
|
|
||||||
In `C:\Projects\geointel` staat een tweede, volledige kopie van het project
|
De vroegere geneste mirror `C:\Projects\geointel\geointel` is geretireerd. De
|
||||||
onder `C:\Projects\geointel\geointel\`. Die bevat dezelfde bestanden en
|
immutable Git-herkomst, niet-getrackte recoverybestanden en verificatiegrens
|
||||||
dezelfde datum, maar staat buiten git. Zolang die er staat:
|
staan in `docs/accuracy-program/13-nested-mirror-retirement.md`. Behandel die
|
||||||
|
evidence als herstelreferentie; deze handleiding vraagt geen extra kopieer- of
|
||||||
- wordt de Docker build-context onnodig verdubbeld;
|
opruimactie.
|
||||||
- weet je bij het bewerken van bijvoorbeeld `docker-compose.unraid.yml` niet
|
|
||||||
welke versie je te pakken hebt.
|
|
||||||
|
|
||||||
`.dockerignore` sluit hem nu uit, maar ruim hem op zodra je zeker weet dat er
|
|
||||||
niets unieks in staat. Vergelijk eerst:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
robocopy C:\Projects\geointel\geointel C:\Projects\geointel /L /E /NJH /NJS /NDL /XF *.pyc
|
|
||||||
```
|
|
||||||
|
|
||||||
Regels die als `New File` verschijnen bestaan alleen in de kopie.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -190,11 +188,15 @@ Regels die als `New File` verschijnen bestaan alleen in de kopie.
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/user/appdata/geointel
|
cd /mnt/user/appdata/geointel
|
||||||
bash deploy/unraid/rollback-dockerman-container.sh
|
bash deploy/unraid/rollback-dockerman-container.sh \
|
||||||
|
--backup-dir /mnt/user/appdata/geointel/backups/<predeploy-release-id> \
|
||||||
|
--confirm-production-database-restore
|
||||||
```
|
```
|
||||||
|
|
||||||
Rollback hergebruikt dezelfde PostGIS- en storage-paden en draait nooit een
|
Rollback hergebruikt dezelfde storage-paden, bewijst de geverifieerde dump
|
||||||
Alembic-downgrade.
|
eerst geïsoleerd, wisselt daarna databases via no-clobber namen en draait nooit
|
||||||
|
een Alembic-downgrade. De oude productiedatabase blijft staan totdat een
|
||||||
|
operator haar na controle expliciet opruimt.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -202,7 +204,7 @@ Alembic-downgrade.
|
|||||||
|
|
||||||
| Bestand | Aanpassing |
|
| Bestand | Aanpassing |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `.dockerignore` | Sluit root-`node_modules`, de dubbele `geointel/`-map, `.git`, `docs/`, `artifacts/` en testoutput uit de build-context |
|
| `.dockerignore` | Sluit root-`node_modules`, `.git`, `docs/`, `artifacts/` en testoutput uit de build-context |
|
||||||
| `deploy/unraid/deploy-release.sh` | `git rev-parse HEAD` crashte op een kopie zonder `.git`. Valt nu terug op `GEOINTEL_BUILD_SHA`, een `RELEASE_SHA`-bestand of een content-hash van de broncode |
|
| `deploy/unraid/deploy-release.sh` | `git rev-parse HEAD` crashte op een kopie zonder `.git`. Valt nu terug op `GEOINTEL_BUILD_SHA`, een `RELEASE_SHA`-bestand of een content-hash van de broncode |
|
||||||
| `deploy/unraid/deploy-release.sh` | Smoke-scripts worden op bestaan getest in plaats van op de execute-bit, die bij een Windows-kopie verloren gaat |
|
| `deploy/unraid/deploy-release.sh` | Smoke-scripts worden op bestaan getest in plaats van op de execute-bit, die bij een Windows-kopie verloren gaat |
|
||||||
| `deploy/unraid/run-dockerman-container.sh` | Ruimt expliciet zowel `docker-compose.yml` (3 containers) als `docker-compose.unraid.yml` op, zodat poort 1202 gegarandeerd vrij is |
|
| `deploy/unraid/run-dockerman-container.sh` | Ruimt expliciet zowel `docker-compose.yml` (3 containers) als `docker-compose.unraid.yml` op, zodat poort 1202 gegarandeerd vrij is |
|
||||||
|
|||||||
+30
-22
@@ -106,27 +106,25 @@ customer or operational data. Deploy a separate demo container and storage
|
|||||||
root for public or recruiter-facing access.
|
root for public or recruiter-facing access.
|
||||||
|
|
||||||
The repository deploy scripts run the same flow automatically. They validate
|
The repository deploy scripts run the same flow automatically. They validate
|
||||||
the Compose reference, preserve the current image as
|
the Compose reference, preserve the current image under a unique
|
||||||
`geointel-all-in-one:previous`, build an immutable `<commit-sha>-ai` or
|
backup-specific `rollback-predeploy-*` tag, build the immutable production
|
||||||
`<commit-sha>-gis` tag plus `latest`, install the DockerMan metadata and start
|
`<commit-sha>-ai` tag plus `latest`, attest its exact local image ID, generate
|
||||||
the immutable image. An existing matching tag is reused, never rebuilt. A
|
an SBOM and enforce the Trivy policy before starting that same ID. An existing
|
||||||
failed start, live migration smoke or browser/API smoke automatically attempts
|
matching tag is reused, never rebuilt. A failed start, live migration smoke or
|
||||||
the previous image without changing the configured PostGIS or storage paths.
|
browser/API smoke automatically attempts the previous image without changing
|
||||||
|
the configured PostGIS or storage paths.
|
||||||
|
|
||||||
`scripts/deploy_tower.sh` and `scripts/deploy_tower.ps1` source the remote
|
`scripts/deploy_tower.sh` and `scripts/deploy_tower.ps1` source the remote
|
||||||
`.env` before building the image. That means `GEOINTEL_INSTALL_AI=true` in
|
`.env` before building the image. Production deployment requires
|
||||||
`/mnt/user/appdata/geointel/.env` is enough for the automatic deploy to build
|
`GEOINTEL_INSTALL_AI=true`; an explicit false value fails closed before the
|
||||||
the AI-enabled image. Set `GEOINTEL_INSTALL_AI` in the local shell or pass
|
image or running container is replaced.
|
||||||
`-InstallAi true/false` to the PowerShell wrapper only when you intentionally
|
|
||||||
want to override the remote `.env` for that deploy.
|
|
||||||
|
|
||||||
Database credentials are runtime configuration, not image metadata. The
|
Database credentials are runtime configuration, not image metadata. The
|
||||||
all-in-one image does not bake `GEOINTEL_POSTGRES_PASSWORD` into the Dockerfile;
|
all-in-one image does not bake `GEOINTEL_POSTGRES_PASSWORD` into the Dockerfile;
|
||||||
set it through `.env`, the Unraid template or `docker run -e`.
|
set it through `.env`, the Unraid template or `docker run -e`.
|
||||||
|
|
||||||
AI dependencies are opt-in. Leave `GEOINTEL_INSTALL_AI=false` for the default
|
Production images always include the pinned AI dependencies. Mount models
|
||||||
GIS-only image. Set `GEOINTEL_INSTALL_AI=true`, mount models through
|
through `GEOINTEL_MODELS_PATH` and configure `YOLO_ENABLED=true` plus
|
||||||
`GEOINTEL_MODELS_PATH` and configure `YOLO_ENABLED=true` plus
|
|
||||||
`YOLO_MODELS_DIR=/app/models` and `YOLO_MODEL_PATH=/app/models/<model>.pt` only
|
`YOLO_MODELS_DIR=/app/models` and `YOLO_MODEL_PATH=/app/models/<model>.pt` only
|
||||||
when you have a local model file.
|
when you have a local model file.
|
||||||
The AI-enabled image installs PyTorch/Ultralytics plus the native OpenCV runtime
|
The AI-enabled image installs PyTorch/Ultralytics plus the native OpenCV runtime
|
||||||
@@ -264,7 +262,8 @@ git reset --hard origin/main
|
|||||||
bash deploy/unraid/deploy-release.sh
|
bash deploy/unraid/deploy-release.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
The equivalent low-level build remains available for debugging:
|
The equivalent low-level GIS-only build remains available only for local
|
||||||
|
debugging; it is not a production deployment path:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker build --build-arg GEOINTEL_INSTALL_AI=${GEOINTEL_INSTALL_AI:-false} -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest .
|
docker build --build-arg GEOINTEL_INSTALL_AI=${GEOINTEL_INSTALL_AI:-false} -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest .
|
||||||
@@ -305,20 +304,27 @@ bash scripts/verify_release_upgrade_smoke.sh \
|
|||||||
Return to the image that was active immediately before the latest deployment:
|
Return to the image that was active immediately before the latest deployment:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bash deploy/unraid/rollback-dockerman-container.sh
|
bash deploy/unraid/rollback-dockerman-container.sh \
|
||||||
|
--backup-dir /mnt/user/appdata/geointel/backups/<predeploy-release-id> \
|
||||||
|
--confirm-production-database-restore
|
||||||
```
|
```
|
||||||
|
|
||||||
For an older retained commit, select its immutable tag explicitly:
|
For an older retained commit, select its immutable tag explicitly:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
GEOINTEL_ROLLBACK_IMAGE=geointel-all-in-one:<commit-sha>-ai \
|
GEOINTEL_ROLLBACK_IMAGE=geointel-all-in-one:<commit-sha>-ai \
|
||||||
bash deploy/unraid/rollback-dockerman-container.sh
|
bash deploy/unraid/rollback-dockerman-container.sh \
|
||||||
|
--backup-dir /mnt/user/appdata/geointel/backups/<matching-predeploy-release-id> \
|
||||||
|
--confirm-production-database-restore
|
||||||
```
|
```
|
||||||
|
|
||||||
Rollback reuses the configured PostGIS and storage mounts and never runs an
|
Rollback restores and verifies the selected dump in an isolated proof database
|
||||||
Alembic downgrade. If a future release has a backward-incompatible migration,
|
before any production replacement. It then swaps database names, retains the
|
||||||
restore its verified pre-release backup instead of forcing an older app
|
pre-restore production database for operator recovery, reuses the configured
|
||||||
against a newer schema.
|
storage mount and never runs an Alembic downgrade or an older app against an
|
||||||
|
unknown newer schema. Remove the retained recovery database and old backup
|
||||||
|
directories only in a separately reviewed operator retention step; deployment
|
||||||
|
never deletes them automatically.
|
||||||
|
|
||||||
The configured upload limit is shared by FastAPI and the generated nginx
|
The configured upload limit is shared by FastAPI and the generated nginx
|
||||||
runtime configuration. Values outside `1..2048` MiB are rejected before the
|
runtime configuration. Values outside `1..2048` MiB are rejected before the
|
||||||
@@ -339,7 +345,9 @@ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py
|
|||||||
```
|
```
|
||||||
|
|
||||||
The full backup, confirmation, candidate-limit and apply sequence is in
|
The full backup, confirmation, candidate-limit and apply sequence is in
|
||||||
`docs/DATA_OPERATIONS_RUNBOOK.md`. GeoIntel installs no automatic cleanup
|
`docs/DATA_OPERATIONS_RUNBOOK.md`. Apply moves bytes to protected,
|
||||||
|
checksum-bound quarantine rather than deleting them; a separate confirmed
|
||||||
|
restore command reverses the move. GeoIntel installs no automatic cleanup
|
||||||
schedule.
|
schedule.
|
||||||
|
|
||||||
## Safe cleanup
|
## Safe cleanup
|
||||||
|
|||||||
+372
-22
@@ -26,7 +26,11 @@ if [ -n "${DEPLOY_GEOINTEL_INSTALL_AI:-}" ]; then
|
|||||||
GEOINTEL_INSTALL_AI="$DEPLOY_GEOINTEL_INSTALL_AI"
|
GEOINTEL_INSTALL_AI="$DEPLOY_GEOINTEL_INSTALL_AI"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-false}"
|
GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"
|
||||||
|
if [ "$GEOINTEL_INSTALL_AI" != "true" ]; then
|
||||||
|
echo "Production release deployment requires the gated AI image (GEOINTEL_INSTALL_AI=true)." >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
GEOINTEL_APP_VERSION="$(tr -d '[:space:]' < VERSION)"
|
GEOINTEL_APP_VERSION="$(tr -d '[:space:]' < VERSION)"
|
||||||
if ! [[ "$GEOINTEL_APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then
|
if ! [[ "$GEOINTEL_APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then
|
||||||
echo "Invalid semantic version in VERSION: ${GEOINTEL_APP_VERSION}" >&2
|
echo "Invalid semantic version in VERSION: ${GEOINTEL_APP_VERSION}" >&2
|
||||||
@@ -57,7 +61,52 @@ source_tree_hash() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
resolve_build_sha() {
|
resolve_build_sha() {
|
||||||
local head="" content=""
|
local head="" content="" controller_sha="" controller_source=""
|
||||||
|
|
||||||
|
if [ -n "${GITEA_COMMIT_SHA:-}" ]; then
|
||||||
|
controller_sha="$GITEA_COMMIT_SHA"
|
||||||
|
controller_source="GITEA_COMMIT_SHA"
|
||||||
|
fi
|
||||||
|
if [ -n "${GITHUB_SHA:-}" ]; then
|
||||||
|
if ! [[ "$GITHUB_SHA" =~ ^[0-9A-Fa-f]{40}$ ]]; then
|
||||||
|
echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
if [ -n "$controller_sha" ] && [ "${controller_sha,,}" != "${GITHUB_SHA,,}" ]; then
|
||||||
|
echo "Controller commit variables disagree." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
controller_sha="$GITHUB_SHA"
|
||||||
|
controller_source="${controller_source:-GITHUB_SHA}"
|
||||||
|
fi
|
||||||
|
if [ -n "$controller_sha" ]; then
|
||||||
|
if ! [[ "$controller_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
|
||||||
|
echo "${controller_source} must contain one full 40-character Git commit SHA." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
controller_sha="${controller_sha,,}"
|
||||||
|
if [ -n "${GEOINTEL_BUILD_SHA:-}" ] && [ "${GEOINTEL_BUILD_SHA,,}" != "$controller_sha" ]; then
|
||||||
|
echo "Explicit build revision differs from the controller revision." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
if command -v git >/dev/null 2>&1 && git rev-parse --git-dir >/dev/null 2>&1; then
|
||||||
|
head="$(git rev-parse HEAD 2>/dev/null || true)"
|
||||||
|
if [ "${head,,}" != "$controller_sha" ]; then
|
||||||
|
echo "Prepared Git checkout does not match the controller revision." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
|
||||||
|
echo "Prepared Git checkout contains changes outside the controller revision." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
printf '%s' "$controller_sha"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [ -n "${GITEA_REPOSITORY:-}" ] || [ -n "${GITHUB_REPOSITORY:-}" ]; then
|
||||||
|
echo "Automated deployment context is missing GITEA_COMMIT_SHA/GITHUB_SHA." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
|
||||||
# 1. Explicit override wins.
|
# 1. Explicit override wins.
|
||||||
if [ -n "${GEOINTEL_BUILD_SHA:-}" ]; then
|
if [ -n "${GEOINTEL_BUILD_SHA:-}" ]; then
|
||||||
@@ -107,7 +156,13 @@ if [ -z "$GEOINTEL_BUILD_SHA" ]; then
|
|||||||
echo "Could not determine a build revision for this deployment." >&2
|
echo "Could not determine a build revision for this deployment." >&2
|
||||||
exit 2
|
exit 2
|
||||||
fi
|
fi
|
||||||
|
export GEOINTEL_BUILD_SHA
|
||||||
echo "Build revision: ${GEOINTEL_BUILD_SHA}"
|
echo "Build revision: ${GEOINTEL_BUILD_SHA}"
|
||||||
|
GEOINTEL_RELEASE_TOKEN="$(printf '%s' "$GEOINTEL_BUILD_SHA" | tr -c 'A-Za-z0-9._-' '_' | cut -c1-48)"
|
||||||
|
if [ -z "$GEOINTEL_RELEASE_TOKEN" ]; then
|
||||||
|
echo "Could not derive a safe release evidence identifier." >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
GEOINTEL_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
GEOINTEL_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
GEOINTEL_IMAGE_REPOSITORY="${GEOINTEL_IMAGE_REPOSITORY:-geointel-all-in-one}"
|
GEOINTEL_IMAGE_REPOSITORY="${GEOINTEL_IMAGE_REPOSITORY:-geointel-all-in-one}"
|
||||||
if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then
|
if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then
|
||||||
@@ -116,8 +171,23 @@ else
|
|||||||
GEOINTEL_RELEASE_VARIANT="gis"
|
GEOINTEL_RELEASE_VARIANT="gis"
|
||||||
fi
|
fi
|
||||||
GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"
|
GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"
|
||||||
GEOINTEL_PREVIOUS_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:previous"
|
|
||||||
FRONTEND_URL="${FRONTEND_URL:-http://127.0.0.1:${GEOINTEL_FRONTEND_PORT:-1202}}"
|
FRONTEND_URL="${FRONTEND_URL:-http://127.0.0.1:${GEOINTEL_FRONTEND_PORT:-1202}}"
|
||||||
|
GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
|
||||||
|
GEOINTEL_STORAGE_PATH="${GEOINTEL_STORAGE_PATH:-/mnt/user/appdata/geointel/storage}"
|
||||||
|
GEOINTEL_MODELS_PATH="${GEOINTEL_MODELS_PATH:-/mnt/user/appdata/geointel/models}"
|
||||||
|
GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}"
|
||||||
|
GEOINTEL_DEPLOY_EVIDENCE_DIR="${GEOINTEL_DEPLOY_EVIDENCE_DIR:-artifacts/release-evidence/deploy/${GEOINTEL_RELEASE_TOKEN}-ai}"
|
||||||
|
GEOINTEL_PREDEPLOY_BACKUP_DIR=""
|
||||||
|
GEOINTEL_RELEASE_IMAGE_ID=""
|
||||||
|
GEOINTEL_BACKUP_LINK_DEST=""
|
||||||
|
GEOINTEL_PREDEPLOY_ROLLBACK_TAG=""
|
||||||
|
|
||||||
|
case "$GEOINTEL_DEPLOY_EVIDENCE_DIR" in
|
||||||
|
/*|*..*)
|
||||||
|
echo "Deployment evidence directory must be repository-relative and must not contain '..'." >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
wait_for_geointel_health() {
|
wait_for_geointel_health() {
|
||||||
local status=""
|
local status=""
|
||||||
@@ -141,35 +211,312 @@ wait_for_geointel_health() {
|
|||||||
|
|
||||||
start_image() {
|
start_image() {
|
||||||
local image="$1"
|
local image="$1"
|
||||||
|
local running_image_id=""
|
||||||
|
local running_revision=""
|
||||||
|
local running_ai=""
|
||||||
GEOINTEL_IMAGE="$image" bash deploy/unraid/run-dockerman-container.sh
|
GEOINTEL_IMAGE="$image" bash deploy/unraid/run-dockerman-container.sh
|
||||||
wait_for_geointel_health
|
wait_for_geointel_health
|
||||||
|
running_image_id="$(docker inspect --format '{{.Image}}' geointel)"
|
||||||
|
if [ "$running_image_id" != "$image" ]; then
|
||||||
|
echo "Running container image ${running_image_id} differs from attested image ${image}." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
running_revision="$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' geointel)"
|
||||||
|
running_ai="$(docker inspect --format '{{index .Config.Labels "io.geointel.ai.enabled"}}' geointel)"
|
||||||
|
if [ "$running_revision" != "$GEOINTEL_BUILD_SHA" ] || [ "$running_ai" != "true" ]; then
|
||||||
|
echo "Running container labels do not match the attested AI revision." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "Running container matches attested image: ${running_image_id}"
|
||||||
|
}
|
||||||
|
|
||||||
|
scan_release_image() {
|
||||||
|
local scanned_image_id=""
|
||||||
|
local current_image_id=""
|
||||||
|
local inspect_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/image-inspect.json"
|
||||||
|
local sbom_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-sbom.spdx.json"
|
||||||
|
local vulnerability_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-container-vulnerabilities.json"
|
||||||
|
local attestation_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/deployment-attestation.json"
|
||||||
|
|
||||||
|
scanned_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
|
||||||
|
test -n "$scanned_image_id"
|
||||||
|
mkdir -p "$ROOT/$GEOINTEL_DEPLOY_EVIDENCE_DIR"
|
||||||
|
docker image inspect "$GEOINTEL_RELEASE_IMAGE" > "$ROOT/$inspect_output"
|
||||||
|
bash scripts/generate_container_sbom.sh "$GEOINTEL_RELEASE_IMAGE" "$sbom_output"
|
||||||
|
bash scripts/scan_container_image.sh "$GEOINTEL_RELEASE_IMAGE" "$vulnerability_output"
|
||||||
|
current_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
|
||||||
|
if [ "$current_image_id" != "$scanned_image_id" ]; then
|
||||||
|
echo "Release image tag changed while SBOM/scan evidence was being generated." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
test -s "$ROOT/$inspect_output"
|
||||||
|
test -s "$ROOT/$sbom_output"
|
||||||
|
test -s "$ROOT/$vulnerability_output"
|
||||||
|
GEOINTEL_RELEASE_IMAGE_ID="$scanned_image_id"
|
||||||
|
python3 - \
|
||||||
|
"$ROOT/$attestation_output" \
|
||||||
|
"$GEOINTEL_RELEASE_IMAGE" \
|
||||||
|
"$GEOINTEL_RELEASE_IMAGE_ID" \
|
||||||
|
"$GEOINTEL_BUILD_SHA" \
|
||||||
|
"$inspect_output" \
|
||||||
|
"$sbom_output" \
|
||||||
|
"$vulnerability_output" <<'PY'
|
||||||
|
import datetime
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
output, image_tag, image_id, revision, inspect_path, sbom_path, vulnerability_path = sys.argv[1:]
|
||||||
|
payload = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"attested_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||||
|
"image_tag": image_tag,
|
||||||
|
"image_id": image_id,
|
||||||
|
"image_config_digest": image_id,
|
||||||
|
"revision": revision,
|
||||||
|
"variant": "ai",
|
||||||
|
"evidence": {
|
||||||
|
"image_inspect": inspect_path,
|
||||||
|
"sbom": sbom_path,
|
||||||
|
"vulnerabilities": vulnerability_path,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
path = pathlib.Path(output)
|
||||||
|
temporary = path.with_suffix(".json.partial")
|
||||||
|
temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
temporary.replace(path)
|
||||||
|
PY
|
||||||
|
test -s "$ROOT/$attestation_output"
|
||||||
|
echo "Exact deployment image scanned: ${GEOINTEL_RELEASE_IMAGE_ID}"
|
||||||
|
}
|
||||||
|
|
||||||
|
preflight_backup_capacity() {
|
||||||
|
local database_name=""
|
||||||
|
local database_user=""
|
||||||
|
local database_size_bytes=""
|
||||||
|
|
||||||
|
database_name="$(docker exec geointel sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"')"
|
||||||
|
database_user="$(docker exec geointel sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"')"
|
||||||
|
database_size_bytes="$(docker exec geointel psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$database_user" -d "$database_name" -Atqc \
|
||||||
|
'SELECT pg_database_size(current_database());')"
|
||||||
|
mkdir -p "$GEOINTEL_BACKUPS_PATH"
|
||||||
|
python3 - \
|
||||||
|
"$GEOINTEL_BACKUPS_PATH" \
|
||||||
|
"$GEOINTEL_STORAGE_PATH" \
|
||||||
|
"$GEOINTEL_MODELS_PATH" \
|
||||||
|
"$database_size_bytes" <<'PY'
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import sys
|
||||||
|
|
||||||
|
backup_root = pathlib.Path(sys.argv[1]).expanduser().resolve()
|
||||||
|
sources = [pathlib.Path(value).expanduser().resolve() for value in sys.argv[2:4]]
|
||||||
|
database_bytes = int(sys.argv[4])
|
||||||
|
|
||||||
|
def retained_bytes(root: pathlib.Path) -> int:
|
||||||
|
if not root.is_dir():
|
||||||
|
raise SystemExit(f"Mandatory snapshot source is not a directory: {root}")
|
||||||
|
total = 0
|
||||||
|
for current, directories, files in os.walk(root, topdown=True, followlinks=False):
|
||||||
|
current_path = pathlib.Path(current)
|
||||||
|
for name in [*directories, *files]:
|
||||||
|
path = current_path / name
|
||||||
|
details = path.lstat()
|
||||||
|
if stat.S_ISLNK(details.st_mode):
|
||||||
|
raise SystemExit(f"Mandatory snapshot refuses symlinked content: {path}")
|
||||||
|
if name in directories and not stat.S_ISDIR(details.st_mode):
|
||||||
|
raise SystemExit(f"Snapshot directory changed during capacity preflight: {path}")
|
||||||
|
if name in files:
|
||||||
|
if not stat.S_ISREG(details.st_mode):
|
||||||
|
raise SystemExit(f"Mandatory snapshot refuses non-regular content: {path}")
|
||||||
|
total += details.st_size
|
||||||
|
return total
|
||||||
|
|
||||||
|
source_bytes = sum(retained_bytes(source) for source in sources)
|
||||||
|
# Reflink clones are used when the backing filesystem supports them. Budget for
|
||||||
|
# a complete copy plus two uncompressed database sizes (dump and isolated
|
||||||
|
# restore/cutover recovery) so fallback still fails before the live backend is
|
||||||
|
# quiesced rather than midway through the snapshot.
|
||||||
|
required = source_bytes + (2 * database_bytes)
|
||||||
|
headroom = max(5 * 1024**3, required // 10)
|
||||||
|
free = shutil.disk_usage(backup_root).free
|
||||||
|
if free < required + headroom:
|
||||||
|
raise SystemExit(
|
||||||
|
"Insufficient free space for a fail-safe predeploy snapshot: "
|
||||||
|
f"required={required + headroom} free={free} source={source_bytes} database={database_bytes}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"Predeploy snapshot capacity: "
|
||||||
|
f"source_bytes={source_bytes} database_bytes={database_bytes} free_bytes={free}"
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
select_verified_link_dest() {
|
||||||
|
local candidate=""
|
||||||
|
GEOINTEL_BACKUP_LINK_DEST=""
|
||||||
|
while IFS= read -r candidate; do
|
||||||
|
if (
|
||||||
|
cd "$candidate" \
|
||||||
|
&& sha256sum -c CHECKSUMS.sha256 >/dev/null \
|
||||||
|
&& python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate"
|
||||||
|
); then
|
||||||
|
GEOINTEL_BACKUP_LINK_DEST="$candidate"
|
||||||
|
echo "Using verified prior byte snapshot as link-dest: ${candidate}"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "Skipping unusable prior backup link-dest: ${candidate}" >&2
|
||||||
|
done < <(
|
||||||
|
python3 - "$GEOINTEL_BACKUPS_PATH" <<'PY'
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
root = pathlib.Path(sys.argv[1]).expanduser().resolve()
|
||||||
|
candidates = sorted(
|
||||||
|
(
|
||||||
|
path
|
||||||
|
for path in root.iterdir()
|
||||||
|
if path.is_dir() and not path.name.startswith(".") and (path / "manifest.json").is_file()
|
||||||
|
),
|
||||||
|
key=lambda path: path.stat().st_mtime_ns,
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
for candidate in candidates:
|
||||||
|
print(candidate)
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
echo "No verified prior byte snapshot found; this deployment will create a full first snapshot."
|
||||||
|
}
|
||||||
|
|
||||||
|
create_predeploy_backup() {
|
||||||
|
local container_exists="false"
|
||||||
|
local container_running="false"
|
||||||
|
local release_id=""
|
||||||
|
local backup_link_args=()
|
||||||
|
local current_image_id=""
|
||||||
|
local existing_rollback_id=""
|
||||||
|
|
||||||
|
if docker ps -a --format '{{.Names}}' | grep -Fxq geointel; then
|
||||||
|
container_exists="true"
|
||||||
|
fi
|
||||||
|
if [ "$(docker inspect -f '{{.State.Running}}' geointel 2>/dev/null || true)" = "true" ]; then
|
||||||
|
container_running="true"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$container_exists" = "false" ]; then
|
||||||
|
if [ -f "$GEOINTEL_POSTGIS_DATA_PATH/PG_VERSION" ]; then
|
||||||
|
echo "PostGIS data exists without a running GeoIntel container; refusing an unbacked migration." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "No existing GeoIntel state found; pre-deploy backup is not required for this fresh install."
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [ "$container_running" != "true" ]; then
|
||||||
|
echo "Existing GeoIntel container is not running; refusing deployment because a consistent backup cannot be created." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
current_image_id="$(docker inspect --format '{{.Image}}' geointel)"
|
||||||
|
if ! [[ "$current_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
|
||||||
|
echo "Running release does not expose one immutable Docker image ID." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
release_id="predeploy-${GEOINTEL_RELEASE_TOKEN:0:24}-$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
||||||
|
GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"
|
||||||
|
existing_rollback_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" 2>/dev/null || true)"
|
||||||
|
if [ -n "$existing_rollback_id" ] && [ "$existing_rollback_id" != "$current_image_id" ]; then
|
||||||
|
echo "Backup-specific rollback tag already identifies different image bytes." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"
|
||||||
|
|
||||||
|
# This conservative full-copy fallback estimate runs while the existing
|
||||||
|
# release is still healthy. Verified backup-to-backup hardlinks normally
|
||||||
|
# avoid recopying unchanged bytes, but are never assumed for this fail-closed
|
||||||
|
# capacity decision.
|
||||||
|
preflight_backup_capacity
|
||||||
|
select_verified_link_dest
|
||||||
|
if [ -n "$GEOINTEL_BACKUP_LINK_DEST" ]; then
|
||||||
|
backup_link_args=(--link-dest-backup "$GEOINTEL_BACKUP_LINK_DEST")
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Quiescing the current backend so the rollback point cannot miss concurrent writes..."
|
||||||
|
if ! docker exec -i geointel python - <<'PY'
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import signal
|
||||||
|
import time
|
||||||
|
|
||||||
|
matches = []
|
||||||
|
for item in pathlib.Path("/proc").iterdir():
|
||||||
|
if not item.name.isdigit() or int(item.name) in {os.getpid(), os.getppid()}:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
command = (item / "cmdline").read_bytes().replace(b"\0", b" ")
|
||||||
|
except (OSError, PermissionError):
|
||||||
|
continue
|
||||||
|
if b"uvicorn" in command and b"app.main:app" in command:
|
||||||
|
matches.append(int(item.name))
|
||||||
|
if not matches:
|
||||||
|
raise SystemExit("Could not identify the running GeoIntel backend")
|
||||||
|
for process_id in matches:
|
||||||
|
os.kill(process_id, signal.SIGTERM)
|
||||||
|
deadline = time.monotonic() + 60
|
||||||
|
remaining = matches
|
||||||
|
while remaining and time.monotonic() < deadline:
|
||||||
|
time.sleep(0.25)
|
||||||
|
remaining = [process_id for process_id in remaining if pathlib.Path(f"/proc/{process_id}").exists()]
|
||||||
|
if remaining:
|
||||||
|
raise SystemExit(f"Backend did not stop cleanly: {remaining}")
|
||||||
|
print(f"Stopped {len(matches)} backend process(es)")
|
||||||
|
PY
|
||||||
|
then
|
||||||
|
echo "Could not quiesce the current backend; refusing a potentially inconsistent backup." >&2
|
||||||
|
docker restart geointel >/dev/null || true
|
||||||
|
wait_for_geointel_health || true
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}"
|
||||||
|
echo "Creating mandatory pre-deploy backup ${release_id}..."
|
||||||
|
if ! bash scripts/backup_release_state.sh \
|
||||||
|
--container geointel \
|
||||||
|
--output-root "$GEOINTEL_BACKUPS_PATH" \
|
||||||
|
--release-id "$release_id" \
|
||||||
|
--storage-path "$GEOINTEL_STORAGE_PATH" \
|
||||||
|
--models-path "$GEOINTEL_MODELS_PATH" \
|
||||||
|
--inventory-mode sha256 \
|
||||||
|
--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \
|
||||||
|
"${backup_link_args[@]}" \
|
||||||
|
|| ! bash scripts/verify_release_backup.sh \
|
||||||
|
--container geointel \
|
||||||
|
--backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then
|
||||||
|
echo "Pre-deploy backup failed; restarting the unchanged current release." >&2
|
||||||
|
docker restart geointel >/dev/null || true
|
||||||
|
wait_for_geointel_health || true
|
||||||
|
GEOINTEL_PREDEPLOY_BACKUP_DIR=""
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "Pre-deploy backup verified: ${GEOINTEL_PREDEPLOY_BACKUP_DIR}"
|
||||||
}
|
}
|
||||||
|
|
||||||
rollback_previous() {
|
rollback_previous() {
|
||||||
if ! docker image inspect "$GEOINTEL_PREVIOUS_IMAGE" >/dev/null 2>&1; then
|
if [ -z "$GEOINTEL_PREDEPLOY_BACKUP_DIR" ]; then
|
||||||
echo "Automatic rollback unavailable: ${GEOINTEL_PREVIOUS_IMAGE} does not exist." >&2
|
echo "Automatic rollback unavailable: no verified pre-deploy database backup was created." >&2
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
echo "Rolling back to ${GEOINTEL_PREVIOUS_IMAGE}..."
|
echo "Rolling back database and image to the verified pre-deploy state..."
|
||||||
start_image "$GEOINTEL_PREVIOUS_IMAGE"
|
GEOINTEL_DEPLOY_LOCK_HELD=true \
|
||||||
|
bash deploy/unraid/rollback-dockerman-container.sh \
|
||||||
|
--backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR" \
|
||||||
|
--confirm-production-database-restore
|
||||||
}
|
}
|
||||||
|
|
||||||
docker compose -f docker-compose.unraid.yml config >/dev/null
|
docker compose -f docker-compose.unraid.yml config >/dev/null
|
||||||
|
|
||||||
current_image_id="$(docker inspect --format '{{.Image}}' geointel 2>/dev/null || true)"
|
|
||||||
release_image_id="$(
|
|
||||||
docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE" 2>/dev/null || true
|
|
||||||
)"
|
|
||||||
if (
|
|
||||||
[ -n "$current_image_id" ] &&
|
|
||||||
[ "$current_image_id" != "$release_image_id" ] &&
|
|
||||||
docker image inspect "$current_image_id" >/dev/null 2>&1
|
|
||||||
); then
|
|
||||||
docker tag "$current_image_id" "$GEOINTEL_PREVIOUS_IMAGE"
|
|
||||||
elif [ -n "$current_image_id" ] && [ "$current_image_id" = "$release_image_id" ]; then
|
|
||||||
echo "Current container already uses ${GEOINTEL_RELEASE_IMAGE}; preserving the existing previous image."
|
|
||||||
fi
|
|
||||||
|
|
||||||
if docker image inspect "$GEOINTEL_RELEASE_IMAGE" >/dev/null 2>&1; then
|
if docker image inspect "$GEOINTEL_RELEASE_IMAGE" >/dev/null 2>&1; then
|
||||||
stored_revision="$(
|
stored_revision="$(
|
||||||
docker image inspect \
|
docker image inspect \
|
||||||
@@ -208,7 +555,10 @@ else
|
|||||||
.
|
.
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! start_image "$GEOINTEL_RELEASE_IMAGE"; then
|
scan_release_image
|
||||||
|
create_predeploy_backup
|
||||||
|
|
||||||
|
if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"; then
|
||||||
rollback_previous || true
|
rollback_previous || true
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -32,11 +32,17 @@
|
|||||||
<Config Name="Postgres Password" Target="GEOINTEL_POSTGRES_PASSWORD" Default="change-me-before-shared-use" Mode="" Description="Embedded PostGIS database password. Change before shared use." Type="Variable" Display="advanced" Required="true" Mask="true">change-me-before-shared-use</Config>
|
<Config Name="Postgres Password" Target="GEOINTEL_POSTGRES_PASSWORD" Default="change-me-before-shared-use" Mode="" Description="Embedded PostGIS database password. Change before shared use." Type="Variable" Display="advanced" Required="true" Mask="true">change-me-before-shared-use</Config>
|
||||||
<Config Name="CORS Origins" Target="GEOINTEL_CORS_ORIGINS" Default="http://localhost:1202,http://127.0.0.1:1202,http://192.168.10.150:1202" Mode="" Description="Comma-separated browser origins allowed to call the backend directly." Type="Variable" Display="advanced" Required="false" Mask="false">http://localhost:1202,http://127.0.0.1:1202,http://192.168.10.150:1202</Config>
|
<Config Name="CORS Origins" Target="GEOINTEL_CORS_ORIGINS" Default="http://localhost:1202,http://127.0.0.1:1202,http://192.168.10.150:1202" Mode="" Description="Comma-separated browser origins allowed to call the backend directly." Type="Variable" Display="advanced" Required="false" Mask="false">http://localhost:1202,http://127.0.0.1:1202,http://192.168.10.150:1202</Config>
|
||||||
<Config Name="Max Upload MB" Target="GEOINTEL_MAX_UPLOAD_MB" Default="500" Mode="" Description="Maximum upload size in MiB enforced consistently by nginx and the backend (1-2048)." Type="Variable" Display="advanced" Required="true" Mask="false">500</Config>
|
<Config Name="Max Upload MB" Target="GEOINTEL_MAX_UPLOAD_MB" Default="500" Mode="" Description="Maximum upload size in MiB enforced consistently by nginx and the backend (1-2048)." Type="Variable" Display="advanced" Required="true" Mask="false">500</Config>
|
||||||
|
<Config Name="Max In-memory Vector MB" Target="GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" Default="64" Mode="" Description="Maximum decompressed vector payload processed fully in memory by the backend (1-256 MiB)." Type="Variable" Display="advanced" Required="true" Mask="false">64</Config>
|
||||||
<Config Name="Operator Login Enabled" Target="GEOINTEL_AUTH_ENABLED" Default="false" Mode="" Description="Require the single configured operator login before the browser may access workbench APIs." Type="Variable" Display="advanced" Required="true" Mask="false">false</Config>
|
<Config Name="Operator Login Enabled" Target="GEOINTEL_AUTH_ENABLED" Default="false" Mode="" Description="Require the single configured operator login before the browser may access workbench APIs." Type="Variable" Display="advanced" Required="true" Mask="false">false</Config>
|
||||||
<Config Name="Operator Username" Target="GEOINTEL_AUTH_USERNAME" Default="" Mode="" Description="Exact username for the single operator account." Type="Variable" Display="advanced" Required="false" Mask="false"></Config>
|
<Config Name="Operator Username" Target="GEOINTEL_AUTH_USERNAME" Default="" Mode="" Description="Exact username for the single operator account." Type="Variable" Display="advanced" Required="false" Mask="false"></Config>
|
||||||
<Config Name="Operator Password Hash" Target="GEOINTEL_AUTH_PASSWORD_HASH" Default="" Mode="" Description="PBKDF2-SHA256 password hash. Never enter a plaintext password." Type="Variable" Display="advanced" Required="false" Mask="true"></Config>
|
<Config Name="Operator Password Hash" Target="GEOINTEL_AUTH_PASSWORD_HASH" Default="" Mode="" Description="PBKDF2-SHA256 password hash. Never enter a plaintext password." Type="Variable" Display="advanced" Required="false" Mask="true"></Config>
|
||||||
<Config Name="Operator Session Secret" Target="GEOINTEL_AUTH_SESSION_SECRET" Default="" Mode="" Description="Random secret of at least 32 characters used only to sign browser sessions." Type="Variable" Display="advanced" Required="false" Mask="true"></Config>
|
<Config Name="Operator Session Secret" Target="GEOINTEL_AUTH_SESSION_SECRET" Default="" Mode="" Description="Random secret of at least 32 characters used only to sign browser sessions." Type="Variable" Display="advanced" Required="false" Mask="true"></Config>
|
||||||
<Config Name="Operator Session TTL" Target="GEOINTEL_AUTH_SESSION_TTL_SECONDS" Default="43200" Mode="" Description="Session lifetime in seconds (900-604800)." Type="Variable" Display="advanced" Required="true" Mask="false">43200</Config>
|
<Config Name="Operator Session TTL" Target="GEOINTEL_AUTH_SESSION_TTL_SECONDS" Default="43200" Mode="" Description="Session lifetime in seconds (900-604800)." Type="Variable" Display="advanced" Required="true" Mask="false">43200</Config>
|
||||||
|
<Config Name="Public Base URL" Target="GEOINTEL_PUBLIC_BASE_URL" Default="http://localhost:1202" Mode="" Description="Public browser origin. Must be HTTPS when Authentik is configured." Type="Variable" Display="advanced" Required="true" Mask="false">http://localhost:1202</Config>
|
||||||
|
<Config Name="Authentik Issuer" Target="GEOINTEL_AUTHENTIK_ISSUER" Default="" Mode="" Description="Optional HTTPS OIDC issuer. Configure all Authentik values together." Type="Variable" Display="advanced" Required="false" Mask="false"></Config>
|
||||||
|
<Config Name="Authentik Client ID" Target="GEOINTEL_AUTHENTIK_CLIENT_ID" Default="" Mode="" Description="OIDC client identifier for GeoIntel." Type="Variable" Display="advanced" Required="false" Mask="false"></Config>
|
||||||
|
<Config Name="Authentik Client Secret" Target="GEOINTEL_AUTHENTIK_CLIENT_SECRET" Default="" Mode="" Description="OIDC client secret; never commit this value." Type="Variable" Display="advanced" Required="false" Mask="true"></Config>
|
||||||
|
<Config Name="Authentik Allowed Email" Target="GEOINTEL_AUTHENTIK_ALLOWED_EMAIL" Default="" Mode="" Description="Exact verified operator e-mail address allowed to create an operator session." Type="Variable" Display="advanced" Required="false" Mask="false"></Config>
|
||||||
<Config Name="Guest Demo Enabled" Target="GEOINTEL_GUEST_ACCESS_ENABLED" Default="true" Mode="" Description="Show a guest button that opens only the seeded, restricted demo workspace. Enabled by default when operator login is active; set false on non-demo instances." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
<Config Name="Guest Demo Enabled" Target="GEOINTEL_GUEST_ACCESS_ENABLED" Default="true" Mode="" Description="Show a guest button that opens only the seeded, restricted demo workspace. Enabled by default when operator login is active; set false on non-demo instances." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||||
<Config Name="Guest Display Name" Target="GEOINTEL_GUEST_DISPLAY_NAME" Default="Gast" Mode="" Description="Label shown for the temporary guest session." Type="Variable" Display="advanced" Required="true" Mask="false">Gast</Config>
|
<Config Name="Guest Display Name" Target="GEOINTEL_GUEST_DISPLAY_NAME" Default="Gast" Mode="" Description="Label shown for the temporary guest session." Type="Variable" Display="advanced" Required="true" Mask="false">Gast</Config>
|
||||||
<Config Name="Guest Session TTL" Target="GEOINTEL_GUEST_SESSION_TTL_SECONDS" Default="7200" Mode="" Description="Temporary guest session lifetime in seconds (900-86400)." Type="Variable" Display="advanced" Required="true" Mask="false">7200</Config>
|
<Config Name="Guest Session TTL" Target="GEOINTEL_GUEST_SESSION_TTL_SECONDS" Default="7200" Mode="" Description="Temporary guest session lifetime in seconds (900-86400)." Type="Variable" Display="advanced" Required="true" Mask="false">7200</Config>
|
||||||
|
|||||||
@@ -34,6 +34,8 @@ GEOINTEL_CORS_ORIGINS=https://geointel.itworx.tech,http://geointel.itworx.tech,h
|
|||||||
|
|
||||||
# Upload guard in MiB. The same 1-2048 limit is applied by nginx and FastAPI.
|
# Upload guard in MiB. The same 1-2048 limit is applied by nginx and FastAPI.
|
||||||
GEOINTEL_MAX_UPLOAD_MB=500
|
GEOINTEL_MAX_UPLOAD_MB=500
|
||||||
|
# Maximum decompressed vector payload processed fully in memory (1-256 MiB).
|
||||||
|
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB=64
|
||||||
GEOINTEL_AOI_WORKER_ENABLED=true
|
GEOINTEL_AOI_WORKER_ENABLED=true
|
||||||
GEOINTEL_AOI_WORKER_POLL_SECONDS=2
|
GEOINTEL_AOI_WORKER_POLL_SECONDS=2
|
||||||
|
|
||||||
@@ -46,6 +48,14 @@ GEOINTEL_AUTH_PASSWORD_HASH=
|
|||||||
GEOINTEL_AUTH_SESSION_SECRET=
|
GEOINTEL_AUTH_SESSION_SECRET=
|
||||||
GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200
|
GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200
|
||||||
|
|
||||||
|
# Optional additive Authentik OIDC login. Configure all fields together. The
|
||||||
|
# local operator credentials above remain the recovery login.
|
||||||
|
GEOINTEL_PUBLIC_BASE_URL=http://localhost:1202
|
||||||
|
GEOINTEL_AUTHENTIK_ISSUER=
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_ID=
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_SECRET=
|
||||||
|
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL=
|
||||||
|
|
||||||
# Guest access is enabled by default whenever operator authentication is active.
|
# Guest access is enabled by default whenever operator authentication is active.
|
||||||
# It opens the seeded GeoIntel demo in a temporary, API-enforced restricted
|
# It opens the seeded GeoIntel demo in a temporary, API-enforced restricted
|
||||||
# session. Set this to false on installations containing private project data.
|
# session. Set this to false on installations containing private project data.
|
||||||
@@ -153,8 +163,9 @@ SPW_TERRAIN_ANALYSIS_RESOLUTION_M=5
|
|||||||
SPW_TERRAIN_MAX_SIDE_M=20000
|
SPW_TERRAIN_MAX_SIDE_M=20000
|
||||||
SPW_TERRAIN_MAX_PIXELS=12000000
|
SPW_TERRAIN_MAX_PIXELS=12000000
|
||||||
|
|
||||||
# Optional configured-YOLO runtime. Keep disabled unless a local model is mounted.
|
# Production releases always contain the pinned AI dependencies. Inference may
|
||||||
GEOINTEL_INSTALL_AI=false
|
# remain disabled until an integrity-bound local model is mounted.
|
||||||
|
GEOINTEL_INSTALL_AI=true
|
||||||
YOLO_ENABLED=false
|
YOLO_ENABLED=false
|
||||||
YOLO_MODELS_DIR=/app/models
|
YOLO_MODELS_DIR=/app/models
|
||||||
YOLO_MODEL_PATH=
|
YOLO_MODEL_PATH=
|
||||||
|
|||||||
@@ -1,3 +1,18 @@
|
|||||||
|
geo $geointel_trusted_forwarder {
|
||||||
|
default 0;
|
||||||
|
127.0.0.0/8 1;
|
||||||
|
::1/128 1;
|
||||||
|
# The outer Nginx Proxy Manager reaches this container through Docker's
|
||||||
|
# internal bridge; public/LAN clients are not trusted forwarders.
|
||||||
|
172.16.0.0/12 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
map "$geointel_trusted_forwarder:$http_x_forwarded_proto" $geointel_forwarded_proto {
|
||||||
|
default $scheme;
|
||||||
|
"1:https" https;
|
||||||
|
"1:http" http;
|
||||||
|
}
|
||||||
|
|
||||||
server {
|
server {
|
||||||
listen 80;
|
listen 80;
|
||||||
server_name _;
|
server_name _;
|
||||||
@@ -5,21 +20,42 @@ server {
|
|||||||
proxy_read_timeout 600s;
|
proxy_read_timeout 600s;
|
||||||
proxy_send_timeout 600s;
|
proxy_send_timeout 600s;
|
||||||
|
|
||||||
|
add_header Content-Security-Policy "frame-ancestors 'none'" always;
|
||||||
|
add_header X-Frame-Options "DENY" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
|
||||||
|
|
||||||
root /usr/share/nginx/html;
|
root /usr/share/nginx/html;
|
||||||
index index.html;
|
index index.html;
|
||||||
|
|
||||||
location = /index.html {
|
location = /index.html {
|
||||||
add_header Cache-Control "no-cache";
|
add_header Cache-Control "no-cache";
|
||||||
|
add_header Content-Security-Policy "frame-ancestors 'none'" always;
|
||||||
|
add_header X-Frame-Options "DENY" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
|
||||||
try_files /index.html =404;
|
try_files /index.html =404;
|
||||||
}
|
}
|
||||||
|
|
||||||
location = /geointel-icon.svg {
|
location = /geointel-icon.svg {
|
||||||
add_header Cache-Control "public, max-age=3600";
|
add_header Cache-Control "public, max-age=3600";
|
||||||
|
add_header Content-Security-Policy "frame-ancestors 'none'" always;
|
||||||
|
add_header X-Frame-Options "DENY" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
|
||||||
try_files /geointel-icon.svg =404;
|
try_files /geointel-icon.svg =404;
|
||||||
}
|
}
|
||||||
|
|
||||||
location = /geointel-icon.png {
|
location = /geointel-icon.png {
|
||||||
add_header Cache-Control "public, max-age=3600";
|
add_header Cache-Control "public, max-age=3600";
|
||||||
|
add_header Content-Security-Policy "frame-ancestors 'none'" always;
|
||||||
|
add_header X-Frame-Options "DENY" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
|
||||||
# Preserve the stable DockerMan/public URL while the frontend keeps
|
# Preserve the stable DockerMan/public URL while the frontend keeps
|
||||||
# its explicit 32px and 180px icon variants.
|
# its explicit 32px and 180px icon variants.
|
||||||
try_files /geointel-icon-180.png =404;
|
try_files /geointel-icon-180.png =404;
|
||||||
@@ -27,6 +63,11 @@ server {
|
|||||||
|
|
||||||
location /assets/ {
|
location /assets/ {
|
||||||
add_header Cache-Control "no-cache";
|
add_header Cache-Control "no-cache";
|
||||||
|
add_header Content-Security-Policy "frame-ancestors 'none'" always;
|
||||||
|
add_header X-Frame-Options "DENY" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
|
||||||
try_files $uri =404;
|
try_files $uri =404;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -36,7 +77,7 @@ server {
|
|||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;
|
||||||
}
|
}
|
||||||
|
|
||||||
location = /health {
|
location = /health {
|
||||||
@@ -45,7 +86,7 @@ server {
|
|||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;
|
||||||
}
|
}
|
||||||
|
|
||||||
location = /health/live {
|
location = /health/live {
|
||||||
|
|||||||
@@ -0,0 +1,323 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Restore the production database from a verified pre-deploy dump while the
|
||||||
|
# normal GeoIntel container is stopped. This is intentionally a separate,
|
||||||
|
# explicitly confirmed operation: starting an older image against a schema
|
||||||
|
# migrated by a newer image is not a safe rollback strategy.
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
GEOINTEL_DEPLOY_LOCK_FILE="${GEOINTEL_DEPLOY_LOCK_FILE:-/tmp/geointel-release-deploy.lock}"
|
||||||
|
if [ "${GEOINTEL_DEPLOY_LOCK_HELD:-false}" != "true" ]; then
|
||||||
|
command -v flock >/dev/null 2>&1 || {
|
||||||
|
echo "GeoIntel database restore requires flock to prevent concurrent deployment." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
exec 9>"$GEOINTEL_DEPLOY_LOCK_FILE"
|
||||||
|
if ! flock -n 9; then
|
||||||
|
echo "Another GeoIntel deployment or rollback is already running." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
BACKUP_DIR=""
|
||||||
|
CONFIRMED="false"
|
||||||
|
RESTORE_IMAGE="${GEOINTEL_ROLLBACK_IMAGE:-}"
|
||||||
|
GEOINTEL_CONTAINER_NAME="${GEOINTEL_CONTAINER_NAME:-geointel}"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: bash deploy/unraid/restore-predeploy-database.sh \
|
||||||
|
--backup-dir PATH --confirm-production-database-restore [options]
|
||||||
|
|
||||||
|
Stops the normal GeoIntel container, starts an isolated PostGIS recovery
|
||||||
|
container on the same persistent database path, restores the checksum-verified
|
||||||
|
custom-format dump, validates Alembic/table counts, and stops recovery again.
|
||||||
|
The caller must start the rollback image after this command succeeds.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--image IMAGE Recovery image containing PostgreSQL/PostGIS tools
|
||||||
|
--container NAME Normal application container (default: geointel)
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
while [ "$#" -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--backup-dir) BACKUP_DIR="$2"; shift 2 ;;
|
||||||
|
--confirm-production-database-restore) CONFIRMED="true"; shift ;;
|
||||||
|
--image) RESTORE_IMAGE="$2"; shift 2 ;;
|
||||||
|
--container) GEOINTEL_CONTAINER_NAME="$2"; shift 2 ;;
|
||||||
|
--help|-h) usage; exit 0 ;;
|
||||||
|
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$CONFIRMED" != "true" ] || [ -z "$BACKUP_DIR" ]; then
|
||||||
|
echo "Explicit --confirm-production-database-restore and --backup-dir are required." >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
for required in docker python3 sha256sum; do
|
||||||
|
command -v "$required" >/dev/null 2>&1 || {
|
||||||
|
echo "Missing required command: $required" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -f .env ]; then
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
. ./.env
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
|
|
||||||
|
GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
|
||||||
|
GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}"
|
||||||
|
GEOINTEL_POSTGRES_DB="${GEOINTEL_POSTGRES_DB:-geointel}"
|
||||||
|
GEOINTEL_POSTGRES_USER="${GEOINTEL_POSTGRES_USER:-geointel}"
|
||||||
|
GEOINTEL_POSTGRES_PASSWORD="${GEOINTEL_POSTGRES_PASSWORD:-}"
|
||||||
|
|
||||||
|
if ! [[ "$GEOINTEL_POSTGRES_DB" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] \
|
||||||
|
|| ! [[ "$GEOINTEL_POSTGRES_USER" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
|
||||||
|
echo "Configured PostGIS database and user names must be simple SQL identifiers." >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
case "$GEOINTEL_POSTGRES_PASSWORD" in
|
||||||
|
''|geointel|postgres|password|changeme|change-me-before-shared-use)
|
||||||
|
echo "Refusing database restore with an empty or known-default PostGIS password." >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
test -f "$GEOINTEL_POSTGIS_DATA_PATH/PG_VERSION" || {
|
||||||
|
echo "Persistent PostGIS data path is not initialized: $GEOINTEL_POSTGIS_DATA_PATH" >&2
|
||||||
|
exit 3
|
||||||
|
}
|
||||||
|
|
||||||
|
GEOINTEL_BACKUPS_PATH="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$GEOINTEL_BACKUPS_PATH")"
|
||||||
|
BACKUP_DIR="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$BACKUP_DIR")"
|
||||||
|
python3 - "$GEOINTEL_BACKUPS_PATH" "$BACKUP_DIR" <<'PY'
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
root = pathlib.Path(sys.argv[1])
|
||||||
|
backup = pathlib.Path(sys.argv[2])
|
||||||
|
try:
|
||||||
|
backup.relative_to(root)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise SystemExit(f"Backup directory must be below {root}") from exc
|
||||||
|
if backup == root:
|
||||||
|
raise SystemExit("Backup directory must identify one immutable backup")
|
||||||
|
PY
|
||||||
|
|
||||||
|
for required_file in manifest.json database.dump database.list database-metadata.tsv table-counts.tsv CHECKSUMS.sha256; do
|
||||||
|
test -s "$BACKUP_DIR/$required_file" || {
|
||||||
|
echo "Missing or empty backup artifact: $required_file" >&2
|
||||||
|
exit 3
|
||||||
|
}
|
||||||
|
done
|
||||||
|
(
|
||||||
|
cd "$BACKUP_DIR"
|
||||||
|
sha256sum -c CHECKSUMS.sha256
|
||||||
|
)
|
||||||
|
python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$BACKUP_DIR"
|
||||||
|
|
||||||
|
IFS=$'\t' read -r BACKUP_DB BACKUP_USER BACKUP_IMAGE_ID BACKUP_RELEASE_ID < <(
|
||||||
|
python3 - "$BACKUP_DIR/manifest.json" <<'PY'
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
payload = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))
|
||||||
|
if payload.get("schema_version") != 1 or payload.get("read_only_source") is not True:
|
||||||
|
raise SystemExit("Unsupported or unsafe backup manifest")
|
||||||
|
print(
|
||||||
|
f"{payload.get('database_name', '')}\t{payload.get('database_user', '')}\t"
|
||||||
|
f"{payload.get('image_id', '')}\t{payload.get('release_id', '')}"
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
if [ "$BACKUP_DB" != "$GEOINTEL_POSTGRES_DB" ] || [ "$BACKUP_USER" != "$GEOINTEL_POSTGRES_USER" ]; then
|
||||||
|
echo "Backup database identity does not match the configured production database." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
if ! [[ "$BACKUP_IMAGE_ID" =~ ^sha256:[0-9a-f]{64}$ ]]; then
|
||||||
|
echo "Backup manifest does not contain one immutable Docker image ID." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
if [ -z "$RESTORE_IMAGE" ]; then
|
||||||
|
RESTORE_IMAGE="$BACKUP_IMAGE_ID"
|
||||||
|
fi
|
||||||
|
docker image inspect "$RESTORE_IMAGE" >/dev/null
|
||||||
|
RESTORE_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$RESTORE_IMAGE")"
|
||||||
|
if [ -z "$BACKUP_IMAGE_ID" ] || [ "$BACKUP_IMAGE_ID" != "$RESTORE_IMAGE_ID" ]; then
|
||||||
|
echo "Backup image identity does not match the retained rollback image." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
case "$BACKUP_RELEASE_ID" in
|
||||||
|
predeploy-*) ;;
|
||||||
|
*) echo "Production rollback requires a predeploy backup." >&2; exit 3 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
if docker ps -a --format '{{.Names}}' | grep -Fxq "$GEOINTEL_CONTAINER_NAME"; then
|
||||||
|
docker rm -f "$GEOINTEL_CONTAINER_NAME" >/dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
RECOVERY_CONTAINER="geointel-db-restore-$(date -u +%Y%m%d%H%M%S)-$$"
|
||||||
|
RESTORE_PROOF_DB="geointel_restore_proof_$(date -u +%Y%m%d%H%M%S)_$$"
|
||||||
|
RECOVERY_DB="geointel_pre_restore_$(date -u +%Y%m%d%H%M%S)_$$"
|
||||||
|
FAILED_RESTORE_DB="geointel_failed_restore_$(date -u +%Y%m%d%H%M%S)_$$"
|
||||||
|
SWAP_COMPLETE="false"
|
||||||
|
cleanup_recovery() {
|
||||||
|
if [ "$SWAP_COMPLETE" != "true" ] \
|
||||||
|
&& [ "$(docker inspect -f '{{.State.Running}}' "$RECOVERY_CONTAINER" 2>/dev/null || true)" = "true" ]; then
|
||||||
|
docker exec "$RECOVERY_CONTAINER" dropdb --if-exists --force \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" "$RESTORE_PROOF_DB" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
docker rm -f "$RECOVERY_CONTAINER" >/dev/null 2>&1 || true
|
||||||
|
}
|
||||||
|
trap cleanup_recovery EXIT
|
||||||
|
|
||||||
|
docker run -d \
|
||||||
|
--name "$RECOVERY_CONTAINER" \
|
||||||
|
--restart no \
|
||||||
|
-e PGDATA=/var/lib/postgresql/data \
|
||||||
|
-e PGPASSWORD="$GEOINTEL_POSTGRES_PASSWORD" \
|
||||||
|
-v "$GEOINTEL_POSTGIS_DATA_PATH:/var/lib/postgresql/data" \
|
||||||
|
-v "$BACKUP_DIR:/restore:ro" \
|
||||||
|
--entrypoint /bin/bash \
|
||||||
|
"$RESTORE_IMAGE" \
|
||||||
|
-c 'set -euo pipefail; chown postgres:postgres "$PGDATA"; exec gosu postgres postgres' \
|
||||||
|
>/dev/null
|
||||||
|
|
||||||
|
for attempt in $(seq 1 180); do
|
||||||
|
if docker exec "$RECOVERY_CONTAINER" pg_isready -h 127.0.0.1 -U "$GEOINTEL_POSTGRES_USER" -d postgres >/dev/null 2>&1; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if [ "$(docker inspect -f '{{.State.Running}}' "$RECOVERY_CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||||
|
echo "Database recovery container exited before PostGIS became ready." >&2
|
||||||
|
docker logs "$RECOVERY_CONTAINER" >&2 || true
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
|
if [ "$attempt" -eq 180 ]; then
|
||||||
|
echo "PostGIS recovery did not become ready within six minutes." >&2
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
RESTORED_LIST="$(mktemp)"
|
||||||
|
trap 'rm -f -- "$RESTORED_LIST"; cleanup_recovery' EXIT
|
||||||
|
docker exec "$RECOVERY_CONTAINER" pg_restore --list /restore/database.dump > "$RESTORED_LIST"
|
||||||
|
cmp -s "$RESTORED_LIST" "$BACKUP_DIR/database.list" || {
|
||||||
|
echo "Recovery image reads a different PostgreSQL archive listing." >&2
|
||||||
|
exit 4
|
||||||
|
}
|
||||||
|
|
||||||
|
if ! docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" -d postgres -Atqc \
|
||||||
|
"SELECT 1 FROM pg_database WHERE datname = '${GEOINTEL_POSTGRES_DB}';" | grep -Fxq 1; then
|
||||||
|
echo "Configured production database does not exist; refusing replacement." >&2
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
|
for generated_database in "$RESTORE_PROOF_DB" "$RECOVERY_DB" "$FAILED_RESTORE_DB"; do
|
||||||
|
if docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" -d postgres -Atqc \
|
||||||
|
"SELECT 1 FROM pg_database WHERE datname = '${generated_database}';" | grep -Fxq 1; then
|
||||||
|
echo "Generated recovery database already exists: ${generated_database}" >&2
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Prove the complete archive in a separate database before touching production.
|
||||||
|
docker exec "$RECOVERY_CONTAINER" createdb \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" "$RESTORE_PROOF_DB"
|
||||||
|
docker exec "$RECOVERY_CONTAINER" pg_restore \
|
||||||
|
--exit-on-error \
|
||||||
|
--no-owner \
|
||||||
|
--no-privileges \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" \
|
||||||
|
-d "$RESTORE_PROOF_DB" \
|
||||||
|
/restore/database.dump
|
||||||
|
|
||||||
|
EXPECTED_HEAD="$(awk -F $'\t' '$1 == "alembic_head" { print $2 }' "$BACKUP_DIR/database-metadata.tsv")"
|
||||||
|
validate_restored_database() {
|
||||||
|
local database_name="$1"
|
||||||
|
local restored_head=""
|
||||||
|
restored_head="$(docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" -d "$database_name" -Atqc \
|
||||||
|
'SELECT version_num FROM alembic_version;')"
|
||||||
|
if [ -z "$EXPECTED_HEAD" ] || [ "$restored_head" != "$EXPECTED_HEAD" ]; then
|
||||||
|
echo "Restored Alembic head '$restored_head' differs from backup head '$EXPECTED_HEAD'." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
while IFS=$'\t' read -r table expected; do
|
||||||
|
[[ "$table" =~ ^[a-z_]+$ ]] || {
|
||||||
|
echo "Unsafe table name in retained counts: $table" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
actual="$(docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" -d "$database_name" -Atqc \
|
||||||
|
"SELECT count(*) FROM public.${table};")"
|
||||||
|
if [ "$actual" != "$expected" ]; then
|
||||||
|
echo "Restored count mismatch for $table: expected $expected, got $actual." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
done < "$BACKUP_DIR/table-counts.tsv"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_restored_database "$RESTORE_PROOF_DB"
|
||||||
|
echo "Isolated predeploy restore proof passed: ${RESTORE_PROOF_DB}"
|
||||||
|
|
||||||
|
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
|
||||||
|
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IN ('${GEOINTEL_POSTGRES_DB}', '${RESTORE_PROOF_DB}') AND pid <> pg_backend_pid();" \
|
||||||
|
>/dev/null
|
||||||
|
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
|
||||||
|
"ALTER DATABASE ${GEOINTEL_POSTGRES_DB} RENAME TO ${RECOVERY_DB};"
|
||||||
|
if ! docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
|
||||||
|
"ALTER DATABASE ${RESTORE_PROOF_DB} RENAME TO ${GEOINTEL_POSTGRES_DB};"; then
|
||||||
|
echo "Restored database cutover failed; restoring the untouched production database name." >&2
|
||||||
|
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
|
||||||
|
"ALTER DATABASE ${RECOVERY_DB} RENAME TO ${GEOINTEL_POSTGRES_DB};"
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
|
SWAP_COMPLETE="true"
|
||||||
|
|
||||||
|
if ! validate_restored_database "$GEOINTEL_POSTGRES_DB"; then
|
||||||
|
echo "Post-cutover validation failed; restoring the retained pre-restore database." >&2
|
||||||
|
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
|
||||||
|
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${GEOINTEL_POSTGRES_DB}' AND pid <> pg_backend_pid();" \
|
||||||
|
>/dev/null
|
||||||
|
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
|
||||||
|
"ALTER DATABASE ${GEOINTEL_POSTGRES_DB} RENAME TO ${FAILED_RESTORE_DB};"
|
||||||
|
docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \
|
||||||
|
-U "$GEOINTEL_POSTGRES_USER" -d postgres -c \
|
||||||
|
"ALTER DATABASE ${RECOVERY_DB} RENAME TO ${GEOINTEL_POSTGRES_DB};"
|
||||||
|
SWAP_COMPLETE="false"
|
||||||
|
echo "Original production database was restored; failed restore retained as ${FAILED_RESTORE_DB}." >&2
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "$RECOVERY_DB" ]; then
|
||||||
|
echo "Recovery database identity was not retained." >&2
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
|
|
||||||
|
while IFS=$'\t' read -r table expected; do
|
||||||
|
[[ "$table" =~ ^[a-z_]+$ ]] || {
|
||||||
|
echo "Unsafe table name in retained counts: $table" >&2
|
||||||
|
exit 4
|
||||||
|
}
|
||||||
|
done < "$BACKUP_DIR/table-counts.tsv"
|
||||||
|
|
||||||
|
rm -f -- "$RESTORED_LIST"
|
||||||
|
cleanup_recovery
|
||||||
|
trap - EXIT
|
||||||
|
echo "Production database restored and verified from: $BACKUP_DIR"
|
||||||
|
echo "Pre-restore production database retained for operator recovery as: $RECOVERY_DB"
|
||||||
@@ -4,14 +4,78 @@ set -euo pipefail
|
|||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
|
||||||
GEOINTEL_ROLLBACK_IMAGE="${GEOINTEL_ROLLBACK_IMAGE:-geointel-all-in-one:previous}"
|
GEOINTEL_DEPLOY_LOCK_FILE="${GEOINTEL_DEPLOY_LOCK_FILE:-/tmp/geointel-release-deploy.lock}"
|
||||||
|
if [ "${GEOINTEL_DEPLOY_LOCK_HELD:-false}" != "true" ]; then
|
||||||
|
command -v flock >/dev/null 2>&1 || {
|
||||||
|
echo "GeoIntel rollback requires flock to prevent concurrent deployment." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
exec 9>"$GEOINTEL_DEPLOY_LOCK_FILE"
|
||||||
|
if ! flock -n 9; then
|
||||||
|
echo "Another GeoIntel deployment or rollback is already running." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
GEOINTEL_DEPLOY_LOCK_HELD="true"
|
||||||
|
export GEOINTEL_DEPLOY_LOCK_HELD
|
||||||
|
fi
|
||||||
|
|
||||||
if ! docker image inspect "$GEOINTEL_ROLLBACK_IMAGE" >/dev/null 2>&1; then
|
GEOINTEL_ROLLBACK_IMAGE="${GEOINTEL_ROLLBACK_IMAGE:-}"
|
||||||
echo "Rollback image does not exist: ${GEOINTEL_ROLLBACK_IMAGE}" >&2
|
BACKUP_DIR="${GEOINTEL_ROLLBACK_BACKUP_DIR:-}"
|
||||||
|
CONFIRM_RESTORE="false"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: bash deploy/unraid/rollback-dockerman-container.sh \
|
||||||
|
--backup-dir PATH --confirm-production-database-restore
|
||||||
|
|
||||||
|
Restores the verified pre-deploy PostgreSQL dump first and only then starts the
|
||||||
|
retained previous image. Image-only rollback against an unknown migrated
|
||||||
|
schema is deliberately not supported.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
while [ "$#" -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--backup-dir) BACKUP_DIR="$2"; shift 2 ;;
|
||||||
|
--confirm-production-database-restore) CONFIRM_RESTORE="true"; shift ;;
|
||||||
|
--help|-h) usage; exit 0 ;;
|
||||||
|
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z "$BACKUP_DIR" ] || [ "$CONFIRM_RESTORE" != "true" ]; then
|
||||||
|
echo "Rollback requires a verified pre-deploy backup and explicit database-restore confirmation." >&2
|
||||||
|
usage >&2
|
||||||
exit 2
|
exit 2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "Starting rollback image ${GEOINTEL_ROLLBACK_IMAGE} without changing persistent volumes..."
|
restore_image_args=()
|
||||||
|
if [ -n "$GEOINTEL_ROLLBACK_IMAGE" ]; then
|
||||||
|
restore_image_args=(--image "$GEOINTEL_ROLLBACK_IMAGE")
|
||||||
|
fi
|
||||||
|
echo "Restoring the pre-deploy database before starting its checksum-bound image..."
|
||||||
|
bash deploy/unraid/restore-predeploy-database.sh \
|
||||||
|
--backup-dir "$BACKUP_DIR" \
|
||||||
|
"${restore_image_args[@]}" \
|
||||||
|
--confirm-production-database-restore
|
||||||
|
|
||||||
|
if [ -z "$GEOINTEL_ROLLBACK_IMAGE" ]; then
|
||||||
|
GEOINTEL_ROLLBACK_IMAGE="$(python3 - "$BACKUP_DIR/manifest.json" <<'PY'
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
image_id = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")).get("image_id", "")
|
||||||
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
||||||
|
raise SystemExit("Backup manifest lacks an immutable rollback image ID")
|
||||||
|
print(image_id)
|
||||||
|
PY
|
||||||
|
)"
|
||||||
|
fi
|
||||||
|
docker image inspect "$GEOINTEL_ROLLBACK_IMAGE" >/dev/null
|
||||||
|
|
||||||
|
echo "Starting rollback image ${GEOINTEL_ROLLBACK_IMAGE} with the restored persistent database..."
|
||||||
GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE" bash deploy/unraid/run-dockerman-container.sh
|
GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE" bash deploy/unraid/run-dockerman-container.sh
|
||||||
|
|
||||||
for attempt in $(seq 1 90); do
|
for attempt in $(seq 1 90); do
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ GEOINTEL_POSTGRES_USER="${GEOINTEL_POSTGRES_USER:-geointel}"
|
|||||||
GEOINTEL_POSTGRES_PASSWORD="${GEOINTEL_POSTGRES_PASSWORD:-}"
|
GEOINTEL_POSTGRES_PASSWORD="${GEOINTEL_POSTGRES_PASSWORD:-}"
|
||||||
GEOINTEL_CORS_ORIGINS="${GEOINTEL_CORS_ORIGINS:-http://localhost:${GEOINTEL_FRONTEND_PORT},http://127.0.0.1:${GEOINTEL_FRONTEND_PORT},http://192.168.10.150:${GEOINTEL_FRONTEND_PORT}}"
|
GEOINTEL_CORS_ORIGINS="${GEOINTEL_CORS_ORIGINS:-http://localhost:${GEOINTEL_FRONTEND_PORT},http://127.0.0.1:${GEOINTEL_FRONTEND_PORT},http://192.168.10.150:${GEOINTEL_FRONTEND_PORT}}"
|
||||||
GEOINTEL_MAX_UPLOAD_MB="${GEOINTEL_MAX_UPLOAD_MB:-500}"
|
GEOINTEL_MAX_UPLOAD_MB="${GEOINTEL_MAX_UPLOAD_MB:-500}"
|
||||||
|
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}"
|
||||||
GEOINTEL_AOI_WORKER_ENABLED="${GEOINTEL_AOI_WORKER_ENABLED:-true}"
|
GEOINTEL_AOI_WORKER_ENABLED="${GEOINTEL_AOI_WORKER_ENABLED:-true}"
|
||||||
GEOINTEL_AOI_WORKER_POLL_SECONDS="${GEOINTEL_AOI_WORKER_POLL_SECONDS:-2}"
|
GEOINTEL_AOI_WORKER_POLL_SECONDS="${GEOINTEL_AOI_WORKER_POLL_SECONDS:-2}"
|
||||||
GEOINTEL_AUTH_ENABLED="${GEOINTEL_AUTH_ENABLED:-false}"
|
GEOINTEL_AUTH_ENABLED="${GEOINTEL_AUTH_ENABLED:-false}"
|
||||||
@@ -40,6 +41,11 @@ GEOINTEL_AUTH_USERNAME="${GEOINTEL_AUTH_USERNAME:-}"
|
|||||||
GEOINTEL_AUTH_PASSWORD_HASH="${GEOINTEL_AUTH_PASSWORD_HASH:-}"
|
GEOINTEL_AUTH_PASSWORD_HASH="${GEOINTEL_AUTH_PASSWORD_HASH:-}"
|
||||||
GEOINTEL_AUTH_SESSION_SECRET="${GEOINTEL_AUTH_SESSION_SECRET:-}"
|
GEOINTEL_AUTH_SESSION_SECRET="${GEOINTEL_AUTH_SESSION_SECRET:-}"
|
||||||
GEOINTEL_AUTH_SESSION_TTL_SECONDS="${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200}"
|
GEOINTEL_AUTH_SESSION_TTL_SECONDS="${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200}"
|
||||||
|
GEOINTEL_PUBLIC_BASE_URL="${GEOINTEL_PUBLIC_BASE_URL:-http://localhost:${GEOINTEL_FRONTEND_PORT}}"
|
||||||
|
GEOINTEL_AUTHENTIK_ISSUER="${GEOINTEL_AUTHENTIK_ISSUER:-}"
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_ID="${GEOINTEL_AUTHENTIK_CLIENT_ID:-}"
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_SECRET="${GEOINTEL_AUTHENTIK_CLIENT_SECRET:-}"
|
||||||
|
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL="${GEOINTEL_AUTHENTIK_ALLOWED_EMAIL:-}"
|
||||||
GEOINTEL_GUEST_ACCESS_ENABLED="${GEOINTEL_GUEST_ACCESS_ENABLED:-true}"
|
GEOINTEL_GUEST_ACCESS_ENABLED="${GEOINTEL_GUEST_ACCESS_ENABLED:-true}"
|
||||||
GEOINTEL_GUEST_DISPLAY_NAME="${GEOINTEL_GUEST_DISPLAY_NAME:-Gast}"
|
GEOINTEL_GUEST_DISPLAY_NAME="${GEOINTEL_GUEST_DISPLAY_NAME:-Gast}"
|
||||||
GEOINTEL_GUEST_SESSION_TTL_SECONDS="${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200}"
|
GEOINTEL_GUEST_SESSION_TTL_SECONDS="${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200}"
|
||||||
@@ -189,6 +195,17 @@ validate_runtime_config() {
|
|||||||
return 2
|
return 2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
case "$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" in
|
||||||
|
''|*[!0-9]*)
|
||||||
|
echo "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be a whole number." >&2
|
||||||
|
return 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
if [ "$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" -lt 1 ] || [ "$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" -gt 256 ]; then
|
||||||
|
echo "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be between 1 and 256." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
|
||||||
case "$GEOINTEL_AUTH_ENABLED" in
|
case "$GEOINTEL_AUTH_ENABLED" in
|
||||||
true|false) ;;
|
true|false) ;;
|
||||||
*)
|
*)
|
||||||
@@ -212,6 +229,36 @@ validate_runtime_config() {
|
|||||||
esac
|
esac
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
local authentik_count=0
|
||||||
|
local authentik_value
|
||||||
|
for authentik_value in \
|
||||||
|
"$GEOINTEL_AUTHENTIK_ISSUER" \
|
||||||
|
"$GEOINTEL_AUTHENTIK_CLIENT_ID" \
|
||||||
|
"$GEOINTEL_AUTHENTIK_CLIENT_SECRET" \
|
||||||
|
"$GEOINTEL_AUTHENTIK_ALLOWED_EMAIL"; do
|
||||||
|
if [ -n "$authentik_value" ]; then
|
||||||
|
authentik_count=$((authentik_count + 1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ "$authentik_count" -ne 0 ] && [ "$authentik_count" -ne 4 ]; then
|
||||||
|
echo "All GEOINTEL_AUTHENTIK_* values must be configured together." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
if [ "$authentik_count" -eq 4 ]; then
|
||||||
|
if [ "$GEOINTEL_AUTH_ENABLED" != "true" ]; then
|
||||||
|
echo "GEOINTEL_AUTH_ENABLED must be true when Authentik is configured." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
case "$GEOINTEL_AUTHENTIK_ISSUER" in
|
||||||
|
https://*) ;;
|
||||||
|
*) echo "GEOINTEL_AUTHENTIK_ISSUER must use HTTPS." >&2; return 2 ;;
|
||||||
|
esac
|
||||||
|
case "$GEOINTEL_PUBLIC_BASE_URL" in
|
||||||
|
https://*) ;;
|
||||||
|
*) echo "GEOINTEL_PUBLIC_BASE_URL must use HTTPS for Authentik." >&2; return 2 ;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
case "$GEOINTEL_GUEST_ACCESS_ENABLED" in
|
case "$GEOINTEL_GUEST_ACCESS_ENABLED" in
|
||||||
true|false) ;;
|
true|false) ;;
|
||||||
*)
|
*)
|
||||||
@@ -314,6 +361,7 @@ docker run -d \
|
|||||||
-e GEOINTEL_STORAGE_ROOT=/app/storage \
|
-e GEOINTEL_STORAGE_ROOT=/app/storage \
|
||||||
-e GEOINTEL_CORS_ORIGINS="$GEOINTEL_CORS_ORIGINS" \
|
-e GEOINTEL_CORS_ORIGINS="$GEOINTEL_CORS_ORIGINS" \
|
||||||
-e GEOINTEL_MAX_UPLOAD_MB="$GEOINTEL_MAX_UPLOAD_MB" \
|
-e GEOINTEL_MAX_UPLOAD_MB="$GEOINTEL_MAX_UPLOAD_MB" \
|
||||||
|
-e GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" \
|
||||||
-e GEOINTEL_AOI_WORKER_ENABLED="$GEOINTEL_AOI_WORKER_ENABLED" \
|
-e GEOINTEL_AOI_WORKER_ENABLED="$GEOINTEL_AOI_WORKER_ENABLED" \
|
||||||
-e GEOINTEL_AOI_WORKER_POLL_SECONDS="$GEOINTEL_AOI_WORKER_POLL_SECONDS" \
|
-e GEOINTEL_AOI_WORKER_POLL_SECONDS="$GEOINTEL_AOI_WORKER_POLL_SECONDS" \
|
||||||
-e GEOINTEL_AUTH_ENABLED="$GEOINTEL_AUTH_ENABLED" \
|
-e GEOINTEL_AUTH_ENABLED="$GEOINTEL_AUTH_ENABLED" \
|
||||||
@@ -321,6 +369,11 @@ docker run -d \
|
|||||||
-e GEOINTEL_AUTH_PASSWORD_HASH="$GEOINTEL_AUTH_PASSWORD_HASH" \
|
-e GEOINTEL_AUTH_PASSWORD_HASH="$GEOINTEL_AUTH_PASSWORD_HASH" \
|
||||||
-e GEOINTEL_AUTH_SESSION_SECRET="$GEOINTEL_AUTH_SESSION_SECRET" \
|
-e GEOINTEL_AUTH_SESSION_SECRET="$GEOINTEL_AUTH_SESSION_SECRET" \
|
||||||
-e GEOINTEL_AUTH_SESSION_TTL_SECONDS="$GEOINTEL_AUTH_SESSION_TTL_SECONDS" \
|
-e GEOINTEL_AUTH_SESSION_TTL_SECONDS="$GEOINTEL_AUTH_SESSION_TTL_SECONDS" \
|
||||||
|
-e GEOINTEL_PUBLIC_BASE_URL="$GEOINTEL_PUBLIC_BASE_URL" \
|
||||||
|
-e GEOINTEL_AUTHENTIK_ISSUER="$GEOINTEL_AUTHENTIK_ISSUER" \
|
||||||
|
-e GEOINTEL_AUTHENTIK_CLIENT_ID="$GEOINTEL_AUTHENTIK_CLIENT_ID" \
|
||||||
|
-e GEOINTEL_AUTHENTIK_CLIENT_SECRET="$GEOINTEL_AUTHENTIK_CLIENT_SECRET" \
|
||||||
|
-e GEOINTEL_AUTHENTIK_ALLOWED_EMAIL="$GEOINTEL_AUTHENTIK_ALLOWED_EMAIL" \
|
||||||
-e GEOINTEL_GUEST_ACCESS_ENABLED="$GEOINTEL_GUEST_ACCESS_ENABLED" \
|
-e GEOINTEL_GUEST_ACCESS_ENABLED="$GEOINTEL_GUEST_ACCESS_ENABLED" \
|
||||||
-e GEOINTEL_GUEST_DISPLAY_NAME="$GEOINTEL_GUEST_DISPLAY_NAME" \
|
-e GEOINTEL_GUEST_DISPLAY_NAME="$GEOINTEL_GUEST_DISPLAY_NAME" \
|
||||||
-e GEOINTEL_GUEST_SESSION_TTL_SECONDS="$GEOINTEL_GUEST_SESSION_TTL_SECONDS" \
|
-e GEOINTEL_GUEST_SESSION_TTL_SECONDS="$GEOINTEL_GUEST_SESSION_TTL_SECONDS" \
|
||||||
|
|||||||
@@ -21,11 +21,17 @@ services:
|
|||||||
GEOINTEL_AOI_WORKER_POLL_SECONDS: ${GEOINTEL_AOI_WORKER_POLL_SECONDS:-2}
|
GEOINTEL_AOI_WORKER_POLL_SECONDS: ${GEOINTEL_AOI_WORKER_POLL_SECONDS:-2}
|
||||||
GEOINTEL_CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202}
|
GEOINTEL_CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202}
|
||||||
GEOINTEL_MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500}
|
GEOINTEL_MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500}
|
||||||
|
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB: ${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}
|
||||||
GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-false}
|
GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-false}
|
||||||
GEOINTEL_AUTH_USERNAME: ${GEOINTEL_AUTH_USERNAME:-}
|
GEOINTEL_AUTH_USERNAME: ${GEOINTEL_AUTH_USERNAME:-}
|
||||||
GEOINTEL_AUTH_PASSWORD_HASH: ${GEOINTEL_AUTH_PASSWORD_HASH:-}
|
GEOINTEL_AUTH_PASSWORD_HASH: ${GEOINTEL_AUTH_PASSWORD_HASH:-}
|
||||||
GEOINTEL_AUTH_SESSION_SECRET: ${GEOINTEL_AUTH_SESSION_SECRET:-}
|
GEOINTEL_AUTH_SESSION_SECRET: ${GEOINTEL_AUTH_SESSION_SECRET:-}
|
||||||
GEOINTEL_AUTH_SESSION_TTL_SECONDS: ${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200}
|
GEOINTEL_AUTH_SESSION_TTL_SECONDS: ${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200}
|
||||||
|
GEOINTEL_PUBLIC_BASE_URL: ${GEOINTEL_PUBLIC_BASE_URL:-http://localhost:1202}
|
||||||
|
GEOINTEL_AUTHENTIK_ISSUER: ${GEOINTEL_AUTHENTIK_ISSUER:-}
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_ID: ${GEOINTEL_AUTHENTIK_CLIENT_ID:-}
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_SECRET: ${GEOINTEL_AUTHENTIK_CLIENT_SECRET:-}
|
||||||
|
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL: ${GEOINTEL_AUTHENTIK_ALLOWED_EMAIL:-}
|
||||||
GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-true}
|
GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-true}
|
||||||
GEOINTEL_GUEST_DISPLAY_NAME: ${GEOINTEL_GUEST_DISPLAY_NAME:-Gast}
|
GEOINTEL_GUEST_DISPLAY_NAME: ${GEOINTEL_GUEST_DISPLAY_NAME:-Gast}
|
||||||
GEOINTEL_GUEST_SESSION_TTL_SECONDS: ${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200}
|
GEOINTEL_GUEST_SESSION_TTL_SECONDS: ${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200}
|
||||||
|
|||||||
@@ -23,11 +23,17 @@ services:
|
|||||||
STORAGE_ROOT: /app/storage
|
STORAGE_ROOT: /app/storage
|
||||||
CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202}
|
CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202}
|
||||||
MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500}
|
MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500}
|
||||||
|
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB: ${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}
|
||||||
GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-false}
|
GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-false}
|
||||||
GEOINTEL_AUTH_USERNAME: ${GEOINTEL_AUTH_USERNAME:-}
|
GEOINTEL_AUTH_USERNAME: ${GEOINTEL_AUTH_USERNAME:-}
|
||||||
GEOINTEL_AUTH_PASSWORD_HASH: ${GEOINTEL_AUTH_PASSWORD_HASH:-}
|
GEOINTEL_AUTH_PASSWORD_HASH: ${GEOINTEL_AUTH_PASSWORD_HASH:-}
|
||||||
GEOINTEL_AUTH_SESSION_SECRET: ${GEOINTEL_AUTH_SESSION_SECRET:-}
|
GEOINTEL_AUTH_SESSION_SECRET: ${GEOINTEL_AUTH_SESSION_SECRET:-}
|
||||||
GEOINTEL_AUTH_SESSION_TTL_SECONDS: ${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200}
|
GEOINTEL_AUTH_SESSION_TTL_SECONDS: ${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200}
|
||||||
|
GEOINTEL_PUBLIC_BASE_URL: ${GEOINTEL_PUBLIC_BASE_URL:-http://localhost:1202}
|
||||||
|
GEOINTEL_AUTHENTIK_ISSUER: ${GEOINTEL_AUTHENTIK_ISSUER:-}
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_ID: ${GEOINTEL_AUTHENTIK_CLIENT_ID:-}
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_SECRET: ${GEOINTEL_AUTHENTIK_CLIENT_SECRET:-}
|
||||||
|
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL: ${GEOINTEL_AUTHENTIK_ALLOWED_EMAIL:-}
|
||||||
GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-true}
|
GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-true}
|
||||||
GEOINTEL_GUEST_DISPLAY_NAME: ${GEOINTEL_GUEST_DISPLAY_NAME:-Gast}
|
GEOINTEL_GUEST_DISPLAY_NAME: ${GEOINTEL_GUEST_DISPLAY_NAME:-Gast}
|
||||||
GEOINTEL_GUEST_SESSION_TTL_SECONDS: ${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200}
|
GEOINTEL_GUEST_SESSION_TTL_SECONDS: ${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200}
|
||||||
|
|||||||
+36
-20
@@ -5,8 +5,10 @@ GeoIntel uses the same release gates in Gitea Actions and GitHub Actions:
|
|||||||
- `.gitea/workflows/release-gates.yml`
|
- `.gitea/workflows/release-gates.yml`
|
||||||
- `.github/workflows/release-gates.yml`
|
- `.github/workflows/release-gates.yml`
|
||||||
|
|
||||||
Gitea is the operational source-control platform. The GitHub workflow is kept
|
Gitea is the operational source-control platform. Its release workflow builds
|
||||||
equivalent so a mirror or external review does not receive a weaker gate.
|
the production AI variant and is the only workflow that can automatically
|
||||||
|
deploy. The GitHub mirror builds and scans the same AI variant for external
|
||||||
|
review but is not a production deployment authority.
|
||||||
|
|
||||||
## Runner requirements
|
## Runner requirements
|
||||||
|
|
||||||
@@ -16,11 +18,11 @@ The `ubuntu-latest` runner must provide:
|
|||||||
- Python 3.11 and Node 20 through the official setup actions;
|
- Python 3.11 and Node 20 through the official setup actions;
|
||||||
- Bash and Docker with Compose v2;
|
- Bash and Docker with Compose v2;
|
||||||
- permission to build images and mount `/var/run/docker.sock`;
|
- permission to build images and mount `/var/run/docker.sock`;
|
||||||
- sufficient disk for the all-in-one GIS image and scanner databases.
|
- sufficient disk for the all-in-one AI image and scanner databases.
|
||||||
|
|
||||||
The container job builds the GIS release variant only. PyTorch and
|
Both container jobs build the explicit AI image, including the
|
||||||
Ultralytics remain in the optional `ai` extra and in the explicit AI image
|
PyTorch/Ultralytics layers used on Tower. AI packages remain outside the
|
||||||
variant; CI does not silently make them base dependencies.
|
standard backend lock and are pinned by Docker build arguments.
|
||||||
|
|
||||||
## Quality gate
|
## Quality gate
|
||||||
|
|
||||||
@@ -33,11 +35,25 @@ cd frontend && npm ci
|
|||||||
```
|
```
|
||||||
|
|
||||||
It then validates the lock policy and runs the complete readiness script. The
|
It then validates the lock policy and runs the complete readiness script. The
|
||||||
readiness script covers backend compile/tests, contract audits, Alembic
|
readiness script covers Ruff, repository-layout validation, backend
|
||||||
single-head, frontend typecheck/build and release-script syntax. CI also
|
compile/tests, contract audits, Alembic single-head, frontend typecheck/build
|
||||||
|
and release-script syntax. CI also
|
||||||
renders offline migration SQL and resolved Compose configuration as retained
|
renders offline migration SQL and resolved Compose configuration as retained
|
||||||
evidence.
|
evidence.
|
||||||
|
|
||||||
|
Pull requests run managed validation against the real `backend/` and
|
||||||
|
`frontend/` projects plus the complete release gates. On a `main` push, the
|
||||||
|
Unraid deploy job has explicit `needs` dependencies on quality, dependency and
|
||||||
|
AI-container jobs. There is no separate deployment workflow: manual validation
|
||||||
|
uses `workflow_dispatch` on this same release-gates workflow and cannot skip
|
||||||
|
quality, dependency or AI-container jobs.
|
||||||
|
|
||||||
|
The deploy host requires a full controller commit SHA, builds the AI variant,
|
||||||
|
records its Docker image/config digest, generates SBOM and Trivy evidence for
|
||||||
|
that exact local image, and starts the immutable image ID. Deployment fails if
|
||||||
|
the running container ID, revision label or AI label differs from the retained
|
||||||
|
attestation.
|
||||||
|
|
||||||
## Reproducible Python lock
|
## Reproducible Python lock
|
||||||
|
|
||||||
`backend/requirements-runtime.lock` and `backend/requirements-ci.lock` are
|
`backend/requirements-runtime.lock` and `backend/requirements-ci.lock` are
|
||||||
@@ -73,12 +89,12 @@ The dependency job:
|
|||||||
- publishes both unfiltered and policy-filtered Python JSON reports plus the
|
- publishes both unfiltered and policy-filtered Python JSON reports plus the
|
||||||
npm JSON report, including on failure.
|
npm JSON report, including on failure.
|
||||||
|
|
||||||
The only current Python/container exceptions are the Starlette 2026 advisories recorded
|
There are currently no Python or container vulnerability exceptions.
|
||||||
in `security/pip-audit-exceptions.json`. FastAPI 0.139.2 still constrains
|
`security/pip-audit-exceptions.json` remains as a strict, machine-readable
|
||||||
Starlette below 0.53 while patched releases begin at 1.x. GeoIntel applies
|
registry: every future exception must identify one advisory, package, specific
|
||||||
request-target, form-content, route-class and Linux-runtime compensating
|
reason and expiry date. Readiness and CI fail on malformed or expired entries;
|
||||||
controls. The exception file has a mandatory review date; readiness and CI
|
new advisories are never auto-ignored. GeoIntel requires Starlette 1.3.1 or
|
||||||
fail automatically after it expires. New advisories are never auto-ignored.
|
newer and therefore no longer suppresses the five 2026 Starlette advisories.
|
||||||
The all-in-one image replaces the Go-based base-image `gosu` helper with a
|
The all-in-one image replaces the Go-based base-image `gosu` helper with a
|
||||||
small `setpriv` exec wrapper and upgrades packaged setuptools/wheel metadata;
|
small `setpriv` exec wrapper and upgrades packaged setuptools/wheel metadata;
|
||||||
the final runtime filesystem no longer exposes the vulnerable Go executable.
|
the final runtime filesystem no longer exposes the vulnerable Go executable.
|
||||||
@@ -89,8 +105,8 @@ verified to contain the audited shell wrapper. This is not a vulnerability
|
|||||||
exception: the raw evidence remains published and the runtime wrapper is
|
exception: the raw evidence remains published and the runtime wrapper is
|
||||||
exercised during live release validation.
|
exercised during live release validation.
|
||||||
|
|
||||||
The container job builds a non-AI all-in-one image and uses digest-pinned
|
The operational container job builds the production AI all-in-one image and
|
||||||
scanner images:
|
uses digest-pinned scanner images:
|
||||||
|
|
||||||
- Syft 1.44.0 generates an SPDX JSON SBOM;
|
- Syft 1.44.0 generates an SPDX JSON SBOM;
|
||||||
- Trivy 0.70.0 generates a complete JSON vulnerability report;
|
- Trivy 0.70.0 generates a complete JSON vulnerability report;
|
||||||
@@ -103,12 +119,12 @@ Run these controls on a Docker-enabled workstation:
|
|||||||
```bash
|
```bash
|
||||||
docker build \
|
docker build \
|
||||||
-f deploy/unraid/Dockerfile.all-in-one \
|
-f deploy/unraid/Dockerfile.all-in-one \
|
||||||
--build-arg GEOINTEL_INSTALL_AI=false \
|
--build-arg GEOINTEL_INSTALL_AI=true \
|
||||||
--build-arg GEOINTEL_BUILD_SHA=local \
|
--build-arg GEOINTEL_BUILD_SHA=local \
|
||||||
--build-arg GEOINTEL_BUILD_TIME=local \
|
--build-arg GEOINTEL_BUILD_TIME=local \
|
||||||
-t geointel-ci:local .
|
-t geointel-ci:local-ai .
|
||||||
bash scripts/generate_container_sbom.sh geointel-ci:local
|
bash scripts/generate_container_sbom.sh geointel-ci:local-ai
|
||||||
bash scripts/scan_container_image.sh geointel-ci:local
|
bash scripts/scan_container_image.sh geointel-ci:local-ai
|
||||||
```
|
```
|
||||||
|
|
||||||
Outputs are written below ignored `artifacts/`; scanner cache is written below
|
Outputs are written below ignored `artifacts/`; scanner cache is written below
|
||||||
|
|||||||
@@ -89,9 +89,9 @@ The normal project lifecycle archive path does not need destructive
|
|||||||
confirmation because it changes only `status=archived` and preserves all
|
confirmation because it changes only `status=archived` and preserves all
|
||||||
data.
|
data.
|
||||||
|
|
||||||
## Destructive apply gate
|
## Recoverable quarantine gate
|
||||||
|
|
||||||
First create a fresh backup with a SHA-256 storage inventory on the host:
|
First create a fresh backup with a byte-complete SHA-256 storage snapshot on the host:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bash scripts/backup_release_state.sh \
|
bash scripts/backup_release_state.sh \
|
||||||
@@ -112,7 +112,8 @@ bash scripts/verify_release_backup.sh \
|
|||||||
```
|
```
|
||||||
|
|
||||||
The Unraid runtime mounts `GEOINTEL_BACKUPS_PATH` read-only at `/app/backups`.
|
The Unraid runtime mounts `GEOINTEL_BACKUPS_PATH` read-only at `/app/backups`.
|
||||||
Only after reviewing the dry run may an operator execute:
|
Only after reviewing the dry run may an operator move the exact candidates to
|
||||||
|
a protected quarantine:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
|
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
|
||||||
@@ -120,14 +121,34 @@ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
|
|||||||
--max-delete <reviewed-candidate-count> \
|
--max-delete <reviewed-candidate-count> \
|
||||||
--backup-dir /app/backups/<release-id> \
|
--backup-dir /app/backups/<release-id> \
|
||||||
--backup-max-age-hours 24 \
|
--backup-max-age-hours 24 \
|
||||||
--confirm DELETE_STORAGE_ARTIFACTS \
|
--confirm QUARANTINE_STORAGE_ARTIFACTS \
|
||||||
--apply
|
--apply
|
||||||
```
|
```
|
||||||
|
|
||||||
The command re-runs the audit immediately before deletion. It refuses the
|
The command re-runs the audit immediately before quarantine. It refuses the
|
||||||
operation when the exact token is absent, the candidate count exceeds the
|
operation when the exact token is absent, the candidate count exceeds the
|
||||||
operator limit, the backup is stale/incomplete, checksums differ, the storage
|
operator limit, the backup is stale/incomplete, snapshot checksums differ, or
|
||||||
inventory is not SHA-256, or the path is outside the cleanup allowlist.
|
the path is outside the cleanup allowlist.
|
||||||
|
Every moved byte is hashed and retained below
|
||||||
|
`operator-evidence/cleanup-quarantine/<operation-id>/files/`; an atomic
|
||||||
|
manifest records its original path, quarantine path, size, checksum and
|
||||||
|
backup identity. Cleanup uses a backup-to-quarantine hard-link state machine:
|
||||||
|
the manifest records `planned`, `linked` and `quarantined` transitions so each
|
||||||
|
crash window can be reconciled without losing the retained inode.
|
||||||
|
|
||||||
|
Restore a reviewed quarantine without overwriting any path:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec geointel python /app/scripts/restore_storage_quarantine.py \
|
||||||
|
--storage-root /app/storage \
|
||||||
|
--manifest /app/storage/operator-evidence/cleanup-quarantine/<operation-id>/manifest.json \
|
||||||
|
--confirm RESTORE_QUARANTINED_ARTIFACTS
|
||||||
|
```
|
||||||
|
|
||||||
|
Restore validates every retained checksum and never replaces an existing
|
||||||
|
destination. It uses an exclusive hard link and reconciles both-file and
|
||||||
|
one-file interruption states before updating the manifest, so an interrupted
|
||||||
|
restore can be resumed and verified.
|
||||||
|
|
||||||
The older demo-export cleanup has the same gate and uses confirmation token
|
The older demo-export cleanup has the same gate and uses confirmation token
|
||||||
`DELETE_DEMO_EXPORTS`.
|
`DELETE_DEMO_EXPORTS`.
|
||||||
|
|||||||
+46
-7
@@ -15,7 +15,7 @@ The repository version is stored in `VERSION`. The current release is
|
|||||||
- clean `main` worktree at the commit being released;
|
- clean `main` worktree at the commit being released;
|
||||||
- secure non-default PostGIS password in the Tower `.env`;
|
- secure non-default PostGIS password in the Tower `.env`;
|
||||||
- existing local AI model only when the AI image is enabled;
|
- existing local AI model only when the AI image is enabled;
|
||||||
- recent checksum-verified backup with SHA-256 storage inventory;
|
- recent checksum-verified database dump and byte-complete storage/model snapshot;
|
||||||
- Docker, `ssh-keygen`, Python 3.11, Node 20 and Bash available;
|
- Docker, `ssh-keygen`, Python 3.11, Node 20 and Bash available;
|
||||||
- one Alembic head and no unsupported metric represented as successful.
|
- one Alembic head and no unsupported metric represented as successful.
|
||||||
|
|
||||||
@@ -39,6 +39,24 @@ docker compose config
|
|||||||
|
|
||||||
## Immutable deployment
|
## Immutable deployment
|
||||||
|
|
||||||
|
`deploy-release.sh` first builds/reuses the candidate while the current release
|
||||||
|
stays available. Immediately before replacement it quiesces backend writes and
|
||||||
|
creates and checksum-verifies a database dump plus byte-complete SHA-256
|
||||||
|
storage/model snapshots under `/mnt/user/appdata/geointel/backups`. The first
|
||||||
|
snapshot copies every byte (using CoW reflinks when supported); later snapshots
|
||||||
|
hard-link only checksum-identical bytes from a verified older backup, never
|
||||||
|
from live storage. A conservative full-copy/free-space preflight runs before
|
||||||
|
backend quiescence. Only then may the
|
||||||
|
candidate start or run Alembic. If backup fails, the unchanged release is
|
||||||
|
restarted; deployment stops if existing PostGIS state cannot be backed up
|
||||||
|
consistently.
|
||||||
|
|
||||||
|
The backup manifest separates provenance intentionally:
|
||||||
|
`backup_tool_revision` is the new candidate source running the backup tool,
|
||||||
|
while `running_image_revision` is the OCI label of the old release whose data
|
||||||
|
is being captured. The retained Docker `image_id`, not either descriptive
|
||||||
|
revision field, is authoritative for rollback.
|
||||||
|
|
||||||
On the Codex workstation:
|
On the Codex workstation:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
@@ -58,8 +76,9 @@ bash scripts/live_migration_smoke.sh
|
|||||||
|
|
||||||
## Backup and recovery proof
|
## Backup and recovery proof
|
||||||
|
|
||||||
Create an immutable backup. The SHA-256 inventory can take several minutes on
|
Create an immutable byte-complete backup. Initial storage/model copy and
|
||||||
large storage:
|
verification can be I/O-heavy; subsequent backups deduplicate unchanged bytes
|
||||||
|
against the newest verified prior snapshot:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bash scripts/backup_release_state.sh \
|
bash scripts/backup_release_state.sh \
|
||||||
@@ -124,17 +143,37 @@ gate fails on reachable fixed HIGH/CRITICAL findings.
|
|||||||
|
|
||||||
## Rollback proof
|
## Rollback proof
|
||||||
|
|
||||||
The rollback command reuses persistent paths and never downgrades Alembic:
|
Use the exact backup printed by the deployment. Rollback first stops the
|
||||||
|
candidate, restores and verifies the pre-deploy PostgreSQL dump in a temporary
|
||||||
|
proof database, then swaps database names while retaining the pre-restore
|
||||||
|
database as a recovery point. Only then does it start the exact image ID bound
|
||||||
|
into that backup; it never relies on a mutable global `previous` tag. It never runs an
|
||||||
|
Alembic downgrade or starts an old image against an unknown newer schema:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bash deploy/unraid/rollback-dockerman-container.sh
|
bash deploy/unraid/rollback-dockerman-container.sh \
|
||||||
|
--backup-dir /mnt/user/appdata/geointel/backups/<predeploy-release-id> \
|
||||||
|
--confirm-production-database-restore
|
||||||
curl -fsS http://127.0.0.1:1202/health/ready
|
curl -fsS http://127.0.0.1:1202/health/ready
|
||||||
bash deploy/unraid/deploy-release.sh
|
bash deploy/unraid/deploy-release.sh
|
||||||
curl -fsS http://127.0.0.1:1202/health/ready
|
curl -fsS http://127.0.0.1:1202/health/ready
|
||||||
```
|
```
|
||||||
|
|
||||||
For a future backward-incompatible migration, restore the verified pre-release
|
Database rollback restores persisted rows and schema. Files newly written by a
|
||||||
backup instead of running an older image against a newer schema.
|
failed candidate remain in storage as unreferenced evidence; the protected,
|
||||||
|
recoverable quarantine flow in `DATA_OPERATIONS_RUNBOOK.md` handles those
|
||||||
|
files without deleting source data.
|
||||||
|
|
||||||
|
After the rollback has remained healthy and its retained evidence has been
|
||||||
|
reviewed, list the recovery database printed by the script. Remove it only by
|
||||||
|
an explicit, separately approved `dropdb` maintenance command; deployment and
|
||||||
|
rollback never auto-delete recovery databases or backup directories. Retain at
|
||||||
|
least the current successful predeploy backup and its predecessor. Before
|
||||||
|
removing an older backup, run `verify_release_backup.sh` on the backups that
|
||||||
|
remain and confirm no newer snapshot hard-links depend on operator policy for
|
||||||
|
retention (hard-linked bytes remain allocated while any retained backup names
|
||||||
|
them). Remove its `rollback-predeploy-*` image tag only in the same explicitly
|
||||||
|
reviewed retention operation.
|
||||||
|
|
||||||
## Tag and signed package
|
## Tag and signed package
|
||||||
|
|
||||||
|
|||||||
@@ -28,10 +28,11 @@ The restore smoke may only create databases whose name starts with
|
|||||||
use `pg_restore --clean` and drops the temporary database unless an operator
|
use `pg_restore --clean` and drops the temporary database unless an operator
|
||||||
explicitly asks to retain it.
|
explicitly asks to retain it.
|
||||||
|
|
||||||
Storage and model files are inventoried rather than copied into the database
|
Storage and model files are retained as byte-complete, SHA-256 verified
|
||||||
dump. Release backups must therefore be paired with the persistent storage
|
snapshots alongside the database dump. Snapshot creation rejects symlinks and
|
||||||
volume backup policy. Use `--inventory-mode sha256` for final release
|
special files. The first snapshot uses CoW reflinks when supported and falls
|
||||||
evidence.
|
back to full copies; later snapshots hard-link checksum-identical files only
|
||||||
|
from a verified older backup, never from live storage.
|
||||||
|
|
||||||
An old persistent volume can also retain glibc collation metadata for the
|
An old persistent volume can also retain glibc collation metadata for the
|
||||||
empty `postgres` and `template1` system databases. If `createdb` fails for that
|
empty `postgres` and `template1` system databases. If `createdb` fails for that
|
||||||
|
|||||||
@@ -5,3 +5,5 @@ __pycache__
|
|||||||
.pytest_cache
|
.pytest_cache
|
||||||
.vite
|
.vite
|
||||||
.env
|
.env
|
||||||
|
.env.*
|
||||||
|
!.env.example
|
||||||
|
|||||||
+31
-2
@@ -1,3 +1,16 @@
|
|||||||
|
geo $geointel_trusted_forwarder {
|
||||||
|
default 0;
|
||||||
|
127.0.0.0/8 1;
|
||||||
|
::1/128 1;
|
||||||
|
172.16.0.0/12 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
map "$geointel_trusted_forwarder:$http_x_forwarded_proto" $geointel_forwarded_proto {
|
||||||
|
default $scheme;
|
||||||
|
"1:https" https;
|
||||||
|
"1:http" http;
|
||||||
|
}
|
||||||
|
|
||||||
server {
|
server {
|
||||||
listen 80;
|
listen 80;
|
||||||
server_name _;
|
server_name _;
|
||||||
@@ -5,16 +18,32 @@ server {
|
|||||||
proxy_read_timeout 600s;
|
proxy_read_timeout 600s;
|
||||||
proxy_send_timeout 600s;
|
proxy_send_timeout 600s;
|
||||||
|
|
||||||
|
add_header Content-Security-Policy "frame-ancestors 'none'" always;
|
||||||
|
add_header X-Frame-Options "DENY" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
|
||||||
|
|
||||||
root /usr/share/nginx/html;
|
root /usr/share/nginx/html;
|
||||||
index index.html;
|
index index.html;
|
||||||
|
|
||||||
location = /index.html {
|
location = /index.html {
|
||||||
add_header Cache-Control "no-cache";
|
add_header Cache-Control "no-cache";
|
||||||
|
add_header Content-Security-Policy "frame-ancestors 'none'" always;
|
||||||
|
add_header X-Frame-Options "DENY" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
|
||||||
try_files /index.html =404;
|
try_files /index.html =404;
|
||||||
}
|
}
|
||||||
|
|
||||||
location /assets/ {
|
location /assets/ {
|
||||||
add_header Cache-Control "no-cache";
|
add_header Cache-Control "no-cache";
|
||||||
|
add_header Content-Security-Policy "frame-ancestors 'none'" always;
|
||||||
|
add_header X-Frame-Options "DENY" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
|
||||||
try_files $uri =404;
|
try_files $uri =404;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -24,7 +53,7 @@ server {
|
|||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;
|
||||||
}
|
}
|
||||||
|
|
||||||
location = /health {
|
location = /health {
|
||||||
@@ -33,7 +62,7 @@ server {
|
|||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;
|
||||||
}
|
}
|
||||||
|
|
||||||
location = /health/live {
|
location = /health/live {
|
||||||
|
|||||||
+19
-9
@@ -2142,11 +2142,17 @@ bash scripts/backup_release_state.sh \
|
|||||||
```
|
```
|
||||||
|
|
||||||
The backup is written atomically and contains a PostgreSQL custom-format dump,
|
The backup is written atomically and contains a PostgreSQL custom-format dump,
|
||||||
archive listing, Alembic/PostGIS metadata, critical table counts, optional
|
archive listing, Alembic/PostGIS metadata, critical table counts and
|
||||||
storage/model inventories and SHA-256 checksums. An empty or known-default
|
byte-complete SHA-256-verified storage/model snapshots. The first snapshot is a
|
||||||
database password leaves the release gate failed. For an emergency backup
|
full copy; a later deployment may hard-link only checksum-identical files from
|
||||||
before rotating that password, add `--allow-insecure-password`; the manifest
|
another completed, fully verified backup with `--link-dest-backup`. It never
|
||||||
still records the insecure state.
|
hard-links a live source file and never deletes an older backup. An empty or
|
||||||
|
known-default database password leaves the release gate failed. For an
|
||||||
|
emergency backup before rotating that password, add
|
||||||
|
`--allow-insecure-password`; the manifest still records the insecure state.
|
||||||
|
`backup_tool_revision` identifies the candidate source that executed the
|
||||||
|
backup; `running_image_revision` identifies the currently running old image.
|
||||||
|
Rollback is always bound to the retained immutable Docker `image_id`.
|
||||||
|
|
||||||
Verify without changing any database:
|
Verify without changing any database:
|
||||||
|
|
||||||
@@ -2190,7 +2196,7 @@ docker exec geointel python /app/scripts/audit_data_operations.py \
|
|||||||
--output /app/storage/release-evidence/rc-current/data-operations.json
|
--output /app/storage/release-evidence/rc-current/data-operations.json
|
||||||
```
|
```
|
||||||
|
|
||||||
Preview old unreferenced derived/cache/export candidates without deletion:
|
Preview old unreferenced derived/cache/export candidates without mutation:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
|
docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
|
||||||
@@ -2199,10 +2205,14 @@ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \
|
|||||||
```
|
```
|
||||||
|
|
||||||
Apply requires a reviewed candidate count, the exact
|
Apply requires a reviewed candidate count, the exact
|
||||||
`DELETE_STORAGE_ARTIFACTS` token and a backup no older than 24 hours with a
|
`QUARANTINE_STORAGE_ARTIFACTS` token and a backup no older than 24 hours with a
|
||||||
checksum-verified database dump and SHA-256 storage inventory. The host backup
|
checksum-verified database dump and byte-complete storage snapshot. The host backup
|
||||||
root is mounted read-only at `/app/backups`. See
|
root is mounted read-only at `/app/backups`. See
|
||||||
`docs/DATA_OPERATIONS_RUNBOOK.md`. No cleanup is scheduled by GeoIntel.
|
`docs/DATA_OPERATIONS_RUNBOOK.md`. Candidates enter protected
|
||||||
|
`operator-evidence/cleanup-quarantine` storage through an interruption-safe
|
||||||
|
hard-link/unlink state machine. `restore_storage_quarantine.py` reverses that
|
||||||
|
move with the exact `RESTORE_QUARANTINED_ARTIFACTS` token and refuses to
|
||||||
|
overwrite an existing original path. No cleanup is scheduled by GeoIntel.
|
||||||
|
|
||||||
## RC-8 Belgium/North Sea release journeys
|
## RC-8 Belgium/North Sea release journeys
|
||||||
|
|
||||||
|
|||||||
@@ -14,17 +14,26 @@ from datetime import datetime, timedelta, timezone
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
from sqlalchemy import func
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
BACKEND_ROOT = ROOT / "backend" if (ROOT / "backend" / "app").is_dir() else ROOT
|
BACKEND_ROOT = ROOT / "backend" if (ROOT / "backend" / "app").is_dir() else ROOT
|
||||||
if str(BACKEND_ROOT) not in sys.path:
|
if str(BACKEND_ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(BACKEND_ROOT))
|
sys.path.insert(0, str(BACKEND_ROOT))
|
||||||
|
|
||||||
from app.core.config import get_settings
|
from app.core.config import get_settings # noqa: E402 - imported after backend path bootstrap
|
||||||
from app.db.session import SessionLocal
|
from app.db.session import SessionLocal # noqa: E402 - imported after backend path bootstrap
|
||||||
from sqlalchemy import func
|
from app.models import ( # noqa: E402 - imported after backend path bootstrap
|
||||||
|
AnalysisRun,
|
||||||
from app.models import AnalysisRun, Dataset, DatasetVersion, Detection, Export, Job, Project, Segmentation
|
Dataset,
|
||||||
|
DatasetVersion,
|
||||||
|
Detection,
|
||||||
|
Export,
|
||||||
|
Job,
|
||||||
|
Project,
|
||||||
|
Segmentation,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
NATIONAL_PROJECT_NAME = "Belgium and North Sea Workbench"
|
NATIONAL_PROJECT_NAME = "Belgium and North Sea Workbench"
|
||||||
|
|||||||
+163
-48
@@ -3,12 +3,14 @@ set -euo pipefail
|
|||||||
|
|
||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
CONTAINER="geointel"
|
CONTAINER="geointel"
|
||||||
OUTPUT_ROOT="backups"
|
OUTPUT_ROOT="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
|
||||||
RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)"
|
RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||||
STORAGE_PATH=""
|
STORAGE_PATH=""
|
||||||
MODELS_PATH=""
|
MODELS_PATH=""
|
||||||
INVENTORY_MODE="metadata"
|
INVENTORY_MODE="metadata"
|
||||||
ALLOW_INSECURE_PASSWORD="false"
|
ALLOW_INSECURE_PASSWORD="false"
|
||||||
|
LINK_DEST_BACKUP=""
|
||||||
|
ROLLBACK_IMAGE_TAG=""
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
cat <<'EOF'
|
cat <<'EOF'
|
||||||
@@ -19,11 +21,16 @@ GeoIntel container. It never deletes or restores application data.
|
|||||||
|
|
||||||
Options:
|
Options:
|
||||||
--container NAME Docker container (default: geointel)
|
--container NAME Docker container (default: geointel)
|
||||||
--output-root PATH Host backup root (default: backups)
|
--output-root PATH Host backup root (default:
|
||||||
|
/mnt/user/appdata/geointel/backups)
|
||||||
--release-id ID Safe backup directory name
|
--release-id ID Safe backup directory name
|
||||||
--storage-path PATH Optional host storage path to inventory
|
--storage-path PATH Host storage path to snapshot byte-for-byte
|
||||||
--models-path PATH Optional host model path to inventory
|
--models-path PATH Host model path to snapshot byte-for-byte
|
||||||
--inventory-mode metadata|sha256 Hash all inventoried files only with sha256
|
--inventory-mode metadata|sha256 Retained manifest compatibility setting
|
||||||
|
--link-dest-backup PATH Verified older backup used only to hard-link
|
||||||
|
checksum-identical backup-to-backup files
|
||||||
|
--rollback-image-tag TAG Immutable backup-specific tag bound to the
|
||||||
|
running image ID
|
||||||
--allow-insecure-password Complete emergency backup despite an
|
--allow-insecure-password Complete emergency backup despite an
|
||||||
empty/default production DB password
|
empty/default production DB password
|
||||||
EOF
|
EOF
|
||||||
@@ -37,6 +44,8 @@ while [ "$#" -gt 0 ]; do
|
|||||||
--storage-path) STORAGE_PATH="$2"; shift 2 ;;
|
--storage-path) STORAGE_PATH="$2"; shift 2 ;;
|
||||||
--models-path) MODELS_PATH="$2"; shift 2 ;;
|
--models-path) MODELS_PATH="$2"; shift 2 ;;
|
||||||
--inventory-mode) INVENTORY_MODE="$2"; shift 2 ;;
|
--inventory-mode) INVENTORY_MODE="$2"; shift 2 ;;
|
||||||
|
--link-dest-backup) LINK_DEST_BACKUP="$2"; shift 2 ;;
|
||||||
|
--rollback-image-tag) ROLLBACK_IMAGE_TAG="$2"; shift 2 ;;
|
||||||
--allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;;
|
--allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;;
|
||||||
--help|-h) usage; exit 0 ;;
|
--help|-h) usage; exit 0 ;;
|
||||||
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||||
@@ -51,12 +60,93 @@ if [ "$INVENTORY_MODE" != "metadata" ] && [ "$INVENTORY_MODE" != "sha256" ]; the
|
|||||||
echo "--inventory-mode must be metadata or sha256" >&2
|
echo "--inventory-mode must be metadata or sha256" >&2
|
||||||
exit 2
|
exit 2
|
||||||
fi
|
fi
|
||||||
for required in docker python3 sha256sum git; do
|
for required in docker python3 sha256sum; do
|
||||||
if ! command -v "$required" >/dev/null 2>&1; then
|
if ! command -v "$required" >/dev/null 2>&1; then
|
||||||
echo "Missing required command: $required" >&2
|
echo "Missing required command: $required" >&2
|
||||||
exit 2
|
exit 2
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
resolve_source_revision() {
|
||||||
|
local controller_sha="" explicit_sha="${GEOINTEL_BUILD_SHA:-}"
|
||||||
|
local gitea_sha="${GITEA_COMMIT_SHA:-}" github_sha="${GITHUB_SHA:-}"
|
||||||
|
local git_head="" git_dirty="false" source=""
|
||||||
|
|
||||||
|
if [ -n "$gitea_sha" ]; then
|
||||||
|
if ! [[ "$gitea_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
|
||||||
|
echo "GITEA_COMMIT_SHA must contain one full 40-character Git commit SHA." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
controller_sha="${gitea_sha,,}"
|
||||||
|
source="GITEA_COMMIT_SHA"
|
||||||
|
fi
|
||||||
|
if [ -n "$github_sha" ]; then
|
||||||
|
if ! [[ "$github_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
|
||||||
|
echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
github_sha="${github_sha,,}"
|
||||||
|
if [ -n "$controller_sha" ] && [ "$controller_sha" != "$github_sha" ]; then
|
||||||
|
echo "Controller commit variables disagree." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
controller_sha="$github_sha"
|
||||||
|
source="${source:-GITHUB_SHA}"
|
||||||
|
fi
|
||||||
|
if [ -n "$explicit_sha" ]; then
|
||||||
|
if ! [[ "$explicit_sha" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]]; then
|
||||||
|
echo "GEOINTEL_BUILD_SHA contains an unsafe release revision." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
explicit_sha="${explicit_sha,,}"
|
||||||
|
fi
|
||||||
|
if [ -n "$controller_sha" ]; then
|
||||||
|
if [ -n "$explicit_sha" ] && [ "$explicit_sha" != "$controller_sha" ]; then
|
||||||
|
echo "GEOINTEL_BUILD_SHA differs from the controller revision." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
explicit_sha="$controller_sha"
|
||||||
|
elif [ -n "${GITEA_REPOSITORY:-}${GITHUB_REPOSITORY:-}" ]; then
|
||||||
|
if ! [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]]; then
|
||||||
|
echo "Automated backup requires a full controller or GEOINTEL_BUILD_SHA revision." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
source="GEOINTEL_BUILD_SHA"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if command -v git >/dev/null 2>&1 && git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1; then
|
||||||
|
git_head="$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)"
|
||||||
|
git_head="${git_head,,}"
|
||||||
|
if ! [[ "$git_head" =~ ^[0-9a-f]{40}$ ]]; then
|
||||||
|
echo "Could not resolve a full Git revision from the source checkout." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
if [ -n "$explicit_sha" ] && [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]] && [ "$git_head" != "$explicit_sha" ]; then
|
||||||
|
echo "Source checkout does not match the supplied release revision." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
if [ -n "$(git -C "$ROOT" status --porcelain=v1 2>/dev/null)" ]; then
|
||||||
|
git_dirty="true"
|
||||||
|
fi
|
||||||
|
if [ -z "$explicit_sha" ]; then
|
||||||
|
explicit_sha="$git_head"
|
||||||
|
source="git"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "$explicit_sha" ]; then
|
||||||
|
echo "Cannot bind backup to a source revision; provide GEOINTEL_BUILD_SHA or a controller SHA." >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
SOURCE_REVISION="$explicit_sha"
|
||||||
|
SOURCE_REVISION_SOURCE="${source:-GEOINTEL_BUILD_SHA}"
|
||||||
|
SOURCE_GIT_DIRTY="$git_dirty"
|
||||||
|
}
|
||||||
|
|
||||||
|
SOURCE_REVISION=""
|
||||||
|
SOURCE_REVISION_SOURCE=""
|
||||||
|
SOURCE_GIT_DIRTY="false"
|
||||||
|
resolve_source_revision
|
||||||
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||||
echo "Container '$CONTAINER' is not running." >&2
|
echo "Container '$CONTAINER' is not running." >&2
|
||||||
exit 3
|
exit 3
|
||||||
@@ -72,6 +162,27 @@ if [ -e "$PARTIAL" ] || [ -e "$FINAL" ]; then
|
|||||||
fi
|
fi
|
||||||
mkdir -p "$PARTIAL"
|
mkdir -p "$PARTIAL"
|
||||||
|
|
||||||
|
if [ -n "$LINK_DEST_BACKUP" ]; then
|
||||||
|
LINK_DEST_BACKUP="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve(strict=True))' "$LINK_DEST_BACKUP")"
|
||||||
|
python3 - "$OUTPUT_ROOT" "$LINK_DEST_BACKUP" <<'PY'
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
root = pathlib.Path(sys.argv[1])
|
||||||
|
candidate = pathlib.Path(sys.argv[2])
|
||||||
|
try:
|
||||||
|
candidate.relative_to(root)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise SystemExit(f"Link-dest backup must remain below {root}") from exc
|
||||||
|
if candidate == root or candidate.name.startswith("."):
|
||||||
|
raise SystemExit("Link-dest backup must identify one completed immutable backup")
|
||||||
|
PY
|
||||||
|
(
|
||||||
|
cd "$LINK_DEST_BACKUP"
|
||||||
|
sha256sum -c CHECKSUMS.sha256 >/dev/null
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
|
||||||
cleanup_partial() {
|
cleanup_partial() {
|
||||||
if [ -d "$PARTIAL" ]; then
|
if [ -d "$PARTIAL" ]; then
|
||||||
rm -rf -- "$PARTIAL"
|
rm -rf -- "$PARTIAL"
|
||||||
@@ -113,10 +224,21 @@ test -s "$PARTIAL/database.list"
|
|||||||
|
|
||||||
IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")"
|
IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")"
|
||||||
IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")"
|
IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")"
|
||||||
GIT_COMMIT="$(git -C "$ROOT" rev-parse HEAD)"
|
RUNNING_IMAGE_REVISION="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CONTAINER")"
|
||||||
GIT_DIRTY="false"
|
RUNNING_IMAGE_REVISION_IS_FULL_SHA="false"
|
||||||
if [ -n "$(git -C "$ROOT" status --porcelain=v1)" ]; then
|
if [[ "$RUNNING_IMAGE_REVISION" =~ ^[0-9A-Fa-f]{40}$ ]]; then
|
||||||
GIT_DIRTY="true"
|
RUNNING_IMAGE_REVISION="${RUNNING_IMAGE_REVISION,,}"
|
||||||
|
RUNNING_IMAGE_REVISION_IS_FULL_SHA="true"
|
||||||
|
elif ! [[ "$RUNNING_IMAGE_REVISION" =~ ^[A-Za-z0-9._-]{1,128}$ ]]; then
|
||||||
|
echo "Running image has an unsafe or missing OCI revision label." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
if [ -n "$ROLLBACK_IMAGE_TAG" ]; then
|
||||||
|
TAGGED_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$ROLLBACK_IMAGE_TAG" 2>/dev/null || true)"
|
||||||
|
if [ "$TAGGED_IMAGE_ID" != "$IMAGE_ID" ]; then
|
||||||
|
echo "Backup-specific rollback tag does not resolve to the running image ID." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \
|
docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \
|
||||||
@@ -132,48 +254,32 @@ for table in projects areas datasets dataset_versions vector_features jobs analy
|
|||||||
printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv"
|
printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv"
|
||||||
done
|
done
|
||||||
|
|
||||||
inventory_path() {
|
snapshot_path() {
|
||||||
local source_path="$1"
|
local source_path="$1"
|
||||||
local output_path="$2"
|
local label="$2"
|
||||||
|
local manifest_path="$PARTIAL/${label}-manifest.tsv"
|
||||||
|
local snapshot_path="$PARTIAL/${label}-snapshot"
|
||||||
|
local link_args=()
|
||||||
if [ -z "$source_path" ]; then
|
if [ -z "$source_path" ]; then
|
||||||
printf 'not_requested\n' > "$output_path"
|
printf 'not_requested\n' > "$manifest_path"
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
python3 - "$source_path" "$output_path" "$INVENTORY_MODE" <<'PY'
|
if [ -n "$LINK_DEST_BACKUP" ]; then
|
||||||
import hashlib
|
link_args=(
|
||||||
import os
|
--link-dest-snapshot "$LINK_DEST_BACKUP/${label}-snapshot"
|
||||||
import pathlib
|
--link-dest-manifest "$LINK_DEST_BACKUP/${label}-manifest.tsv"
|
||||||
import sys
|
)
|
||||||
|
fi
|
||||||
root = pathlib.Path(sys.argv[1]).expanduser().resolve()
|
python3 "$ROOT/scripts/release_backup_snapshot.py" create \
|
||||||
output = pathlib.Path(sys.argv[2])
|
--source "$source_path" \
|
||||||
mode = sys.argv[3]
|
--snapshot "$snapshot_path" \
|
||||||
if not root.is_dir():
|
--manifest "$manifest_path" \
|
||||||
raise SystemExit(f"Inventory root is not a directory: {root}")
|
--label "$label" \
|
||||||
|
"${link_args[@]}"
|
||||||
def digest(path: pathlib.Path) -> str:
|
|
||||||
value = hashlib.sha256()
|
|
||||||
with path.open("rb") as handle:
|
|
||||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
||||||
value.update(chunk)
|
|
||||||
return value.hexdigest()
|
|
||||||
|
|
||||||
with output.open("w", encoding="utf-8", newline="\n") as handle:
|
|
||||||
handle.write("relative_path\tsize_bytes\tmtime_ns\tsha256\n")
|
|
||||||
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()):
|
|
||||||
if path.is_symlink() or not path.is_file():
|
|
||||||
continue
|
|
||||||
stat = path.stat()
|
|
||||||
checksum = digest(path) if mode == "sha256" else ""
|
|
||||||
relative = path.relative_to(root).as_posix()
|
|
||||||
if "\t" in relative or "\n" in relative:
|
|
||||||
raise SystemExit(f"Unsupported inventory path: {relative!r}")
|
|
||||||
handle.write(f"{relative}\t{stat.st_size}\t{stat.st_mtime_ns}\t{checksum}\n")
|
|
||||||
PY
|
|
||||||
}
|
}
|
||||||
|
|
||||||
inventory_path "$STORAGE_PATH" "$PARTIAL/storage-manifest.tsv"
|
snapshot_path "$STORAGE_PATH" storage
|
||||||
inventory_path "$MODELS_PATH" "$PARTIAL/models-manifest.tsv"
|
snapshot_path "$MODELS_PATH" models
|
||||||
|
|
||||||
python3 - "$PARTIAL/manifest.json" <<PY
|
python3 - "$PARTIAL/manifest.json" <<PY
|
||||||
import json
|
import json
|
||||||
@@ -190,16 +296,25 @@ payload = {
|
|||||||
"database_password_secure": ${PASSWORD_SECURE@Q} == "true",
|
"database_password_secure": ${PASSWORD_SECURE@Q} == "true",
|
||||||
"image_id": ${IMAGE_ID@Q},
|
"image_id": ${IMAGE_ID@Q},
|
||||||
"image_name": ${IMAGE_NAME@Q},
|
"image_name": ${IMAGE_NAME@Q},
|
||||||
"git_commit": ${GIT_COMMIT@Q},
|
"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None,
|
||||||
"git_dirty": ${GIT_DIRTY@Q} == "true",
|
"backup_tool_revision": ${SOURCE_REVISION@Q},
|
||||||
|
"backup_tool_revision_source": ${SOURCE_REVISION_SOURCE@Q},
|
||||||
|
"backup_tool_git_dirty": ${SOURCE_GIT_DIRTY@Q} == "true",
|
||||||
|
"running_image_revision": ${RUNNING_IMAGE_REVISION@Q},
|
||||||
|
"running_image_revision_is_full_sha": ${RUNNING_IMAGE_REVISION_IS_FULL_SHA@Q} == "true",
|
||||||
"inventory_mode": ${INVENTORY_MODE@Q},
|
"inventory_mode": ${INVENTORY_MODE@Q},
|
||||||
"storage_inventory_requested": bool(${STORAGE_PATH@Q}),
|
"storage_inventory_requested": bool(${STORAGE_PATH@Q}),
|
||||||
"models_inventory_requested": bool(${MODELS_PATH@Q}),
|
"models_inventory_requested": bool(${MODELS_PATH@Q}),
|
||||||
|
"storage_snapshot_requested": bool(${STORAGE_PATH@Q}),
|
||||||
|
"models_snapshot_requested": bool(${MODELS_PATH@Q}),
|
||||||
|
"link_dest_backup": ${LINK_DEST_BACKUP@Q} or None,
|
||||||
}
|
}
|
||||||
path = pathlib.Path(__import__("sys").argv[1])
|
path = pathlib.Path(__import__("sys").argv[1])
|
||||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
PY
|
PY
|
||||||
|
|
||||||
|
python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$PARTIAL"
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$PARTIAL"
|
cd "$PARTIAL"
|
||||||
find . -maxdepth 1 -type f ! -name CHECKSUMS.sha256 -printf '%f\n' \
|
find . -maxdepth 1 -type f ! -name CHECKSUMS.sha256 -printf '%f\n' \
|
||||||
|
|||||||
@@ -1,11 +1,15 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Dry-run-first cleanup for old unreferenced derived/cache artifacts."""
|
"""Dry-run-first quarantine for old unreferenced derived/cache artifacts."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
from audit_data_operations import build_report
|
from audit_data_operations import build_report
|
||||||
from release_backup_guard import require_confirmation, verify_current_backup
|
from release_backup_guard import require_confirmation, verify_current_backup
|
||||||
@@ -14,7 +18,21 @@ from app.core.config import get_settings
|
|||||||
from app.db.session import SessionLocal
|
from app.db.session import SessionLocal
|
||||||
|
|
||||||
|
|
||||||
CONFIRMATION = "DELETE_STORAGE_ARTIFACTS"
|
CONFIRMATION = "QUARANTINE_STORAGE_ARTIFACTS"
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def write_manifest(path: Path, payload: dict[str, object]) -> None:
|
||||||
|
temporary = path.with_suffix(".json.partial")
|
||||||
|
temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
def parse_args() -> argparse.Namespace:
|
||||||
@@ -26,6 +44,11 @@ def parse_args() -> argparse.Namespace:
|
|||||||
parser.add_argument("--confirm")
|
parser.add_argument("--confirm")
|
||||||
parser.add_argument("--backup-dir", type=Path)
|
parser.add_argument("--backup-dir", type=Path)
|
||||||
parser.add_argument("--backup-max-age-hours", type=float, default=24.0)
|
parser.add_argument("--backup-max-age-hours", type=float, default=24.0)
|
||||||
|
parser.add_argument(
|
||||||
|
"--quarantine-root",
|
||||||
|
type=Path,
|
||||||
|
help="Protected destination below the storage root (default: operator-evidence/cleanup-quarantine)",
|
||||||
|
)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
@@ -44,7 +67,8 @@ def main() -> int:
|
|||||||
|
|
||||||
blocked_reason = None
|
blocked_reason = None
|
||||||
backup = None
|
backup = None
|
||||||
deleted: list[str] = []
|
quarantined: list[dict[str, object]] = []
|
||||||
|
quarantine_manifest: Path | None = None
|
||||||
if args.apply:
|
if args.apply:
|
||||||
require_confirmation(args.confirm, CONFIRMATION)
|
require_confirmation(args.confirm, CONFIRMATION)
|
||||||
if args.backup_dir is None:
|
if args.backup_dir is None:
|
||||||
@@ -59,9 +83,80 @@ def main() -> int:
|
|||||||
"review the dry run and raise the explicit limit"
|
"review the dry run and raise the explicit limit"
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
|
quarantine_root = (
|
||||||
|
args.quarantine_root
|
||||||
|
or storage_root / "operator-evidence" / "cleanup-quarantine"
|
||||||
|
).resolve()
|
||||||
|
protected_quarantine_root = (
|
||||||
|
storage_root / "operator-evidence" / "cleanup-quarantine"
|
||||||
|
).resolve()
|
||||||
|
try:
|
||||||
|
quarantine_root.relative_to(protected_quarantine_root)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RuntimeError(
|
||||||
|
"--quarantine-root must remain below "
|
||||||
|
"operator-evidence/cleanup-quarantine in --storage-root"
|
||||||
|
) from exc
|
||||||
|
operation_id = f"cleanup-{datetime.now(timezone.utc):%Y%m%dT%H%M%SZ}-{uuid4().hex[:12]}"
|
||||||
|
operation_root = quarantine_root / operation_id
|
||||||
|
operation_root.mkdir(parents=True, exist_ok=False)
|
||||||
|
quarantine_manifest = operation_root / "manifest.json"
|
||||||
|
entries: list[dict[str, object]] = []
|
||||||
for candidate in candidates:
|
for candidate in candidates:
|
||||||
|
if candidate.path.is_symlink():
|
||||||
|
raise RuntimeError(f"Cleanup candidate became a symlink: {candidate.relative_path}")
|
||||||
|
try:
|
||||||
|
candidate.path.resolve().relative_to(storage_root)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Cleanup candidate escaped storage: {candidate.relative_path}"
|
||||||
|
) from exc
|
||||||
|
destination = operation_root / "files" / candidate.relative_path
|
||||||
|
current_size = candidate.path.stat().st_size
|
||||||
|
if current_size != candidate.size_bytes:
|
||||||
|
raise RuntimeError(f"Cleanup candidate changed size: {candidate.relative_path}")
|
||||||
|
entries.append(
|
||||||
|
{
|
||||||
|
"relative_path": candidate.relative_path,
|
||||||
|
"size_bytes": current_size,
|
||||||
|
"sha256": sha256(candidate.path),
|
||||||
|
"status": "planned",
|
||||||
|
"quarantine_relative_path": destination.relative_to(storage_root).as_posix(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
manifest: dict[str, object] = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"operation_id": operation_id,
|
||||||
|
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"state": "in_progress",
|
||||||
|
"storage_root": str(storage_root),
|
||||||
|
"backup_release_id": backup.release_id,
|
||||||
|
"entries": entries,
|
||||||
|
}
|
||||||
|
write_manifest(quarantine_manifest, manifest)
|
||||||
|
for candidate, entry in zip(candidates, entries, strict=True):
|
||||||
|
destination = storage_root / str(entry["quarantine_relative_path"])
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
try:
|
||||||
|
os.link(candidate.path, destination, follow_symlinks=False)
|
||||||
|
except FileExistsError as exc:
|
||||||
|
raise RuntimeError(f"Quarantine destination already exists: {destination}") from exc
|
||||||
|
if (
|
||||||
|
not destination.is_file()
|
||||||
|
or destination.stat().st_size != entry["size_bytes"]
|
||||||
|
or sha256(destination) != entry["sha256"]
|
||||||
|
):
|
||||||
|
destination.unlink(missing_ok=True)
|
||||||
|
raise RuntimeError(f"Quarantine link verification failed: {candidate.relative_path}")
|
||||||
|
entry["status"] = "linked"
|
||||||
|
write_manifest(quarantine_manifest, manifest)
|
||||||
candidate.path.unlink()
|
candidate.path.unlink()
|
||||||
deleted.append(candidate.relative_path)
|
entry["status"] = "quarantined"
|
||||||
|
quarantined.append(dict(entry))
|
||||||
|
write_manifest(quarantine_manifest, manifest)
|
||||||
|
manifest["state"] = "complete"
|
||||||
|
manifest["completed_at"] = datetime.now(timezone.utc).isoformat()
|
||||||
|
write_manifest(quarantine_manifest, manifest)
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"schema_version": 1,
|
"schema_version": 1,
|
||||||
@@ -72,8 +167,11 @@ def main() -> int:
|
|||||||
"candidate_count": len(candidates),
|
"candidate_count": len(candidates),
|
||||||
"candidate_bytes": sum(item.size_bytes for item in candidates),
|
"candidate_bytes": sum(item.size_bytes for item in candidates),
|
||||||
"candidates": [item.relative_path for item in candidates],
|
"candidates": [item.relative_path for item in candidates],
|
||||||
"deleted_count": len(deleted),
|
"deleted_count": 0,
|
||||||
"deleted": deleted,
|
"deleted": [],
|
||||||
|
"quarantined_count": len(quarantined),
|
||||||
|
"quarantined": quarantined,
|
||||||
|
"quarantine_manifest": str(quarantine_manifest) if quarantine_manifest else None,
|
||||||
"blocked_reason": blocked_reason,
|
"blocked_reason": blocked_reason,
|
||||||
"protected_prefixes": report["cleanup"]["protected_prefixes"],
|
"protected_prefixes": report["cleanup"]["protected_prefixes"],
|
||||||
"backup": (
|
"backup": (
|
||||||
@@ -81,7 +179,7 @@ def main() -> int:
|
|||||||
"release_id": backup.release_id,
|
"release_id": backup.release_id,
|
||||||
"created_at": backup.created_at.isoformat(),
|
"created_at": backup.created_at.isoformat(),
|
||||||
"age_hours": round(backup.age_hours, 3),
|
"age_hours": round(backup.age_hours, 3),
|
||||||
"git_commit": backup.git_commit,
|
"backup_tool_revision": backup.backup_tool_revision,
|
||||||
}
|
}
|
||||||
if backup
|
if backup
|
||||||
else None
|
else None
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ from dataclasses import dataclass
|
|||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from release_backup_snapshot import verify_backup as verify_byte_snapshots
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class VerifiedBackup:
|
class VerifiedBackup:
|
||||||
@@ -16,7 +18,7 @@ class VerifiedBackup:
|
|||||||
release_id: str
|
release_id: str
|
||||||
created_at: datetime
|
created_at: datetime
|
||||||
age_hours: float
|
age_hours: float
|
||||||
git_commit: str
|
backup_tool_revision: str
|
||||||
|
|
||||||
|
|
||||||
def _sha256(path: Path) -> str:
|
def _sha256(path: Path) -> str:
|
||||||
@@ -62,6 +64,8 @@ def verify_current_backup(
|
|||||||
missing = sorted(name for name in required if not (root / name).is_file())
|
missing = sorted(name for name in required if not (root / name).is_file())
|
||||||
if missing:
|
if missing:
|
||||||
raise RuntimeError(f"Backup is incomplete; missing: {', '.join(missing)}")
|
raise RuntimeError(f"Backup is incomplete; missing: {', '.join(missing)}")
|
||||||
|
if not (root / "storage-snapshot").is_dir():
|
||||||
|
raise RuntimeError("Backup is incomplete; missing: storage-snapshot")
|
||||||
|
|
||||||
checksum_lines = (root / "CHECKSUMS.sha256").read_text(encoding="utf-8").splitlines()
|
checksum_lines = (root / "CHECKSUMS.sha256").read_text(encoding="utf-8").splitlines()
|
||||||
checked: set[str] = set()
|
checked: set[str] = set()
|
||||||
@@ -93,6 +97,9 @@ def verify_current_backup(
|
|||||||
raise RuntimeError("Backup was made from an insecure database configuration")
|
raise RuntimeError("Backup was made from an insecure database configuration")
|
||||||
if manifest.get("inventory_mode") != "sha256" or manifest.get("storage_inventory_requested") is not True:
|
if manifest.get("inventory_mode") != "sha256" or manifest.get("storage_inventory_requested") is not True:
|
||||||
raise RuntimeError("Destructive maintenance requires a SHA-256 storage inventory backup")
|
raise RuntimeError("Destructive maintenance requires a SHA-256 storage inventory backup")
|
||||||
|
if manifest.get("storage_snapshot_requested") is not True:
|
||||||
|
raise RuntimeError("Destructive maintenance requires a byte-complete storage snapshot")
|
||||||
|
verify_byte_snapshots(root)
|
||||||
|
|
||||||
created = _created_at(manifest.get("created_at"))
|
created = _created_at(manifest.get("created_at"))
|
||||||
current = now or datetime.now(timezone.utc)
|
current = now or datetime.now(timezone.utc)
|
||||||
@@ -107,17 +114,17 @@ def verify_current_backup(
|
|||||||
)
|
)
|
||||||
|
|
||||||
release_id = manifest.get("release_id")
|
release_id = manifest.get("release_id")
|
||||||
git_commit = manifest.get("git_commit")
|
backup_tool_revision = manifest.get("backup_tool_revision", manifest.get("git_commit"))
|
||||||
if not isinstance(release_id, str) or not release_id:
|
if not isinstance(release_id, str) or not release_id:
|
||||||
raise RuntimeError("Backup release id is missing")
|
raise RuntimeError("Backup release id is missing")
|
||||||
if not isinstance(git_commit, str) or len(git_commit) < 7:
|
if not isinstance(backup_tool_revision, str) or len(backup_tool_revision) < 7:
|
||||||
raise RuntimeError("Backup Git commit is missing")
|
raise RuntimeError("Backup tool revision is missing")
|
||||||
return VerifiedBackup(
|
return VerifiedBackup(
|
||||||
backup_dir=root,
|
backup_dir=root,
|
||||||
release_id=release_id,
|
release_id=release_id,
|
||||||
created_at=created,
|
created_at=created,
|
||||||
age_hours=age_hours,
|
age_hours=age_hours,
|
||||||
git_commit=git_commit,
|
backup_tool_revision=backup_tool_revision,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,376 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Create and verify byte-complete, symlink-safe release backup snapshots."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import errno
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
|
||||||
|
|
||||||
|
MANIFEST_HEADER = "relative_path\tsize_bytes\tmtime_ns\tsha256"
|
||||||
|
SAFE_LABEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
|
||||||
|
FICLONE = 0x40049409
|
||||||
|
FICLONE_FALLBACK_ERRORS = {
|
||||||
|
errno.EXDEV,
|
||||||
|
errno.EOPNOTSUPP,
|
||||||
|
errno.ENOTTY,
|
||||||
|
errno.EINVAL,
|
||||||
|
errno.ENOSYS,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SourceEntry:
|
||||||
|
path: Path
|
||||||
|
relative_path: str
|
||||||
|
stat_result: os.stat_result
|
||||||
|
is_directory: bool
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ManifestEntry:
|
||||||
|
relative_path: str
|
||||||
|
size_bytes: int
|
||||||
|
mtime_ns: int
|
||||||
|
sha256: str
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_relative(value: str) -> str:
|
||||||
|
if not value or "\t" in value or "\n" in value or "\r" in value:
|
||||||
|
raise RuntimeError(f"Unsupported snapshot path: {value!r}")
|
||||||
|
candidate = PurePosixPath(value)
|
||||||
|
if candidate.is_absolute() or any(part in {"", ".", ".."} for part in candidate.parts):
|
||||||
|
raise RuntimeError(f"Unsafe snapshot path: {value!r}")
|
||||||
|
return candidate.as_posix()
|
||||||
|
|
||||||
|
|
||||||
|
def _collect(root: Path) -> list[SourceEntry]:
|
||||||
|
entries: list[SourceEntry] = []
|
||||||
|
for current, directory_names, file_names in os.walk(root, topdown=True, followlinks=False):
|
||||||
|
directory_names.sort()
|
||||||
|
file_names.sort()
|
||||||
|
current_path = Path(current)
|
||||||
|
for name, is_directory in [
|
||||||
|
*((name, True) for name in directory_names),
|
||||||
|
*((name, False) for name in file_names),
|
||||||
|
]:
|
||||||
|
path = current_path / name
|
||||||
|
details = path.lstat()
|
||||||
|
relative = _safe_relative(path.relative_to(root).as_posix())
|
||||||
|
if stat.S_ISLNK(details.st_mode):
|
||||||
|
raise RuntimeError(f"Release snapshot refuses symlinked content: {relative}")
|
||||||
|
if is_directory and not stat.S_ISDIR(details.st_mode):
|
||||||
|
raise RuntimeError(f"Snapshot directory changed during inventory: {relative}")
|
||||||
|
if not is_directory and not stat.S_ISREG(details.st_mode):
|
||||||
|
raise RuntimeError(f"Release snapshot refuses non-regular content: {relative}")
|
||||||
|
entries.append(SourceEntry(path, relative, details, is_directory))
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def _same_file_state(before: os.stat_result, after: os.stat_result) -> bool:
|
||||||
|
return (
|
||||||
|
before.st_dev,
|
||||||
|
before.st_ino,
|
||||||
|
before.st_size,
|
||||||
|
before.st_mtime_ns,
|
||||||
|
) == (
|
||||||
|
after.st_dev,
|
||||||
|
after.st_ino,
|
||||||
|
after.st_size,
|
||||||
|
after.st_mtime_ns,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _copy_all(source_descriptor: int, destination_descriptor: int) -> None:
|
||||||
|
while True:
|
||||||
|
value = os.read(source_descriptor, 1024 * 1024)
|
||||||
|
if not value:
|
||||||
|
return
|
||||||
|
view = memoryview(value)
|
||||||
|
while view:
|
||||||
|
written = os.write(destination_descriptor, view)
|
||||||
|
if written <= 0:
|
||||||
|
raise RuntimeError("Snapshot copy stopped before writing all bytes")
|
||||||
|
view = view[written:]
|
||||||
|
|
||||||
|
|
||||||
|
def _clone_or_copy(entry: SourceEntry, destination: Path) -> ManifestEntry:
|
||||||
|
source_flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
||||||
|
destination_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0)
|
||||||
|
source_descriptor = os.open(entry.path, source_flags)
|
||||||
|
destination_descriptor = -1
|
||||||
|
try:
|
||||||
|
opened = os.fstat(source_descriptor)
|
||||||
|
if not stat.S_ISREG(opened.st_mode) or not _same_file_state(entry.stat_result, opened):
|
||||||
|
raise RuntimeError(f"Snapshot file changed before copying: {entry.relative_path}")
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
destination_descriptor = os.open(destination, destination_flags, stat.S_IMODE(opened.st_mode))
|
||||||
|
cloned = False
|
||||||
|
if os.name == "posix":
|
||||||
|
try:
|
||||||
|
import fcntl
|
||||||
|
|
||||||
|
fcntl.ioctl(destination_descriptor, FICLONE, source_descriptor)
|
||||||
|
cloned = True
|
||||||
|
except OSError as exc:
|
||||||
|
if exc.errno not in FICLONE_FALLBACK_ERRORS:
|
||||||
|
raise
|
||||||
|
if not cloned:
|
||||||
|
os.lseek(source_descriptor, 0, os.SEEK_SET)
|
||||||
|
os.ftruncate(destination_descriptor, 0)
|
||||||
|
_copy_all(source_descriptor, destination_descriptor)
|
||||||
|
os.fsync(destination_descriptor)
|
||||||
|
after = os.fstat(source_descriptor)
|
||||||
|
if not _same_file_state(opened, after):
|
||||||
|
raise RuntimeError(f"Snapshot file changed while copying: {entry.relative_path}")
|
||||||
|
except BaseException:
|
||||||
|
if destination_descriptor >= 0:
|
||||||
|
os.close(destination_descriptor)
|
||||||
|
destination_descriptor = -1
|
||||||
|
destination.unlink(missing_ok=True)
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
if destination_descriptor >= 0:
|
||||||
|
os.close(destination_descriptor)
|
||||||
|
os.close(source_descriptor)
|
||||||
|
|
||||||
|
os.chmod(destination, stat.S_IMODE(entry.stat_result.st_mode) & ~0o222, follow_symlinks=False)
|
||||||
|
retained_times = (entry.stat_result.st_atime_ns, entry.stat_result.st_mtime_ns)
|
||||||
|
try:
|
||||||
|
os.utime(destination, ns=retained_times, follow_symlinks=False)
|
||||||
|
except NotImplementedError:
|
||||||
|
# Windows does not expose no-follow utime. The destination was created
|
||||||
|
# exclusively above; recheck it before using the portable call.
|
||||||
|
if destination.is_symlink():
|
||||||
|
destination.unlink(missing_ok=True)
|
||||||
|
raise RuntimeError(f"Snapshot destination became a symlink: {entry.relative_path}")
|
||||||
|
os.utime(destination, ns=retained_times)
|
||||||
|
source_checksum = _sha256(entry.path)
|
||||||
|
snapshot_checksum = _sha256(destination)
|
||||||
|
final_source = entry.path.lstat()
|
||||||
|
if not _same_file_state(entry.stat_result, final_source):
|
||||||
|
raise RuntimeError(f"Snapshot file changed during checksum verification: {entry.relative_path}")
|
||||||
|
if source_checksum != snapshot_checksum:
|
||||||
|
raise RuntimeError(f"Snapshot checksum differs from source: {entry.relative_path}")
|
||||||
|
return ManifestEntry(
|
||||||
|
relative_path=entry.relative_path,
|
||||||
|
size_bytes=entry.stat_result.st_size,
|
||||||
|
mtime_ns=entry.stat_result.st_mtime_ns,
|
||||||
|
sha256=snapshot_checksum,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _link_verified_prior(
|
||||||
|
entry: SourceEntry,
|
||||||
|
destination: Path,
|
||||||
|
prior_root: Path,
|
||||||
|
prior_manifest: dict[str, ManifestEntry],
|
||||||
|
) -> ManifestEntry | None:
|
||||||
|
retained = prior_manifest.get(entry.relative_path)
|
||||||
|
if retained is None or retained.size_bytes != entry.stat_result.st_size:
|
||||||
|
return None
|
||||||
|
source_checksum = _sha256(entry.path)
|
||||||
|
final_source = entry.path.lstat()
|
||||||
|
if not _same_file_state(entry.stat_result, final_source):
|
||||||
|
raise RuntimeError(f"Snapshot file changed during prior comparison: {entry.relative_path}")
|
||||||
|
if source_checksum != retained.sha256:
|
||||||
|
return None
|
||||||
|
prior_path = prior_root / entry.relative_path
|
||||||
|
try:
|
||||||
|
prior_details = prior_path.lstat()
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None
|
||||||
|
if not stat.S_ISREG(prior_details.st_mode) or prior_details.st_size != retained.size_bytes:
|
||||||
|
raise RuntimeError(f"Prior snapshot file is not reusable: {entry.relative_path}")
|
||||||
|
if _sha256(prior_path) != retained.sha256:
|
||||||
|
raise RuntimeError(f"Prior snapshot checksum changed: {entry.relative_path}")
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
os.link(prior_path, destination, follow_symlinks=False)
|
||||||
|
if destination.stat().st_size != retained.size_bytes or _sha256(destination) != retained.sha256:
|
||||||
|
destination.unlink(missing_ok=True)
|
||||||
|
raise RuntimeError(f"Hard-linked snapshot verification failed: {entry.relative_path}")
|
||||||
|
return ManifestEntry(
|
||||||
|
relative_path=entry.relative_path,
|
||||||
|
size_bytes=entry.stat_result.st_size,
|
||||||
|
mtime_ns=entry.stat_result.st_mtime_ns,
|
||||||
|
sha256=source_checksum,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def read_manifest(path: Path) -> dict[str, ManifestEntry]:
|
||||||
|
lines = path.read_text(encoding="utf-8").splitlines()
|
||||||
|
if not lines or lines[0] != MANIFEST_HEADER:
|
||||||
|
raise RuntimeError(f"Snapshot inventory has an invalid header: {path}")
|
||||||
|
entries: dict[str, ManifestEntry] = {}
|
||||||
|
for line in lines[1:]:
|
||||||
|
fields = line.split("\t")
|
||||||
|
if len(fields) != 4:
|
||||||
|
raise RuntimeError(f"Snapshot inventory has an invalid row: {line!r}")
|
||||||
|
relative_path, size_text, mtime_text, checksum = fields
|
||||||
|
relative_path = _safe_relative(relative_path)
|
||||||
|
if relative_path in entries:
|
||||||
|
raise RuntimeError(f"Snapshot inventory repeats a path: {relative_path}")
|
||||||
|
try:
|
||||||
|
size_bytes = int(size_text)
|
||||||
|
mtime_ns = int(mtime_text)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RuntimeError(f"Snapshot inventory has invalid metadata: {relative_path}") from exc
|
||||||
|
if size_bytes < 0 or mtime_ns < 0 or not re.fullmatch(r"[0-9a-f]{64}", checksum):
|
||||||
|
raise RuntimeError(f"Snapshot inventory has invalid retained state: {relative_path}")
|
||||||
|
entries[relative_path] = ManifestEntry(relative_path, size_bytes, mtime_ns, checksum)
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def verify_snapshot(snapshot_path: Path, manifest_path: Path) -> None:
|
||||||
|
root = snapshot_path.expanduser().resolve(strict=True)
|
||||||
|
if not root.is_dir():
|
||||||
|
raise RuntimeError(f"Snapshot path is not a directory: {root}")
|
||||||
|
expected = read_manifest(manifest_path)
|
||||||
|
observed_entries = _collect(root)
|
||||||
|
observed_files = {item.relative_path: item for item in observed_entries if not item.is_directory}
|
||||||
|
extra = sorted(set(observed_files) - set(expected))
|
||||||
|
missing = sorted(set(expected) - set(observed_files))
|
||||||
|
if extra:
|
||||||
|
raise RuntimeError(f"Snapshot contains unmanifested files: {', '.join(extra[:10])}")
|
||||||
|
if missing:
|
||||||
|
raise RuntimeError(f"Snapshot omits manifested files: {', '.join(missing[:10])}")
|
||||||
|
for relative, retained in expected.items():
|
||||||
|
current = observed_files[relative]
|
||||||
|
if current.stat_result.st_size != retained.size_bytes:
|
||||||
|
raise RuntimeError(f"Snapshot size differs for: {relative}")
|
||||||
|
if _sha256(current.path) != retained.sha256:
|
||||||
|
raise RuntimeError(f"Snapshot checksum differs for: {relative}")
|
||||||
|
|
||||||
|
|
||||||
|
def create_snapshot(
|
||||||
|
source: Path,
|
||||||
|
snapshot_path: Path,
|
||||||
|
manifest_path: Path,
|
||||||
|
*,
|
||||||
|
label: str,
|
||||||
|
link_dest_snapshot: Path | None = None,
|
||||||
|
link_dest_manifest: Path | None = None,
|
||||||
|
) -> None:
|
||||||
|
if not SAFE_LABEL.fullmatch(label):
|
||||||
|
raise RuntimeError(f"Unsafe snapshot label: {label!r}")
|
||||||
|
root = source.expanduser()
|
||||||
|
if root.is_symlink():
|
||||||
|
raise RuntimeError(f"Release snapshot refuses a symlinked root: {root}")
|
||||||
|
root = root.resolve(strict=True)
|
||||||
|
if not root.is_dir():
|
||||||
|
raise RuntimeError(f"Snapshot source is not a directory: {root}")
|
||||||
|
snapshot_path = snapshot_path.expanduser().resolve()
|
||||||
|
manifest_path = manifest_path.expanduser().resolve()
|
||||||
|
for output in (snapshot_path, manifest_path):
|
||||||
|
try:
|
||||||
|
output.relative_to(root)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
raise RuntimeError("Release snapshot output must not be inside its source tree")
|
||||||
|
if snapshot_path.exists():
|
||||||
|
raise RuntimeError(f"Snapshot destination already exists: {snapshot_path}")
|
||||||
|
snapshot_path.mkdir(parents=True, exist_ok=False)
|
||||||
|
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
prior_root: Path | None = None
|
||||||
|
prior_manifest: dict[str, ManifestEntry] = {}
|
||||||
|
if (link_dest_snapshot is None) != (link_dest_manifest is None):
|
||||||
|
raise RuntimeError("Prior snapshot and manifest must be supplied together")
|
||||||
|
if link_dest_snapshot is not None and link_dest_manifest is not None:
|
||||||
|
prior_root = link_dest_snapshot.expanduser().resolve(strict=True)
|
||||||
|
prior_manifest = read_manifest(link_dest_manifest.expanduser().resolve(strict=True))
|
||||||
|
|
||||||
|
initial = _collect(root)
|
||||||
|
retained: list[ManifestEntry] = []
|
||||||
|
for entry in initial:
|
||||||
|
destination = snapshot_path / entry.relative_path
|
||||||
|
if entry.is_directory:
|
||||||
|
destination.mkdir(parents=True, exist_ok=False)
|
||||||
|
os.chmod(destination, stat.S_IMODE(entry.stat_result.st_mode), follow_symlinks=False)
|
||||||
|
continue
|
||||||
|
linked = (
|
||||||
|
_link_verified_prior(entry, destination, prior_root, prior_manifest)
|
||||||
|
if prior_root is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
retained.append(linked or _clone_or_copy(entry, destination))
|
||||||
|
final = _collect(root)
|
||||||
|
if [(item.relative_path, item.is_directory) for item in initial] != [
|
||||||
|
(item.relative_path, item.is_directory) for item in final
|
||||||
|
]:
|
||||||
|
raise RuntimeError(f"Snapshot source contents changed while backup was running: {root}")
|
||||||
|
with manifest_path.open("w", encoding="utf-8", newline="\n") as handle:
|
||||||
|
handle.write(f"{MANIFEST_HEADER}\n")
|
||||||
|
for entry in retained:
|
||||||
|
handle.write(
|
||||||
|
f"{entry.relative_path}\t{entry.size_bytes}\t{entry.mtime_ns}\t{entry.sha256}\n"
|
||||||
|
)
|
||||||
|
verify_snapshot(snapshot_path, manifest_path)
|
||||||
|
|
||||||
|
|
||||||
|
def verify_backup(backup_dir: Path) -> None:
|
||||||
|
root = backup_dir.expanduser().resolve(strict=True)
|
||||||
|
payload = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
|
||||||
|
for label in ("storage", "models"):
|
||||||
|
requested = payload.get(f"{label}_inventory_requested") is True
|
||||||
|
snapshotted = payload.get(f"{label}_snapshot_requested") is True
|
||||||
|
if requested != snapshotted:
|
||||||
|
raise RuntimeError(f"Backup manifest does not bind the {label} inventory to a snapshot")
|
||||||
|
if requested:
|
||||||
|
verify_snapshot(root / f"{label}-snapshot", root / f"{label}-manifest.tsv")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||||
|
create = subparsers.add_parser("create")
|
||||||
|
create.add_argument("--source", type=Path, required=True)
|
||||||
|
create.add_argument("--snapshot", type=Path, required=True)
|
||||||
|
create.add_argument("--manifest", type=Path, required=True)
|
||||||
|
create.add_argument("--label", required=True)
|
||||||
|
create.add_argument("--link-dest-snapshot", type=Path)
|
||||||
|
create.add_argument("--link-dest-manifest", type=Path)
|
||||||
|
verify = subparsers.add_parser("verify")
|
||||||
|
verify.add_argument("--snapshot", type=Path, required=True)
|
||||||
|
verify.add_argument("--manifest", type=Path, required=True)
|
||||||
|
verify_backup_parser = subparsers.add_parser("verify-backup")
|
||||||
|
verify_backup_parser.add_argument("--backup-dir", type=Path, required=True)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = parse_args()
|
||||||
|
if args.command == "create":
|
||||||
|
create_snapshot(
|
||||||
|
args.source,
|
||||||
|
args.snapshot,
|
||||||
|
args.manifest,
|
||||||
|
label=args.label,
|
||||||
|
link_dest_snapshot=args.link_dest_snapshot,
|
||||||
|
link_dest_manifest=args.link_dest_manifest,
|
||||||
|
)
|
||||||
|
elif args.command == "verify":
|
||||||
|
verify_snapshot(args.snapshot, args.manifest)
|
||||||
|
else:
|
||||||
|
verify_backup(args.backup_dir)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Restore a traceable GeoIntel cleanup quarantine without overwriting data."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
CONFIRMATION = "RESTORE_QUARANTINED_ARTIFACTS"
|
||||||
|
CLEANUP_PREFIXES = ("exports", "previews", "tiles", "masks", "derived", "rasters/derived")
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def write_manifest(path: Path, payload: dict[str, object]) -> None:
|
||||||
|
temporary = path.with_suffix(".json.partial")
|
||||||
|
temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--storage-root", type=Path, required=True)
|
||||||
|
parser.add_argument("--manifest", type=Path, required=True)
|
||||||
|
parser.add_argument("--confirm", required=True)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def _within(path: Path, root: Path, *, label: str) -> Path:
|
||||||
|
resolved = path.resolve()
|
||||||
|
try:
|
||||||
|
resolved.relative_to(root)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RuntimeError(f"{label} escapes the storage root") from exc
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = parse_args()
|
||||||
|
if args.confirm != CONFIRMATION:
|
||||||
|
raise RuntimeError(f"Refusing restore; pass --confirm {CONFIRMATION}")
|
||||||
|
storage_root = args.storage_root.expanduser().resolve()
|
||||||
|
manifest_path = _within(args.manifest.expanduser(), storage_root, label="Manifest")
|
||||||
|
protected_quarantine_root = storage_root / "operator-evidence" / "cleanup-quarantine"
|
||||||
|
try:
|
||||||
|
manifest_path.relative_to(protected_quarantine_root.resolve())
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RuntimeError("Manifest is outside the protected cleanup quarantine") from exc
|
||||||
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
if payload.get("schema_version") != 1:
|
||||||
|
raise RuntimeError("Unsupported quarantine manifest")
|
||||||
|
if payload.get("state") not in {"complete", "in_progress", "restore_in_progress", "restored"}:
|
||||||
|
raise RuntimeError("Quarantine manifest is not in a restorable state")
|
||||||
|
raw_entries = payload.get("entries")
|
||||||
|
if not isinstance(raw_entries, list):
|
||||||
|
raise RuntimeError("Quarantine manifest entries are invalid")
|
||||||
|
|
||||||
|
plans: list[tuple[str, dict[str, object], Path, Path]] = []
|
||||||
|
for raw_entry in raw_entries:
|
||||||
|
if not isinstance(raw_entry, dict):
|
||||||
|
raise RuntimeError("Quarantine manifest entry is invalid")
|
||||||
|
status = raw_entry.get("status")
|
||||||
|
if status not in {"planned", "linked", "quarantined", "restore_linked", "restored"}:
|
||||||
|
raise RuntimeError(f"Quarantine manifest entry has an invalid status: {status!r}")
|
||||||
|
relative_path = raw_entry.get("relative_path")
|
||||||
|
quarantine_relative_path = raw_entry.get("quarantine_relative_path")
|
||||||
|
expected_hash = raw_entry.get("sha256")
|
||||||
|
expected_size = raw_entry.get("size_bytes")
|
||||||
|
if (
|
||||||
|
not isinstance(relative_path, str)
|
||||||
|
or not isinstance(expected_hash, str)
|
||||||
|
or not isinstance(expected_size, int)
|
||||||
|
):
|
||||||
|
raise RuntimeError("Quarantine manifest entry lacks recovery metadata")
|
||||||
|
if not any(
|
||||||
|
relative_path == prefix or relative_path.startswith(f"{prefix}/")
|
||||||
|
for prefix in CLEANUP_PREFIXES
|
||||||
|
):
|
||||||
|
raise RuntimeError(f"Original path is outside the cleanup allowlist: {relative_path}")
|
||||||
|
original = _within(storage_root / relative_path, storage_root, label="Original path")
|
||||||
|
if not isinstance(quarantine_relative_path, str):
|
||||||
|
if status != "planned":
|
||||||
|
raise RuntimeError("Quarantine manifest entry lacks its retained path")
|
||||||
|
quarantine_relative_path = (
|
||||||
|
manifest_path.parent / "files" / relative_path
|
||||||
|
).relative_to(storage_root).as_posix()
|
||||||
|
raw_entry["quarantine_relative_path"] = quarantine_relative_path
|
||||||
|
quarantined = _within(
|
||||||
|
storage_root / quarantine_relative_path,
|
||||||
|
storage_root,
|
||||||
|
label="Quarantine path",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
quarantined.relative_to(manifest_path.parent.resolve())
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RuntimeError("Quarantine entry escapes its operation directory") from exc
|
||||||
|
original_exists = original.exists()
|
||||||
|
quarantined_exists = quarantined.exists()
|
||||||
|
if original_exists:
|
||||||
|
if not original.is_file() or original.stat().st_size != expected_size or sha256(original) != expected_hash:
|
||||||
|
raise RuntimeError(f"Restore destination already exists with different bytes: {relative_path}")
|
||||||
|
if quarantined_exists:
|
||||||
|
if (
|
||||||
|
not quarantined.is_file()
|
||||||
|
or quarantined.stat().st_size != expected_size
|
||||||
|
or sha256(quarantined) != expected_hash
|
||||||
|
):
|
||||||
|
raise RuntimeError(f"Quarantined artifact checksum mismatch: {quarantine_relative_path}")
|
||||||
|
if original_exists and quarantined_exists:
|
||||||
|
if not os.path.samefile(original, quarantined):
|
||||||
|
raise RuntimeError(f"Restore destination already exists: {relative_path}")
|
||||||
|
plans.append(("remove_duplicate_link", raw_entry, quarantined, original))
|
||||||
|
elif original_exists:
|
||||||
|
plans.append(("mark_restored", raw_entry, quarantined, original))
|
||||||
|
elif quarantined_exists:
|
||||||
|
plans.append(("restore", raw_entry, quarantined, original))
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f"Both original and quarantined artifacts are missing: {relative_path}")
|
||||||
|
|
||||||
|
payload["state"] = "restore_in_progress"
|
||||||
|
write_manifest(manifest_path, payload)
|
||||||
|
for action, entry, quarantined, original in plans:
|
||||||
|
if action == "restore":
|
||||||
|
original.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
_within(original, storage_root, label="Original path")
|
||||||
|
try:
|
||||||
|
os.link(quarantined, original, follow_symlinks=False)
|
||||||
|
except FileExistsError as exc:
|
||||||
|
raise RuntimeError(f"Restore destination was created concurrently: {original}") from exc
|
||||||
|
if not os.path.samefile(quarantined, original):
|
||||||
|
original.unlink(missing_ok=True)
|
||||||
|
raise RuntimeError(f"Restore link verification failed: {original}")
|
||||||
|
entry["status"] = "restore_linked"
|
||||||
|
write_manifest(manifest_path, payload)
|
||||||
|
quarantined.unlink()
|
||||||
|
elif action == "remove_duplicate_link":
|
||||||
|
quarantined.unlink()
|
||||||
|
entry["status"] = "restored"
|
||||||
|
entry["restored_at"] = datetime.now(timezone.utc).isoformat()
|
||||||
|
write_manifest(manifest_path, payload)
|
||||||
|
payload["state"] = "restored"
|
||||||
|
payload["restored_at"] = datetime.now(timezone.utc).isoformat()
|
||||||
|
write_manifest(manifest_path, payload)
|
||||||
|
print(json.dumps({"state": "restored", "restored_count": len(plans)}, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -43,6 +43,8 @@ echo "== GeoIntel run readiness check =="
|
|||||||
"$PYTHON_BIN" -m py_compile scripts/build_release_package.py
|
"$PYTHON_BIN" -m py_compile scripts/build_release_package.py
|
||||||
"$PYTHON_BIN" -m py_compile scripts/verify_python_lock.py
|
"$PYTHON_BIN" -m py_compile scripts/verify_python_lock.py
|
||||||
"$PYTHON_BIN" scripts/verify_python_lock.py
|
"$PYTHON_BIN" scripts/verify_python_lock.py
|
||||||
|
"$PYTHON_BIN" -m py_compile scripts/verify_repository_layout.py
|
||||||
|
"$PYTHON_BIN" scripts/verify_repository_layout.py
|
||||||
"$PYTHON_BIN" -m py_compile scripts/verify_security_exceptions.py
|
"$PYTHON_BIN" -m py_compile scripts/verify_security_exceptions.py
|
||||||
"$PYTHON_BIN" scripts/verify_security_exceptions.py
|
"$PYTHON_BIN" scripts/verify_security_exceptions.py
|
||||||
bash -n scripts/backup_release_state.sh
|
bash -n scripts/backup_release_state.sh
|
||||||
@@ -55,6 +57,7 @@ bash -n scripts/scan_container_image.sh
|
|||||||
bash -n scripts/audit_python_dependencies.sh
|
bash -n scripts/audit_python_dependencies.sh
|
||||||
echo "Using Python: ${PYTHON_BIN}"
|
echo "Using Python: ${PYTHON_BIN}"
|
||||||
bash scripts/check_repo_structure.sh
|
bash scripts/check_repo_structure.sh
|
||||||
|
"$PYTHON_BIN" -m ruff check backend scripts tests
|
||||||
${PYTHON_BIN} scripts/smoke_docs.py
|
${PYTHON_BIN} scripts/smoke_docs.py
|
||||||
${PYTHON_BIN} scripts/validate_fixtures.py
|
${PYTHON_BIN} scripts/validate_fixtures.py
|
||||||
${PYTHON_BIN} scripts/smoke_contracts.py
|
${PYTHON_BIN} scripts/smoke_contracts.py
|
||||||
@@ -129,8 +132,10 @@ ${PYTHON_BIN} -m py_compile scripts/migrate_runtime_model_provenance.py
|
|||||||
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/archive_technical_projects.py
|
${PYTHON_BIN} -m py_compile scripts/archive_technical_projects.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/release_backup_guard.py
|
${PYTHON_BIN} -m py_compile scripts/release_backup_guard.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/release_backup_snapshot.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/audit_data_operations.py
|
${PYTHON_BIN} -m py_compile scripts/audit_data_operations.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/cleanup_storage_artifacts.py
|
${PYTHON_BIN} -m py_compile scripts/cleanup_storage_artifacts.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/restore_storage_quarantine.py
|
||||||
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
||||||
${PYTHON_BIN} -m compileall backend/app
|
${PYTHON_BIN} -m compileall backend/app
|
||||||
(cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning)
|
(cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning)
|
||||||
@@ -149,6 +154,7 @@ bash -n deploy/unraid/gosu-setpriv
|
|||||||
bash -n deploy/unraid/run-dockerman-container.sh
|
bash -n deploy/unraid/run-dockerman-container.sh
|
||||||
bash -n deploy/unraid/deploy-release.sh
|
bash -n deploy/unraid/deploy-release.sh
|
||||||
bash -n deploy/unraid/rollback-dockerman-container.sh
|
bash -n deploy/unraid/rollback-dockerman-container.sh
|
||||||
|
bash -n deploy/unraid/restore-predeploy-database.sh
|
||||||
bash -n scripts/verify_browser_runtime.sh
|
bash -n scripts/verify_browser_runtime.sh
|
||||||
bash -n scripts/verify_demo_export_workflow.sh
|
bash -n scripts/verify_demo_export_workflow.sh
|
||||||
bash -n scripts/verify_demo_raster_workflow.sh
|
bash -n scripts/verify_demo_raster_workflow.sh
|
||||||
|
|||||||
@@ -21,8 +21,17 @@ esac
|
|||||||
docker image inspect "$TARGET_IMAGE" >/dev/null
|
docker image inspect "$TARGET_IMAGE" >/dev/null
|
||||||
mkdir -p "$ROOT/$(dirname "$OUTPUT")" "$CACHE_DIR"
|
mkdir -p "$ROOT/$(dirname "$OUTPUT")" "$CACHE_DIR"
|
||||||
"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py"
|
"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py"
|
||||||
"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" \
|
mapfile -t ignored_container_ids < <(
|
||||||
--print-container-ids > "$IGNORE_FILE"
|
"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" \
|
||||||
|
--print-container-ids | tr -d '\r'
|
||||||
|
)
|
||||||
|
ignore_args=()
|
||||||
|
trivy_ignore_args=()
|
||||||
|
if [ "${#ignored_container_ids[@]}" -gt 0 ]; then
|
||||||
|
printf '%s\n' "${ignored_container_ids[@]}" > "$IGNORE_FILE"
|
||||||
|
ignore_args=(-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro")
|
||||||
|
trivy_ignore_args=(--ignorefile "$CONTAINER_IGNORE_FILE")
|
||||||
|
fi
|
||||||
|
|
||||||
# Keep the complete report, including vulnerabilities without an available fix.
|
# Keep the complete report, including vulnerabilities without an available fix.
|
||||||
docker run --rm \
|
docker run --rm \
|
||||||
@@ -45,14 +54,14 @@ docker run --rm \
|
|||||||
docker run --rm \
|
docker run --rm \
|
||||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||||
-v "$CACHE_DIR:/root/.cache/trivy" \
|
-v "$CACHE_DIR:/root/.cache/trivy" \
|
||||||
-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro" \
|
"${ignore_args[@]}" \
|
||||||
"$TRIVY_IMAGE" \
|
"$TRIVY_IMAGE" \
|
||||||
image \
|
image \
|
||||||
--scanners vuln \
|
--scanners vuln \
|
||||||
--timeout 20m \
|
--timeout 20m \
|
||||||
--skip-version-check \
|
--skip-version-check \
|
||||||
--ignore-unfixed \
|
--ignore-unfixed \
|
||||||
--ignorefile "$CONTAINER_IGNORE_FILE" \
|
"${trivy_ignore_args[@]}" \
|
||||||
--skip-files /usr/local/bin/gosu \
|
--skip-files /usr/local/bin/gosu \
|
||||||
--severity HIGH,CRITICAL \
|
--severity HIGH,CRITICAL \
|
||||||
--exit-code 1 \
|
--exit-code 1 \
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
CONTAINER="geointel"
|
CONTAINER="geointel"
|
||||||
BACKUP_DIR=""
|
BACKUP_DIR=""
|
||||||
|
|
||||||
@@ -59,15 +60,22 @@ required = {
|
|||||||
"database_name",
|
"database_name",
|
||||||
"database_user",
|
"database_user",
|
||||||
"image_id",
|
"image_id",
|
||||||
"git_commit",
|
|
||||||
}
|
}
|
||||||
missing = sorted(required - payload.keys())
|
missing = sorted(required - payload.keys())
|
||||||
if missing:
|
if missing:
|
||||||
raise SystemExit(f"Backup manifest misses: {', '.join(missing)}")
|
raise SystemExit(f"Backup manifest misses: {', '.join(missing)}")
|
||||||
if payload["schema_version"] != 1 or payload["read_only_source"] is not True:
|
if payload["schema_version"] != 1 or payload["read_only_source"] is not True:
|
||||||
raise SystemExit("Unsupported or unsafe backup manifest")
|
raise SystemExit("Unsupported or unsafe backup manifest")
|
||||||
|
tool_revision = payload.get("backup_tool_revision", payload.get("git_commit"))
|
||||||
|
if not isinstance(tool_revision, str) or len(tool_revision) < 7:
|
||||||
|
raise SystemExit("Backup manifest lacks its backup-tool revision")
|
||||||
|
running_revision = payload.get("running_image_revision")
|
||||||
|
if running_revision is not None and not isinstance(running_revision, str):
|
||||||
|
raise SystemExit("Backup manifest has an invalid running-image revision")
|
||||||
PY
|
PY
|
||||||
|
|
||||||
|
python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$BACKUP_DIR"
|
||||||
|
|
||||||
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||||
echo "Container '$CONTAINER' is required to run pg_restore --list." >&2
|
echo "Container '$CONTAINER' is required to run pg_restore --list." >&2
|
||||||
exit 3
|
exit 3
|
||||||
|
|||||||
@@ -17,29 +17,39 @@ EXCEPTIONS_PATH = ROOT / "security" / "pip-audit-exceptions.json"
|
|||||||
def load_and_validate() -> tuple[dict[str, object], list[str]]:
|
def load_and_validate() -> tuple[dict[str, object], list[str]]:
|
||||||
payload = json.loads(EXCEPTIONS_PATH.read_text(encoding="utf-8"))
|
payload = json.loads(EXCEPTIONS_PATH.read_text(encoding="utf-8"))
|
||||||
errors: list[str] = []
|
errors: list[str] = []
|
||||||
try:
|
if set(payload) != {"schema_version", "advisories"}:
|
||||||
review_by = dt.date.fromisoformat(str(payload["review_by"]))
|
errors.append("exception policy must contain only schema_version and advisories")
|
||||||
except (KeyError, ValueError):
|
if payload.get("schema_version") != 1:
|
||||||
errors.append("review_by must be an ISO date")
|
errors.append("schema_version must be 1")
|
||||||
review_by = dt.date.min
|
|
||||||
if review_by < dt.date.today():
|
|
||||||
errors.append(f"dependency exception review expired on {review_by.isoformat()}")
|
|
||||||
if payload.get("package") != "starlette":
|
|
||||||
errors.append("only the documented Starlette compatibility exception is allowed")
|
|
||||||
controls = payload.get("compensating_controls")
|
|
||||||
if not isinstance(controls, list) or len(controls) < 3:
|
|
||||||
errors.append("at least three compensating controls are required")
|
|
||||||
advisories = payload.get("advisories")
|
advisories = payload.get("advisories")
|
||||||
if not isinstance(advisories, list) or not advisories:
|
if not isinstance(advisories, list):
|
||||||
errors.append("at least one advisory exception is required")
|
errors.append("advisories must be a list")
|
||||||
else:
|
else:
|
||||||
ids = [str(item.get("id", "")) for item in advisories if isinstance(item, dict)]
|
ids = [str(item.get("id", "")) for item in advisories if isinstance(item, dict)]
|
||||||
if len(ids) != len(set(ids)) or any(not item.startswith("PYSEC-") for item in ids):
|
if len(ids) != len(set(ids)) or any(not item.startswith("PYSEC-") for item in ids):
|
||||||
errors.append("advisory IDs must be unique PYSEC identifiers")
|
errors.append("advisory IDs must be unique PYSEC identifiers")
|
||||||
for item in advisories:
|
for item in advisories:
|
||||||
if not isinstance(item, dict) or len(str(item.get("reason", ""))) < 30:
|
if not isinstance(item, dict):
|
||||||
|
errors.append("every advisory must be an object")
|
||||||
|
continue
|
||||||
|
required = {"id", "package", "review_by", "reason"}
|
||||||
|
allowed = required | {"aliases"}
|
||||||
|
if not required.issubset(item) or not set(item).issubset(allowed):
|
||||||
|
errors.append("every advisory must match the documented exception schema")
|
||||||
|
if not str(item.get("package", "")).strip():
|
||||||
|
errors.append("every advisory requires a package")
|
||||||
|
if len(str(item.get("reason", ""))) < 30:
|
||||||
errors.append("every advisory requires a specific reason")
|
errors.append("every advisory requires a specific reason")
|
||||||
break
|
try:
|
||||||
|
review_by = dt.date.fromisoformat(str(item["review_by"]))
|
||||||
|
except (KeyError, ValueError):
|
||||||
|
errors.append("every advisory review_by must be an ISO date")
|
||||||
|
else:
|
||||||
|
if review_by < dt.date.today():
|
||||||
|
errors.append(
|
||||||
|
f"dependency exception {item.get('id', '')} expired on "
|
||||||
|
f"{review_by.isoformat()}"
|
||||||
|
)
|
||||||
aliases = [
|
aliases = [
|
||||||
str(alias)
|
str(alias)
|
||||||
for item in advisories
|
for item in advisories
|
||||||
@@ -72,8 +82,7 @@ def main() -> int:
|
|||||||
print(alias)
|
print(alias)
|
||||||
else:
|
else:
|
||||||
print(
|
print(
|
||||||
"Dependency exceptions valid through "
|
f"Dependency exception policy valid; {len(payload['advisories'])} active exception(s)."
|
||||||
f"{payload['review_by']} with documented compensating controls."
|
|
||||||
)
|
)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|||||||
@@ -1,35 +1,4 @@
|
|||||||
{
|
{
|
||||||
"review_by": "2026-08-31",
|
"schema_version": 1,
|
||||||
"package": "starlette",
|
"advisories": []
|
||||||
"constraint": "FastAPI 0.139.2 requires Starlette below 0.53; patched Starlette releases start at 1.0.1 or later.",
|
|
||||||
"compensating_controls": [
|
|
||||||
"GeoIntel rejects missing or ambiguous Host headers and request targets before request.url is accessed.",
|
|
||||||
"Application code logs the raw ASGI path and does not use request.url for authorization or routing.",
|
|
||||||
"application/x-www-form-urlencoded is rejected; supported uploads use bounded multipart requests behind nginx.",
|
|
||||||
"The release image is Linux, no HTTPEndpoint route class is used, and the Windows StaticFiles advisory is not applicable."
|
|
||||||
],
|
|
||||||
"advisories": [
|
|
||||||
{
|
|
||||||
"id": "PYSEC-2026-161",
|
|
||||||
"reason": "Host/path ambiguity is rejected at the outer request middleware and request.url is not a security boundary."
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "PYSEC-2026-248",
|
|
||||||
"reason": "Non-slash request paths and ambiguous Host values are rejected before URL reconstruction."
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "PYSEC-2026-249",
|
|
||||||
"aliases": ["CVE-2026-54283"],
|
|
||||||
"reason": "GeoIntel rejects application/x-www-form-urlencoded before Starlette form parsing."
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "PYSEC-2026-2280",
|
|
||||||
"reason": "GeoIntel registers FastAPI APIRouter functions and has no Starlette HTTPEndpoint routes."
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "PYSEC-2026-2281",
|
|
||||||
"aliases": ["CVE-2026-48818"],
|
|
||||||
"reason": "The supported all-in-one production runtime is Linux; the advisory affects Windows StaticFiles."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user