diff --git a/.env.example b/.env.example index 17f5cae1..6fbd9236 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,7 @@ GEOINTEL_API_PREFIX=/api/v1 DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1 STORAGE_ROOT=./storage MAX_UPLOAD_MB=500 +GEOINTEL_MAX_IN_MEMORY_VECTOR_MB=64 CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202 # Optional single-operator access gate. Store only a PBKDF2-SHA256 hash and @@ -14,6 +15,11 @@ GEOINTEL_AUTH_USERNAME= GEOINTEL_AUTH_PASSWORD_HASH= GEOINTEL_AUTH_SESSION_SECRET= GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200 +GEOINTEL_PUBLIC_BASE_URL=http://localhost:1202 +GEOINTEL_AUTHENTIK_ISSUER= +GEOINTEL_AUTHENTIK_CLIENT_ID= +GEOINTEL_AUTHENTIK_CLIENT_SECRET= +GEOINTEL_AUTHENTIK_ALLOWED_EMAIL= GEOINTEL_GUEST_ACCESS_ENABLED=true GEOINTEL_GUEST_DISPLAY_NAME=Gast GEOINTEL_GUEST_SESSION_TTL_SECONDS=7200 diff --git a/.gitea/workflows/managed-validation.yml b/.gitea/workflows/managed-validation.yml index 05072feb..abe430e4 100644 --- a/.gitea/workflows/managed-validation.yml +++ b/.gitea/workflows/managed-validation.yml @@ -20,12 +20,22 @@ concurrency: jobs: full: - name: full + name: ${{ inputs.profile || 'full' }} runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Validate repository with a bounded profile + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: backend/requirements-ci.lock + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + cache: npm + cache-dependency-path: frontend/package-lock.json + - name: Validate the requested profile against the real projects shell: bash env: REQUESTED_PROFILE: ${{ inputs.profile }} @@ -42,77 +52,34 @@ jobs: echo "Unresolved merge markers detected" >&2 exit 1 fi + python scripts/verify_repository_layout.py - if [[ -f pyproject.toml || -f requirements.txt ]]; then - # Compile only tracked Python sources. Running compileall after a - # Node install would otherwise traverse node_modules and turn a - # lightweight baseline into a large runner workload. - git ls-files -z '*.py' | xargs -0 -r python -m py_compile - if [[ -f uv.lock ]]; then - python -m venv "${RUNNER_TEMP}/managed-uv" - uv_python="${RUNNER_TEMP}/managed-uv/bin/python" - "${uv_python}" -m pip install --disable-pip-version-check uv==0.10.0 - managed_uv="${RUNNER_TEMP}/managed-uv/bin/uv" - export UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/managed-project-venv" - "${managed_uv}" sync --locked - export PATH="${UV_PROJECT_ENVIRONMENT}/bin:${PATH}" - if [[ "${profile}" == test || "${profile}" == full ]]; then - if "${managed_uv}" run python -c 'import pytest' 2>/dev/null; then - "${managed_uv}" run python -m pytest - fi - fi - if [[ "${profile}" == lint || "${profile}" == full ]]; then - if "${managed_uv}" run python -c 'import ruff' 2>/dev/null; then - "${managed_uv}" run python -m ruff check . - fi - fi - elif [[ -f requirements.txt ]]; then - python -m venv "${RUNNER_TEMP}/managed-python" - managed_python="${RUNNER_TEMP}/managed-python/bin/python" - "${managed_python}" -m pip install --disable-pip-version-check -r requirements.txt - export PATH="${RUNNER_TEMP}/managed-python/bin:${PATH}" - if [[ "${profile}" == test || "${profile}" == full ]]; then - if "${managed_python}" -c 'import pytest' 2>/dev/null; then - "${managed_python}" -m pytest - fi - fi - fi - fi + python -m pip install --disable-pip-version-check --require-hashes -r backend/requirements-ci.lock + python -m pip install --disable-pip-version-check --no-deps -e backend + (cd frontend && npm ci) - # Prepare Python before invoking Node scripts. Polyglot repositories - # commonly delegate their test script to Python and need the managed - # virtual environment to be active first. - if [[ -f package.json ]]; then - corepack enable - if [[ -f pnpm-lock.yaml ]]; then - pnpm install --frozen-lockfile - [[ "${profile}" == test || "${profile}" == full ]] && pnpm --if-present test - [[ "${profile}" == lint || "${profile}" == full ]] && pnpm --if-present lint - [[ "${profile}" == typecheck || "${profile}" == full ]] && pnpm --if-present typecheck - [[ "${profile}" == build || "${profile}" == full ]] && pnpm --if-present build - elif [[ -f package-lock.json ]]; then - npm ci - [[ "${profile}" == test || "${profile}" == full ]] && npm run --if-present test - [[ "${profile}" == lint || "${profile}" == full ]] && npm run --if-present lint - if [[ "${profile}" == typecheck || "${profile}" == full ]]; then - npm run --if-present typecheck - fi - [[ "${profile}" == build || "${profile}" == full ]] && npm run --if-present build - fi - fi - - if [[ -f go.mod ]]; then - if [[ "${profile}" == test || "${profile}" == build || "${profile}" == full ]]; then - go test ./... - fi - fi - if [[ -f Cargo.toml ]]; then - if [[ "${profile}" == test || "${profile}" == build || "${profile}" == full ]]; then - cargo test --locked - fi - fi - if compgen -G '*.sln' >/dev/null; then - if [[ "${profile}" == test || "${profile}" == build || "${profile}" == full ]]; then - dotnet test --configuration Release - fi - fi + case "${profile}" in + test) + (cd backend && python -m pytest -W error::DeprecationWarning) + (cd frontend && npm run test:unit) + ;; + lint) + python -m ruff check backend scripts tests + (cd frontend && npm run lint --if-present) + ;; + typecheck) + (cd frontend && npm run typecheck) + ;; + build) + python -m compileall backend/app + (cd frontend && npm run build) + ;; + security) + python -m pip install --disable-pip-version-check pip-audit==2.10.1 + bash scripts/audit_python_dependencies.sh + (cd frontend && npm audit --audit-level=high) + ;; + full) + PYTHON_BIN=python bash scripts/run_readiness_check.sh + ;; + esac diff --git a/.gitea/workflows/release-gates.yml b/.gitea/workflows/release-gates.yml index 1edfdec3..3c2351ec 100644 --- a/.gitea/workflows/release-gates.yml +++ b/.gitea/workflows/release-gates.yml @@ -1,6 +1,7 @@ name: GeoIntel release gates on: + pull_request: push: branches: [main, develop] workflow_dispatch: @@ -16,21 +17,21 @@ jobs: quality: name: Compile, test, contracts and builds runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Secret scan run: >- docker run --rm --volume "$PWD:/repo:ro" trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3 filesystem /repo --only-verified --no-update - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" cache: pip cache-dependency-path: backend/requirements-ci.lock - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: "20" cache: npm @@ -57,7 +58,7 @@ jobs: docker compose config > artifacts/docker-compose.resolved.yml - name: Publish quality evidence if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: quality-evidence path: | @@ -71,13 +72,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" cache: pip cache-dependency-path: backend/requirements-ci.lock - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: "20" cache: npm @@ -94,7 +95,7 @@ jobs: npm audit --audit-level=high --json > ../artifacts/npm-audit.json - name: Publish dependency evidence if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: dependency-audits path: | @@ -105,41 +106,67 @@ jobs: retention-days: 30 container: - name: GIS image, SBOM and container scan + name: Production AI image, SBOM and container scan runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 steps: - - uses: actions/checkout@v4 - - name: Build non-AI release image + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Build production AI release image env: RELEASE_SHA: ${{ gitea.sha }} run: | mkdir -p artifacts BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + APP_VERSION="$(tr -d '[:space:]' < VERSION)" docker build \ -f deploy/unraid/Dockerfile.all-in-one \ - --build-arg GEOINTEL_INSTALL_AI=false \ + --build-arg GEOINTEL_INSTALL_AI=true \ --build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \ --build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \ - -t "geointel-ci:$RELEASE_SHA-gis" \ + --build-arg GEOINTEL_APP_VERSION="$APP_VERSION" \ + -t "geointel-ci:$RELEASE_SHA-ai" \ . - docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json + IMAGE_ID="$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" + printf '%s\n' "$IMAGE_ID" > artifacts/image-id.txt + docker image inspect "$IMAGE_ID" > artifacts/image-inspect.json - name: Generate SPDX SBOM env: RELEASE_SHA: ${{ gitea.sha }} - run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis" + run: | + IMAGE_ID="$(cat artifacts/image-id.txt)" + test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" + bash scripts/generate_container_sbom.sh "$IMAGE_ID" - name: Enforce container vulnerability policy env: RELEASE_SHA: ${{ gitea.sha }} - run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis" + run: | + IMAGE_ID="$(cat artifacts/image-id.txt)" + test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" + bash scripts/scan_container_image.sh "$IMAGE_ID" + test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" - name: Publish container evidence if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: container-evidence path: | artifacts/image-inspect.json + artifacts/image-id.txt artifacts/geointel-sbom.spdx.json artifacts/geointel-container-vulnerabilities.json if-no-files-found: warn retention-days: 30 + + deploy: + name: Deploy exact gated revision to Unraid + needs: [quality, dependency-audit, container] + if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }} + runs-on: unraid-deploy + timeout-minutes: 180 + steps: + - name: Deploy only after every release gate is green + run: | + set -euo pipefail + docker exec gitea-deploy-control \ + /opt/gitea-deploy/deploy.py deploy \ + "${{ gitea.repository }}" "${{ gitea.sha }}" diff --git a/.gitea/workflows/unraid-deploy.yml b/.gitea/workflows/unraid-deploy.yml deleted file mode 100644 index 4a98a9c4..00000000 --- a/.gitea/workflows/unraid-deploy.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Unraid autoredeploy - -on: - push: - branches: [main] - paths-ignore: - - ".gitea/**" - - "docs/**" - - "**/*.md" - workflow_dispatch: - -concurrency: - group: unraid-production-geointel - cancel-in-progress: false - -permissions: - contents: read - -jobs: - deploy: - name: Deploy geointel - runs-on: unraid-deploy - timeout-minutes: 180 - steps: - - name: Deploy exact Gitea revision - run: | - set -euo pipefail - docker exec gitea-deploy-control \ - /opt/gitea-deploy/deploy.py deploy \ - "$GITHUB_REPOSITORY" "$GITHUB_SHA" - diff --git a/.github/workflows/release-gates.yml b/.github/workflows/release-gates.yml index 7887a2b4..ced85ca3 100644 --- a/.github/workflows/release-gates.yml +++ b/.github/workflows/release-gates.yml @@ -20,19 +20,19 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Secret scan run: >- docker run --rm --volume "$PWD:/repo:ro" trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3 filesystem /repo --only-verified --no-update - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" cache: pip cache-dependency-path: backend/requirements-ci.lock - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: "20" cache: npm @@ -59,7 +59,7 @@ jobs: docker compose config > artifacts/docker-compose.resolved.yml - name: Publish quality evidence if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: quality-evidence path: | @@ -73,13 +73,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" cache: pip cache-dependency-path: backend/requirements-ci.lock - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: "20" cache: npm @@ -96,7 +96,7 @@ jobs: npm audit --audit-level=high --json > ../artifacts/npm-audit.json - name: Publish dependency evidence if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: dependency-audits path: | @@ -107,40 +107,52 @@ jobs: retention-days: 30 container: - name: GIS image, SBOM and container scan + name: Production AI image, SBOM and container scan runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 steps: - - uses: actions/checkout@v4 - - name: Build non-AI release image + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Build production AI release image env: RELEASE_SHA: ${{ github.sha }} run: | mkdir -p artifacts BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + APP_VERSION="$(tr -d '[:space:]' < VERSION)" docker build \ -f deploy/unraid/Dockerfile.all-in-one \ - --build-arg GEOINTEL_INSTALL_AI=false \ + --build-arg GEOINTEL_INSTALL_AI=true \ --build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \ --build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \ - -t "geointel-ci:$RELEASE_SHA-gis" \ + --build-arg GEOINTEL_APP_VERSION="$APP_VERSION" \ + -t "geointel-ci:$RELEASE_SHA-ai" \ . - docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json + IMAGE_ID="$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" + printf '%s\n' "$IMAGE_ID" > artifacts/image-id.txt + docker image inspect "$IMAGE_ID" > artifacts/image-inspect.json - name: Generate SPDX SBOM env: RELEASE_SHA: ${{ github.sha }} - run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis" + run: | + IMAGE_ID="$(cat artifacts/image-id.txt)" + test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" + bash scripts/generate_container_sbom.sh "$IMAGE_ID" - name: Enforce container vulnerability policy env: RELEASE_SHA: ${{ github.sha }} - run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis" + run: | + IMAGE_ID="$(cat artifacts/image-id.txt)" + test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" + bash scripts/scan_container_image.sh "$IMAGE_ID" + test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" - name: Publish container evidence if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: container-evidence path: | artifacts/image-inspect.json + artifacts/image-id.txt artifacts/geointel-sbom.spdx.json artifacts/geointel-container-vulnerabilities.json if-no-files-found: warn diff --git a/backend/.dockerignore b/backend/.dockerignore index 8cd3b7a8..010b0057 100644 --- a/backend/.dockerignore +++ b/backend/.dockerignore @@ -8,3 +8,5 @@ storage dist node_modules .env +.env.* +!.env.example diff --git a/backend/pyproject.toml b/backend/pyproject.toml index fe8bab1a..083b614a 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.11" dependencies = [ "fastapi>=0.112.0", - "starlette>=0.46.0,<1.0.0", + "starlette>=1.3.1,<2.0.0", "uvicorn[standard]>=0.30.6", "SQLAlchemy>=2.0.34", "psycopg[binary]>=3.2.1", @@ -16,6 +16,8 @@ dependencies = [ "shapely>=2.0.4", "pyproj>=3.6.1", "python-multipart>=0.0.9", + "itsdangerous>=2.2.0", + "PyJWT[crypto]>=2.10.1", "rdflib>=7.1,<8", "alembic>=1.13.2", ] @@ -37,7 +39,7 @@ ai = [ "ultralytics>=8.3,<9", "torch>=2.4", ] -dev = ["pytest>=8.3.2", "httpx>=0.27.0", "ruff>=0.6.9"] +dev = ["pytest>=8.3.2", "httpx>=0.27.0", "httpx2>=2.0.0", "ruff>=0.6.9"] [project.scripts] geointel-backend = "app.main:main" diff --git a/backend/requirements-ci.lock b/backend/requirements-ci.lock index 0b0e099e..262afa3e 100644 --- a/backend/requirements-ci.lock +++ b/backend/requirements-ci.lock @@ -2,9 +2,9 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --extra=dev --extra=gis --generate-hashes --output-file=requirements-ci.lock --strip-extras pyproject.toml +# pip-compile --extra=dev --extra=gis --generate-hashes --no-index --output-file=requirements-ci.lock --strip-extras pyproject.toml # -# geointel-input-sha256: 03c20efedd96474cbe62591b7b70cdad2681688b618bdd76731bd4cfaf85b3d4 +# geointel-input-sha256: e1dd11f5b30f4c8c902f33476282da48da386d36093cab68212edc69e75df8e3 affine==2.4.0 \ --hash=sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92 \ --hash=sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea @@ -26,6 +26,7 @@ anyio==4.14.2 \ --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f # via # httpx + # httpx2 # starlette # watchfiles attrs==26.1.0 \ @@ -41,6 +42,108 @@ certifi==2026.6.17 \ # pyogrio # pyproj # rasterio +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography click==8.4.2 \ --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 @@ -57,6 +160,54 @@ cligj==0.7.2 \ --hash=sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27 \ --hash=sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df # via rasterio +cryptography==50.0.1 \ + --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ + --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ + --hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \ + --hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \ + --hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \ + --hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \ + --hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \ + --hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \ + --hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \ + --hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \ + --hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \ + --hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \ + --hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \ + --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \ + --hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \ + --hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \ + --hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \ + --hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \ + --hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \ + --hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \ + --hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \ + --hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \ + --hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \ + --hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \ + --hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \ + --hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \ + --hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \ + --hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \ + --hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \ + --hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \ + --hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \ + --hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \ + --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \ + --hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \ + --hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \ + --hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \ + --hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \ + --hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \ + --hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \ + --hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \ + --hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \ + --hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \ + --hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \ + --hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \ + --hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \ + --hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef + # via pyjwt fastapi==0.139.2 \ --hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \ --hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c @@ -155,11 +306,16 @@ h11==0.16.0 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via # httpcore + # httpcore2 # uvicorn httpcore==1.0.9 \ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 # via httpx +httpcore2==2.12.0 \ + --hash=sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb \ + --hash=sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648 + # via httpx2 httptools==0.8.0 \ --hash=sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683 \ --hash=sha256:0ea897f0c729581ebf72131a438a7932d9b14efef72d75ada966700cac3caaeb \ @@ -216,16 +372,25 @@ httpx==0.28.1 \ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad # via geointel-backend (pyproject.toml) +httpx2==2.12.0 \ + --hash=sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf \ + --hash=sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36 + # via geointel-backend (pyproject.toml) idna==3.18 \ --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 # via # anyio # httpx + # httpx2 iniconfig==2.3.0 \ --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 # via pytest +itsdangerous==2.2.0 \ + --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \ + --hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173 + # via geointel-backend (pyproject.toml) mako==1.3.12 \ --hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \ --hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a @@ -613,6 +778,10 @@ psycopg-binary==3.3.4 \ --hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \ --hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7 # via psycopg +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi pydantic==2.13.4 \ --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \ --hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6 @@ -750,6 +919,12 @@ pygments==2.20.0 \ --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 # via pytest +pyjwt==2.13.0 \ + --hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \ + --hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728 + # via + # geointel-backend (pyproject.toml) + # pyjwt pyogrio==0.13.0 \ --hash=sha256:1b91f6d6e6757a6ea84b9459d24f479dcb52bbf4ebcdb16baf39e49d2836a1cf \ --hash=sha256:220a988ce2a26591d6db5c775b07289d4f54cabdf274cc048f0e17a0b9d5be14 \ @@ -1127,12 +1302,18 @@ sqlalchemy==2.0.51 \ # alembic # geoalchemy2 # geointel-backend (pyproject.toml) -starlette==0.52.1 \ - --hash=sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74 \ - --hash=sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933 +starlette==1.6.0 \ + --hash=sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c \ + --hash=sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b # via # fastapi # geointel-backend (pyproject.toml) +truststore==0.10.4 \ + --hash=sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301 \ + --hash=sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981 + # via + # httpcore2 + # httpx2 typing-extensions==4.16.0 \ --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 @@ -1140,6 +1321,7 @@ typing-extensions==4.16.0 \ # alembic # anyio # fastapi + # httpx2 # psycopg # pydantic # pydantic-core diff --git a/backend/requirements-runtime.lock b/backend/requirements-runtime.lock index 25671d04..362df3d8 100644 --- a/backend/requirements-runtime.lock +++ b/backend/requirements-runtime.lock @@ -2,9 +2,9 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --extra=gis --generate-hashes --output-file=requirements-runtime.lock --strip-extras pyproject.toml +# pip-compile --extra=gis --generate-hashes --no-index --output-file=requirements-runtime.lock --strip-extras pyproject.toml # -# geointel-input-sha256: 0d0d2cdceb01da58354610f03b5ce244523130cf9fd29e3f8988dbe684838e8b +# geointel-input-sha256: 30318170074aad9b91f570cccc574eaaad0eded5814a464d1207a94d2e1d3317 affine==2.4.0 \ --hash=sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92 \ --hash=sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea @@ -38,6 +38,108 @@ certifi==2026.6.17 \ # pyogrio # pyproj # rasterio +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography click==8.4.2 \ --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 @@ -54,6 +156,54 @@ cligj==0.7.2 \ --hash=sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27 \ --hash=sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df # via rasterio +cryptography==50.0.1 \ + --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ + --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ + --hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \ + --hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \ + --hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \ + --hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \ + --hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \ + --hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \ + --hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \ + --hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \ + --hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \ + --hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \ + --hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \ + --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \ + --hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \ + --hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \ + --hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \ + --hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \ + --hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \ + --hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \ + --hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \ + --hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \ + --hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \ + --hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \ + --hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \ + --hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \ + --hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \ + --hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \ + --hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \ + --hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \ + --hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \ + --hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \ + --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \ + --hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \ + --hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \ + --hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \ + --hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \ + --hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \ + --hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \ + --hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \ + --hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \ + --hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \ + --hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \ + --hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \ + --hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \ + --hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef + # via pyjwt fastapi==0.139.2 \ --hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \ --hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c @@ -207,6 +357,10 @@ idna==3.18 \ --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 # via anyio +itsdangerous==2.2.0 \ + --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \ + --hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173 + # via geointel-backend (pyproject.toml) mako==1.3.12 \ --hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \ --hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a @@ -589,6 +743,10 @@ psycopg-binary==3.3.4 \ --hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \ --hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7 # via psycopg +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi pydantic==2.13.4 \ --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \ --hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6 @@ -722,6 +880,12 @@ pydantic-settings==2.14.2 \ --hash=sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440 \ --hash=sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f # via geointel-backend (pyproject.toml) +pyjwt==2.13.0 \ + --hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \ + --hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728 + # via + # geointel-backend (pyproject.toml) + # pyjwt pyogrio==0.13.0 \ --hash=sha256:1b91f6d6e6757a6ea84b9459d24f479dcb52bbf4ebcdb16baf39e49d2836a1cf \ --hash=sha256:220a988ce2a26591d6db5c775b07289d4f54cabdf274cc048f0e17a0b9d5be14 \ @@ -1075,9 +1239,9 @@ sqlalchemy==2.0.51 \ # alembic # geoalchemy2 # geointel-backend (pyproject.toml) -starlette==0.52.1 \ - --hash=sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74 \ - --hash=sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933 +starlette==1.6.0 \ + --hash=sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c \ + --hash=sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b # via # fastapi # geointel-backend (pyproject.toml) diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py index ad2c715c..463dd909 100644 --- a/backend/tests/test_docker_runtime_config.py +++ b/backend/tests/test_docker_runtime_config.py @@ -172,6 +172,25 @@ def test_walloon_runtime_settings_are_editable_in_compose_and_unraid() -> None: assert "WALOUS_MAX_PIXELS" in content +def test_in_memory_vector_limit_is_propagated_and_validated_in_every_runtime() -> None: + expected = "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" + for path in ( + ROOT / ".env.example", + ROOT / "docker-compose.yml", + ROOT / "docker-compose.unraid.yml", + ROOT / "deploy" / "unraid" / "geointel.env.example", + ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml", + ): + assert expected in path.read_text(encoding="utf-8"), path + + run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text( + encoding="utf-8" + ) + assert 'GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}"' in run_script + assert "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be between 1 and 256." in run_script + assert '-e GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB"' in run_script + + def test_frontend_uses_same_origin_api_proxy_by_default() -> None: api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8") nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8") @@ -208,6 +227,45 @@ def test_nginx_runtime_allows_long_ai_and_qa_requests() -> None: assert "proxy_send_timeout 600s;" in config +def test_nginx_preserves_outer_https_scheme_for_secure_session_cookies() -> None: + configs = ( + (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"), + (ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8"), + ) + for config in configs: + assert "geo $geointel_trusted_forwarder" in config + assert "default 0;" in config + assert "172.16.0.0/12 1;" in config + assert 'map "$geointel_trusted_forwarder:$http_x_forwarded_proto"' in config + assert '"1:https" https;' in config + assert "proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;" in config + assert "proxy_set_header X-Forwarded-Proto $scheme;" not in config + + +def test_nginx_runtime_sets_security_headers_on_all_cached_locations() -> None: + configs = ( + (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"), + (ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text( + encoding="utf-8" + ), + ) + required = ( + 'Content-Security-Policy "frame-ancestors \'none\'" always;', + 'X-Frame-Options "DENY" always;', + 'X-Content-Type-Options "nosniff" always;', + 'Referrer-Policy "strict-origin-when-cross-origin" always;', + 'Permissions-Policy "camera=(), microphone=(), geolocation=()" always;', + ) + + for config in configs: + cached_locations = config.count("add_header Cache-Control") + assert cached_locations >= 2 + for header in required: + # Nginx 1.27 locations with Cache-Control do not inherit server-level + # add_header directives, so every cached location repeats the policy. + assert config.count(f"add_header {header}") == cached_locations + 1 + + def test_compose_does_not_publish_postgis_on_default_host_port() -> None: compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") diff --git a/backend/tests/test_rc10_data_operations.py b/backend/tests/test_rc10_data_operations.py index 57ec1f97..b3702d40 100644 --- a/backend/tests/test_rc10_data_operations.py +++ b/backend/tests/test_rc10_data_operations.py @@ -3,6 +3,8 @@ from __future__ import annotations import hashlib import importlib.util import json +import os +import subprocess import sys from datetime import datetime, timedelta, timezone from pathlib import Path @@ -18,6 +20,8 @@ SCRIPTS = ROOT / "scripts" def load_script(name: str): path = SCRIPTS / name + if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) @@ -36,6 +40,9 @@ def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha "database_password_secure": True, "inventory_mode": inventory_mode, "storage_inventory_requested": True, + "storage_snapshot_requested": True, + "models_inventory_requested": False, + "models_snapshot_requested": False, "git_commit": "0123456789abcdef", } files = { @@ -48,6 +55,7 @@ def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha } for name, content in files.items(): (root / name).write_text(content, encoding="utf-8") + (root / "storage-snapshot").mkdir() checksums = [] for name in sorted(files): digest = hashlib.sha256((root / name).read_bytes()).hexdigest() @@ -337,8 +345,11 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8") live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8") - assert "DELETE_STORAGE_ARTIFACTS" in generic + assert "QUARANTINE_STORAGE_ARTIFACTS" in generic assert "verify_current_backup" in generic + assert "os.link" in generic + assert 'entry["status"] = "linked"' in generic + assert "cleanup-quarantine" in generic assert "DELETE_DEMO_EXPORTS" in demo assert "verify_current_backup" in demo assert "/app/backups:ro" in compose @@ -347,6 +358,8 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N "release_backup_guard.py", "audit_data_operations.py", "cleanup_storage_artifacts.py", + "restore_storage_quarantine.py", + "release_backup_snapshot.py", ): assert f"COPY scripts/{name}" in dockerfile assert f"py_compile scripts/{name}" in readiness @@ -356,3 +369,201 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N assert "table-counts-after.tsv" in live_audit assert "deleted_count" in live_audit assert "missing_manifest_artifact_count" in live_audit + + +def test_cleanup_apply_moves_bytes_to_protected_traceable_quarantine( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.syspath_prepend(str(SCRIPTS)) + cleanup = load_script("cleanup_storage_artifacts.py") + storage = tmp_path / "storage" + source = storage / "derived" / "orphan.bin" + source.parent.mkdir(parents=True) + source.write_bytes(b"recoverable-derived-artifact") + candidate = SimpleNamespace( + path=source.resolve(), + relative_path="derived/orphan.bin", + size_bytes=source.stat().st_size, + ) + now = datetime.now(timezone.utc) + + class SessionContext: + def __enter__(self): + return SimpleNamespace() + + def __exit__(self, *_args): + return False + + monkeypatch.setattr( + cleanup, + "parse_args", + lambda: SimpleNamespace( + storage_root=storage, + minimum_age_days=7, + max_delete=1, + apply=True, + confirm="QUARANTINE_STORAGE_ARTIFACTS", + backup_dir=tmp_path / "backup", + backup_max_age_hours=24.0, + quarantine_root=None, + ), + ) + monkeypatch.setattr(cleanup, "SessionLocal", lambda: SessionContext()) + monkeypatch.setattr( + cleanup, + "build_report", + lambda *_args, **_kwargs: ({"cleanup": {"protected_prefixes": []}}, [candidate]), + ) + monkeypatch.setattr( + cleanup, + "verify_current_backup", + lambda *_args, **_kwargs: SimpleNamespace( + release_id="predeploy-test", + created_at=now, + age_hours=0.1, + backup_tool_revision="0123456789abcdef", + ), + ) + + assert cleanup.main() == 0 + payload = json.loads(capsys.readouterr().out) + manifest_path = Path(payload["quarantine_manifest"]) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + quarantined_path = storage / payload["quarantined"][0]["quarantine_relative_path"] + + assert not source.exists() + assert quarantined_path.read_bytes() == b"recoverable-derived-artifact" + assert manifest["state"] == "complete" + assert manifest["backup_release_id"] == "predeploy-test" + assert manifest["entries"][0]["status"] == "quarantined" + assert payload["deleted_count"] == 0 + + restore = subprocess.run( + [ + sys.executable, + str(SCRIPTS / "restore_storage_quarantine.py"), + "--storage-root", + str(storage), + "--manifest", + str(manifest_path), + "--confirm", + "RESTORE_QUARANTINED_ARTIFACTS", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + assert restore.returncode == 0, restore.stderr + assert source.read_bytes() == b"recoverable-derived-artifact" + assert not quarantined_path.exists() + restored_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert restored_manifest["state"] == "restored" + assert restored_manifest["entries"][0]["status"] == "restored" + + +def _write_interrupted_quarantine( + storage: Path, + *, + original_exists: bool, + quarantine_exists: bool, + hard_linked: bool = False, +) -> tuple[Path, Path, Path]: + original = storage / "derived" / "interrupted.bin" + operation = storage / "operator-evidence" / "cleanup-quarantine" / "cleanup-interrupted" + quarantined = operation / "files" / "derived" / "interrupted.bin" + original.parent.mkdir(parents=True, exist_ok=True) + quarantined.parent.mkdir(parents=True, exist_ok=True) + retained = b"interrupted-retained-bytes" + if original_exists: + original.write_bytes(retained) + if quarantine_exists: + if hard_linked: + os.link(original, quarantined) + else: + quarantined.write_bytes(retained) + manifest = operation / "manifest.json" + manifest.write_text( + json.dumps( + { + "schema_version": 1, + "state": "in_progress", + "entries": [ + { + "relative_path": "derived/interrupted.bin", + "quarantine_relative_path": quarantined.relative_to(storage).as_posix(), + "size_bytes": len(retained), + "sha256": hashlib.sha256(retained).hexdigest(), + "status": "linked" if hard_linked else "planned", + } + ], + } + ), + encoding="utf-8", + ) + return manifest, original, quarantined + + +@pytest.mark.parametrize( + ("original_exists", "quarantine_exists", "hard_linked"), + ((False, True, False), (True, True, True)), +) +def test_quarantine_restore_recovers_each_interrupted_move_window( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + original_exists: bool, + quarantine_exists: bool, + hard_linked: bool, +) -> None: + restore = load_script("restore_storage_quarantine.py") + storage = tmp_path / "storage" + manifest, original, quarantined = _write_interrupted_quarantine( + storage, + original_exists=original_exists, + quarantine_exists=quarantine_exists, + hard_linked=hard_linked, + ) + monkeypatch.setattr( + restore, + "parse_args", + lambda: SimpleNamespace( + storage_root=storage, + manifest=manifest, + confirm="RESTORE_QUARANTINED_ARTIFACTS", + ), + ) + + assert restore.main() == 0 + assert original.read_bytes() == b"interrupted-retained-bytes" + assert not quarantined.exists() + assert json.loads(manifest.read_text(encoding="utf-8"))["state"] == "restored" + + +def test_quarantine_restore_never_clobbers_recreated_destination( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + restore = load_script("restore_storage_quarantine.py") + storage = tmp_path / "storage" + manifest, original, quarantined = _write_interrupted_quarantine( + storage, + original_exists=False, + quarantine_exists=True, + ) + original.write_bytes(b"new-runtime-bytes") + monkeypatch.setattr( + restore, + "parse_args", + lambda: SimpleNamespace( + storage_root=storage, + manifest=manifest, + confirm="RESTORE_QUARANTINED_ARTIFACTS", + ), + ) + + with pytest.raises(RuntimeError, match="different bytes"): + restore.main() + assert original.read_bytes() == b"new-runtime-bytes" + assert quarantined.read_bytes() == b"interrupted-retained-bytes" diff --git a/backend/tests/test_rc5_release_deployment.py b/backend/tests/test_rc5_release_deployment.py index 67ea2208..358feeb6 100644 --- a/backend/tests/test_rc5_release_deployment.py +++ b/backend/tests/test_rc5_release_deployment.py @@ -18,17 +18,16 @@ def test_build_identity_does_not_invalidate_dependency_layers() -> None: assert 'io.geointel.ai.enabled="${GEOINTEL_INSTALL_AI}"' in dockerfile -def test_release_deploy_preserves_immutable_and_previous_images() -> None: +def test_release_deploy_preserves_immutable_and_backup_specific_rollback_images() -> None: script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8") assert 'GEOINTEL_RELEASE_VARIANT="ai"' in script assert 'GEOINTEL_RELEASE_VARIANT="gis"' in script assert 'GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"' in script - assert 'GEOINTEL_PREVIOUS_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:previous"' in script - assert 'release_image_id="$(' in script - assert '[ "$current_image_id" != "$release_image_id" ]' in script - assert 'docker tag "$current_image_id" "$GEOINTEL_PREVIOUS_IMAGE"' in script - assert "preserving the existing previous image" in script + assert 'GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"' in script + assert 'docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script + assert '--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script + assert "GEOINTEL_PREVIOUS_IMAGE" not in script assert 'if docker image inspect "$GEOINTEL_RELEASE_IMAGE"' in script assert "Immutable release tag has conflicting metadata" in script assert "Reusing existing immutable image" in script @@ -36,12 +35,62 @@ def test_release_deploy_preserves_immutable_and_previous_images() -> None: assert "Deployed immutable image" in script +def test_release_creates_verified_backup_before_candidate_migrations() -> None: + script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8") + + backup_index = script.index("create_predeploy_backup\n") + scan_index = script.index("scan_release_image\n") + candidate_start_index = script.index('if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"') + assert scan_index < backup_index + assert backup_index < candidate_start_index + assert script.index("docker build") < backup_index + assert "/mnt/user/appdata/geointel/backups" in script + assert "--inventory-mode sha256" in script + assert "scripts/verify_release_backup.sh" in script + assert "refusing an unbacked migration" in script + assert "Quiescing the current backend" in script + assert "restarting the unchanged current release" in script + assert "preflight_backup_capacity" in script + assert "select_verified_link_dest" in script + assert "release_backup_snapshot.py" in script + + +def test_release_starts_only_the_locally_attested_ai_image() -> None: + script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8") + + assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"' in script + assert "Production release deployment requires the gated AI image" in script + assert 'bash scripts/generate_container_sbom.sh "$GEOINTEL_RELEASE_IMAGE"' in script + assert 'bash scripts/scan_container_image.sh "$GEOINTEL_RELEASE_IMAGE"' in script + assert 'running_image_id="$(docker inspect --format \'{{.Image}}\' geointel)"' in script + assert 'if [ "$running_image_id" != "$image" ]' in script + assert "deployment-attestation.json" in script + assert "artifacts/release-evidence/deploy" in script + assert "GITEA_COMMIT_SHA" in script + assert "GITHUB_SHA" in script + assert "must contain one full 40-character Git commit SHA" in script + assert 'running_revision" != "$GEOINTEL_BUILD_SHA"' in script + assert 'running_ai" != "true"' in script + assert '"revision": revision' in script + + def test_release_and_container_replacement_are_serialized() -> None: release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8") run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8") + rollback_script = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text( + encoding="utf-8" + ) + restore_script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text( + encoding="utf-8" + ) assert "GEOINTEL_DEPLOY_LOCK_FILE" in release_script assert "flock -n 9" in release_script + assert "GEOINTEL_DEPLOY_LOCK_FILE" in rollback_script + assert "flock -n 9" in rollback_script + assert "GEOINTEL_DEPLOY_LOCK_FILE" in restore_script + assert "flock -n 9" in restore_script + assert "GEOINTEL_DEPLOY_LOCK_HELD=true" in release_script assert "GEOINTEL_CONTAINER_LOCK_FILE" in run_script assert "flock -w 300 8" in run_script assert "GeoIntel container removal did not complete within 60 seconds" in run_script @@ -89,13 +138,35 @@ def test_manual_rollback_reuses_persistent_paths_and_requires_existing_image() - rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(encoding="utf-8") run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8") - assert "geointel-all-in-one:previous" in rollback + assert "Backup manifest lacks an immutable rollback image ID" in rollback + assert 'get("image_id", "")' in rollback assert 'docker image inspect "$GEOINTEL_ROLLBACK_IMAGE"' in rollback assert 'GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE"' in rollback + assert "restore-predeploy-database.sh" in rollback + assert "--confirm-production-database-restore" in rollback + assert "Image-only rollback" in rollback assert '-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data"' in run_script assert '-v "${GEOINTEL_STORAGE_PATH}:/app/storage"' in run_script +def test_same_revision_redeploy_rolls_back_by_backup_bound_image_id() -> None: + release = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8") + rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text( + encoding="utf-8" + ) + restore = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text( + encoding="utf-8" + ) + backup = (ROOT / "scripts" / "backup_release_state.sh").read_text(encoding="utf-8") + + assert "current_image_id" in release + assert "release_image_id" not in release + assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in backup + assert 'if [ -z "$RESTORE_IMAGE" ]; then\n RESTORE_IMAGE="$BACKUP_IMAGE_ID"' in restore + assert 'GEOINTEL_ROLLBACK_IMAGE="$(python3 - "$BACKUP_DIR/manifest.json"' in rollback + assert "geointel-all-in-one:previous" not in release + rollback + restore + + def test_readiness_checks_all_release_shell_entrypoints() -> None: readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") @@ -107,6 +178,7 @@ def test_readiness_checks_all_release_shell_entrypoints() -> None: "deploy/unraid/run-dockerman-container.sh", "deploy/unraid/deploy-release.sh", "deploy/unraid/rollback-dockerman-container.sh", + "deploy/unraid/restore-predeploy-database.sh", ): assert f"bash -n {path}" in readiness diff --git a/backend/tests/test_rc6_supply_chain.py b/backend/tests/test_rc6_supply_chain.py index 619b2572..0f8684dc 100644 --- a/backend/tests/test_rc6_supply_chain.py +++ b/backend/tests/test_rc6_supply_chain.py @@ -48,13 +48,54 @@ def test_ci_runs_complete_release_and_supply_chain_gates() -> None: assert "pip-audit==2.10.1" in workflow assert "audit_python_dependencies.sh" in workflow assert "npm audit --audit-level=high" in workflow - assert "GEOINTEL_INSTALL_AI=false" in workflow + assert "GEOINTEL_INSTALL_AI=true" in workflow + assert "geointel-ci:$RELEASE_SHA-ai" in workflow + assert "artifacts/image-id.txt" in workflow + assert 'scan_container_image.sh "$IMAGE_ID"' in workflow assert "generate_container_sbom.sh" in workflow assert "scan_container_image.sh" in workflow - assert "actions/upload-artifact@v4" in workflow + assert "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" in workflow assert context in workflow +def test_gitea_production_deploy_depends_on_every_release_gate() -> None: + release = read(".gitea/workflows/release-gates.yml") + legacy_deploy = ROOT / ".gitea" / "workflows" / "unraid-deploy.yml" + + assert "pull_request:" in release + assert "needs: [quality, dependency-audit, container]" in release + assert "gitea.event_name == 'push'" in release + assert "gitea.ref == 'refs/heads/main'" in release + assert "/opt/gitea-deploy/deploy.py deploy" in release + assert not legacy_deploy.exists() + assert "workflow_dispatch:" in release + + +def test_release_workflows_pin_third_party_actions_to_reviewed_commits() -> None: + expected = ( + "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683", + "actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065", + "actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020", + "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02", + ) + for path in (".gitea/workflows/release-gates.yml", ".github/workflows/release-gates.yml"): + workflow = read(path) + for action in expected: + assert action in workflow + + +def test_managed_validation_targets_the_actual_backend_and_frontend_projects() -> None: + workflow = read(".gitea/workflows/managed-validation.yml") + + assert "backend/requirements-ci.lock" in workflow + assert "frontend/package-lock.json" in workflow + assert "python -m pytest -W error::DeprecationWarning" in workflow + assert "cd frontend && npm run test:unit" in workflow + assert "python -m ruff check backend scripts tests" in workflow + assert "python scripts/verify_repository_layout.py" in workflow + assert "[[ -f pyproject.toml" not in workflow + + def test_scanner_images_are_versioned_and_digest_pinned() -> None: sbom = read("scripts/generate_container_sbom.sh") scan = read("scripts/scan_container_image.sh") @@ -65,8 +106,9 @@ def test_scanner_images_are_versioned_and_digest_pinned() -> None: assert "--ignore-unfixed" in scan assert "--timeout 20m" in scan assert "--scanners vuln" in scan - assert '-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro"' in scan - assert '--ignorefile "$CONTAINER_IGNORE_FILE"' in scan + assert 'ignored_container_ids' in scan + assert 'ignore_args=(-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro")' in scan + assert 'trivy_ignore_args=(--ignorefile "$CONTAINER_IGNORE_FILE")' in scan assert "--skip-files /usr/local/bin/gosu" in scan assert "final filesystem replaces it with the audited setpriv shell wrapper" in scan assert "geointel-container-vulnerabilities.json" in scan @@ -86,11 +128,12 @@ def test_readiness_guards_lock_and_supply_chain_entrypoints() -> None: assert f"bash -n {path}" in readiness -def test_python_audit_exceptions_are_timeboxed_and_full_evidence_is_kept() -> None: +def test_python_audit_policy_has_no_active_exceptions_and_keeps_full_evidence() -> None: policy = read("security/pip-audit-exceptions.json") audit_script = read("scripts/audit_python_dependencies.sh") - assert '"review_by": "2026-08-31"' in policy + assert '"schema_version": 1' in policy + assert '"advisories": []' in policy assert "pip-audit-full.json" in audit_script assert "pip-audit-policy.json" in audit_script assert "--ignore-vuln" in audit_script diff --git a/backend/tests/test_rc_backup_restore_scripts.py b/backend/tests/test_rc_backup_restore_scripts.py index 0c543f85..62340d85 100644 --- a/backend/tests/test_rc_backup_restore_scripts.py +++ b/backend/tests/test_rc_backup_restore_scripts.py @@ -20,12 +20,35 @@ def test_backup_is_atomic_read_only_and_checksum_bound() -> None: assert "--no-owner" in script assert "CHECKSUMS.sha256" in script assert "database-password" not in script.lower() - assert 'git -C "$ROOT" rev-parse HEAD' in script - assert 'git -C "$ROOT" status --porcelain=v1' in script + assert 'for required in docker python3 sha256sum; do' in script + assert 'for required in docker python3 sha256sum git; do' not in script + assert "GITEA_COMMIT_SHA" in script + assert "GITHUB_SHA" in script + assert "GEOINTEL_BUILD_SHA" in script + assert 'if command -v git >/dev/null 2>&1' in script assert "mv \"$PARTIAL\" \"$FINAL\"" in script assert "rm -rf -- \"$PARTIAL\"" in script assert "DROP DATABASE" not in script assert "pg_restore --clean" not in script + assert "/mnt/user/appdata/geointel/backups" in script + assert "release_backup_snapshot.py" in script + assert "storage-snapshot" not in script # labels are composed without unsafe path interpolation + assert "--link-dest-backup" in script + assert "--rollback-image-tag" in script + assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in script + + +def test_backup_binds_prepared_source_without_requiring_dot_git() -> None: + script = read("backup_release_state.sh") + + controller_resolution = script.index('local gitea_sha="${GITEA_COMMIT_SHA:-}"') + optional_git_fallback = script.index('if command -v git >/dev/null 2>&1') + docker_access = script.index("docker inspect -f '{{.State.Running}}'") + assert controller_resolution < optional_git_fallback < docker_access + assert 'SOURCE_REVISION="$explicit_sha"' in script + assert '"backup_tool_revision": ${SOURCE_REVISION@Q}' in script + assert '"running_image_revision": ${RUNNING_IMAGE_REVISION@Q}' in script + assert "Cannot bind backup to a source revision" in script def test_backup_verification_is_read_only() -> None: @@ -55,9 +78,11 @@ def test_release_safety_scripts_have_valid_bash_syntax() -> None: "backup_release_state.sh", "verify_release_backup.sh", "restore_release_backup_smoke.sh", + "../deploy/unraid/restore-predeploy-database.sh", ): + script_path = f"scripts/{name}" if not name.startswith("../") else name.removeprefix("../") result = subprocess.run( - ["bash", "-n", f"scripts/{name}"], + ["bash", "-n", script_path], cwd=ROOT, capture_output=True, text=True, @@ -66,6 +91,30 @@ def test_release_safety_scripts_have_valid_bash_syntax() -> None: assert result.returncode == 0, f"{name}: {result.stderr}" +def test_production_restore_is_explicit_bounded_and_verified() -> None: + script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text( + encoding="utf-8" + ) + + assert "--confirm-production-database-restore" in script + assert "/mnt/user/appdata/geointel/backups" in script + assert "backup.relative_to(root)" in script + assert "sha256sum -c CHECKSUMS.sha256" in script + assert '"$RESTORE_PROOF_DB"' in script + assert "pg_restore" in script + assert "Restored Alembic head" in script + assert "Restored count mismatch" in script + assert "pg_restore --clean" not in script + assert "geointel_restore_proof_" in script + assert "Isolated predeploy restore proof passed" in script + assert "ALTER DATABASE" in script + assert "Pre-restore production database retained" in script + drop_start = script.index("dropdb --if-exists --force") + drop_command = script[drop_start : script.index("\n fi", drop_start)] + assert '"$RESTORE_PROOF_DB"' in drop_command + assert '"$GEOINTEL_POSTGRES_DB"' not in drop_command + + def test_readiness_gate_checks_release_safety_scripts() -> None: readiness = read("run_readiness_check.sh") diff --git a/backend/tests/test_readiness_gate.py b/backend/tests/test_readiness_gate.py index 223ada25..f2d7eeb4 100644 --- a/backend/tests/test_readiness_gate.py +++ b/backend/tests/test_readiness_gate.py @@ -1,11 +1,12 @@ from pathlib import Path -def test_backend_keeps_starlette_on_the_supported_pre_httpx2_line() -> None: +def test_backend_uses_patched_starlette_and_explicit_httpx2_test_client() -> None: pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml" content = pyproject.read_text(encoding="utf-8") - assert '"starlette>=0.46.0,<1.0.0"' in content + assert '"starlette>=1.3.1,<2.0.0"' in content + assert '"httpx2>=2.0.0"' in content def test_readiness_gate_treats_deprecation_warnings_as_errors() -> None: diff --git a/backend/tests/test_release_backup_snapshot.py b/backend/tests/test_release_backup_snapshot.py new file mode 100644 index 00000000..41fd411c --- /dev/null +++ b/backend/tests/test_release_backup_snapshot.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "release_backup_snapshot.py" + + +def load_snapshot_module(): + spec = importlib.util.spec_from_file_location("release_backup_snapshot_test", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_snapshot_is_byte_complete_and_reuses_only_verified_backup_bytes(tmp_path: Path) -> None: + snapshot = load_snapshot_module() + source = tmp_path / "source" + source.mkdir() + (source / "same.bin").write_bytes(b"unchanged") + (source / "changed.bin").write_bytes(b"before") + (source / "empty").mkdir() + + prior = tmp_path / "prior" + prior_manifest = tmp_path / "prior.tsv" + snapshot.create_snapshot(source, prior, prior_manifest, label="storage") + snapshot.verify_snapshot(prior, prior_manifest) + + (source / "changed.bin").write_bytes(b"after") + current = tmp_path / "current" + current_manifest = tmp_path / "current.tsv" + snapshot.create_snapshot( + source, + current, + current_manifest, + label="storage", + link_dest_snapshot=prior, + link_dest_manifest=prior_manifest, + ) + snapshot.verify_snapshot(current, current_manifest) + + assert os.path.samefile(prior / "same.bin", current / "same.bin") + assert not os.path.samefile(prior / "changed.bin", current / "changed.bin") + assert (current / "changed.bin").read_bytes() == b"after" + assert (current / "empty").is_dir() + + +def test_snapshot_rejects_symlinked_content(tmp_path: Path) -> None: + snapshot = load_snapshot_module() + source = tmp_path / "source" + source.mkdir() + target = source / "target.bin" + target.write_bytes(b"target") + try: + (source / "link.bin").symlink_to(target) + except OSError: + pytest.skip("Symlink creation is unavailable on this host") + + with pytest.raises(RuntimeError, match="refuses symlinked content"): + snapshot.create_snapshot(source, tmp_path / "snapshot", tmp_path / "manifest.tsv", label="storage") + + +def test_snapshot_verification_rejects_changed_retained_bytes(tmp_path: Path) -> None: + snapshot = load_snapshot_module() + source = tmp_path / "source" + source.mkdir() + (source / "artifact.bin").write_bytes(b"retained") + retained = tmp_path / "snapshot" + manifest = tmp_path / "manifest.tsv" + snapshot.create_snapshot(source, retained, manifest, label="storage") + (retained / "artifact.bin").chmod(0o644) + (retained / "artifact.bin").write_bytes(b"tampered") + + with pytest.raises(RuntimeError, match="checksum differs"): + snapshot.verify_snapshot(retained, manifest) diff --git a/backend/tests/test_sprint31_unraid_template.py b/backend/tests/test_sprint31_unraid_template.py index ff7a78d4..77ac2838 100644 --- a/backend/tests/test_sprint31_unraid_template.py +++ b/backend/tests/test_sprint31_unraid_template.py @@ -150,7 +150,7 @@ def test_tower_deploy_uses_single_container_unraid_compose() -> None: assert "LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh" in release_script -def test_tower_deploy_build_uses_remote_env_ai_setting_by_default() -> None: +def test_tower_deploy_build_requires_the_production_ai_variant_by_default() -> None: powershell = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8") bash = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8") release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8") @@ -161,7 +161,8 @@ def test_tower_deploy_build_uses_remote_env_ai_setting_by_default() -> None: assert "if [ -f .env ]; then" in release_script assert ". ./.env" in release_script - assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-false}"' in release_script + assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"' in release_script + assert "Production release deployment requires the gated AI image" in release_script assert "--build-arg GEOINTEL_INSTALL_AI=" in release_script diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 2e70b5b1..97c55826 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -160,8 +160,10 @@ COPY scripts/migrate_runtime_model_provenance.py /app/scripts/migrate_runtime_mo COPY scripts/archive_technical_projects.py /app/scripts/archive_technical_projects.py COPY scripts/runtime_state_report.py /app/scripts/runtime_state_report.py COPY scripts/release_backup_guard.py /app/scripts/release_backup_guard.py +COPY scripts/release_backup_snapshot.py /app/scripts/release_backup_snapshot.py COPY scripts/audit_data_operations.py /app/scripts/audit_data_operations.py COPY scripts/cleanup_storage_artifacts.py /app/scripts/cleanup_storage_artifacts.py +COPY scripts/restore_storage_quarantine.py /app/scripts/restore_storage_quarantine.py COPY deploy/unraid/nginx-all-in-one.conf /etc/nginx/conf.d/default.conf COPY deploy/unraid/all-in-one-start.sh /usr/local/bin/geointel-all-in-one-start COPY --from=frontend-build /frontend/dist/ /usr/share/nginx/html/ diff --git a/deploy/unraid/MANUAL_DEPLOY.md b/deploy/unraid/MANUAL_DEPLOY.md index 96616161..678834a1 100644 --- a/deploy/unraid/MANUAL_DEPLOY.md +++ b/deploy/unraid/MANUAL_DEPLOY.md @@ -11,9 +11,8 @@ met de naam `geointel`, bereikbaar op `http://192.168.10.150:1202`. Kopieer de **volledige** map `C:\Projects\geointel` naar `/mnt/user/appdata/geointel` op de server. Verder niets uitzoeken. -`.dockerignore` regelt de rest: `node_modules/`, `.git/`, de dubbele -`geointel/`-map, `docs/`, `artifacts/` en testoutput gaan niet mee de -build-context in, ook al staan ze in de map. +`.dockerignore` regelt de rest: `node_modules/`, `.git/`, `docs/`, `artifacts/` +en testoutput gaan niet mee de build-context in, ook al staan ze in de map. Eén waarschuwing bij het overschrijven: laat `storage/`, `postgres-data/`, `backups/` en `models/` op de server **staan**. Dat is je bestaande data, en @@ -71,7 +70,8 @@ die hostname in `GEOINTEL_CORS_ORIGINS` staan — anders blokkeert de browser de API-calls vanaf het publieke adres. De backend doet geen host-validatie, dus verder is er niets nodig aan applicatiekant. -Voor GPU-inferentie (optioneel, kan ook later): +De productie-image bevat altijd de gepinde AI-runtime. Inferentie zelf kan +uitblijven totdat een lokaal, gevalideerd model beschikbaar is: ```env GEOINTEL_INSTALL_AI=true @@ -81,8 +81,8 @@ YOLO_REQUIRE_CUDA=true YOLO_MODEL_PATH=/app/models/.pt ``` -Laat `GEOINTEL_INSTALL_AI=false` staan als je eerst gewoon wilt dat de app -draait — dat scheelt een paar GB aan PyTorch-lagen in de build. +Laat `YOLO_ENABLED=false` zolang er geen geschikt modelbestand is. Het +release-deployscript weigert bewust een GIS-only productie-image. --- @@ -134,12 +134,21 @@ bash deploy/unraid/deploy-release.sh Het script: -1. ruimt een eventueel achtergebleven Compose-stack op (ook de oude +1. bindt het exacte huidige image-ID aan een unieke backup-specifieke + `rollback-predeploy-*`-tag; +2. bouwt of hergebruikt de AI-candidate terwijl de huidige release beschikbaar + blijft, legt het exacte lokale image-ID vast en maakt daarop SBOM- en + Trivy-evidence; +3. controleert vóór het pauzeren de vrije ruimte en maakt vervolgens een + byte-complete, SHA-256-geverifieerde database-, storage- en modelsnapshot in + `/mnt/user/appdata/geointel/backups`; ongewijzigde bestanden mogen alleen + vanuit een oudere geverifieerde backup worden gehardlinkt; +4. ruimt een eventueel achtergebleven Compose-stack op (ook de oude 3-container dev-stack die óók poort 1202 pakt); -2. bewaart de huidige image als `geointel-all-in-one:previous`; -3. bouwt `deploy/unraid/Dockerfile.all-in-one`; -4. start één container `geointel` met `-p 1202:80` en `--gpus all`; -5. rolt automatisch terug naar `:previous` als de healthcheck of smoke faalt. +5. start één container `geointel` met `-p 1202:80` en `--gpus all`; +6. bewijst een rollbackdump eerst in een geïsoleerde tijdelijke database, + bewaart de oude productiedatabase als herstelpad en start pas daarna + automatisch het image-ID dat cryptografisch in die pre-deploybackup staat. De eerste build duurt lang (PostGIS + GDAL + npm build). Volgende deploys hergebruiken de Docker-layercache. @@ -165,24 +174,13 @@ docker logs --tail 200 geointel --- -## 7. Belangrijk: dubbele projectmap lokaal +## 7. Geretireerde dubbele projectmap -In `C:\Projects\geointel` staat een tweede, volledige kopie van het project -onder `C:\Projects\geointel\geointel\`. Die bevat dezelfde bestanden en -dezelfde datum, maar staat buiten git. Zolang die er staat: - -- wordt de Docker build-context onnodig verdubbeld; -- weet je bij het bewerken van bijvoorbeeld `docker-compose.unraid.yml` niet - welke versie je te pakken hebt. - -`.dockerignore` sluit hem nu uit, maar ruim hem op zodra je zeker weet dat er -niets unieks in staat. Vergelijk eerst: - -```powershell -robocopy C:\Projects\geointel\geointel C:\Projects\geointel /L /E /NJH /NJS /NDL /XF *.pyc -``` - -Regels die als `New File` verschijnen bestaan alleen in de kopie. +De vroegere geneste mirror `C:\Projects\geointel\geointel` is geretireerd. De +immutable Git-herkomst, niet-getrackte recoverybestanden en verificatiegrens +staan in `docs/accuracy-program/13-nested-mirror-retirement.md`. Behandel die +evidence als herstelreferentie; deze handleiding vraagt geen extra kopieer- of +opruimactie. --- @@ -190,11 +188,15 @@ Regels die als `New File` verschijnen bestaan alleen in de kopie. ```bash cd /mnt/user/appdata/geointel -bash deploy/unraid/rollback-dockerman-container.sh +bash deploy/unraid/rollback-dockerman-container.sh \ + --backup-dir /mnt/user/appdata/geointel/backups/ \ + --confirm-production-database-restore ``` -Rollback hergebruikt dezelfde PostGIS- en storage-paden en draait nooit een -Alembic-downgrade. +Rollback hergebruikt dezelfde storage-paden, bewijst de geverifieerde dump +eerst geïsoleerd, wisselt daarna databases via no-clobber namen en draait nooit +een Alembic-downgrade. De oude productiedatabase blijft staan totdat een +operator haar na controle expliciet opruimt. --- @@ -202,7 +204,7 @@ Alembic-downgrade. | Bestand | Aanpassing | |---|---| -| `.dockerignore` | Sluit root-`node_modules`, de dubbele `geointel/`-map, `.git`, `docs/`, `artifacts/` en testoutput uit de build-context | +| `.dockerignore` | Sluit root-`node_modules`, `.git`, `docs/`, `artifacts/` en testoutput uit de build-context | | `deploy/unraid/deploy-release.sh` | `git rev-parse HEAD` crashte op een kopie zonder `.git`. Valt nu terug op `GEOINTEL_BUILD_SHA`, een `RELEASE_SHA`-bestand of een content-hash van de broncode | | `deploy/unraid/deploy-release.sh` | Smoke-scripts worden op bestaan getest in plaats van op de execute-bit, die bij een Windows-kopie verloren gaat | | `deploy/unraid/run-dockerman-container.sh` | Ruimt expliciet zowel `docker-compose.yml` (3 containers) als `docker-compose.unraid.yml` op, zodat poort 1202 gegarandeerd vrij is | diff --git a/deploy/unraid/README.md b/deploy/unraid/README.md index 29eba81e..8382084f 100644 --- a/deploy/unraid/README.md +++ b/deploy/unraid/README.md @@ -106,27 +106,25 @@ customer or operational data. Deploy a separate demo container and storage root for public or recruiter-facing access. The repository deploy scripts run the same flow automatically. They validate -the Compose reference, preserve the current image as -`geointel-all-in-one:previous`, build an immutable `-ai` or -`-gis` tag plus `latest`, install the DockerMan metadata and start -the immutable image. An existing matching tag is reused, never rebuilt. A -failed start, live migration smoke or browser/API smoke automatically attempts -the previous image without changing the configured PostGIS or storage paths. +the Compose reference, preserve the current image under a unique +backup-specific `rollback-predeploy-*` tag, build the immutable production +`-ai` tag plus `latest`, attest its exact local image ID, generate +an SBOM and enforce the Trivy policy before starting that same ID. An existing +matching tag is reused, never rebuilt. A failed start, live migration smoke or +browser/API smoke automatically attempts the previous image without changing +the configured PostGIS or storage paths. `scripts/deploy_tower.sh` and `scripts/deploy_tower.ps1` source the remote -`.env` before building the image. That means `GEOINTEL_INSTALL_AI=true` in -`/mnt/user/appdata/geointel/.env` is enough for the automatic deploy to build -the AI-enabled image. Set `GEOINTEL_INSTALL_AI` in the local shell or pass -`-InstallAi true/false` to the PowerShell wrapper only when you intentionally -want to override the remote `.env` for that deploy. +`.env` before building the image. Production deployment requires +`GEOINTEL_INSTALL_AI=true`; an explicit false value fails closed before the +image or running container is replaced. Database credentials are runtime configuration, not image metadata. The all-in-one image does not bake `GEOINTEL_POSTGRES_PASSWORD` into the Dockerfile; set it through `.env`, the Unraid template or `docker run -e`. -AI dependencies are opt-in. Leave `GEOINTEL_INSTALL_AI=false` for the default -GIS-only image. Set `GEOINTEL_INSTALL_AI=true`, mount models through -`GEOINTEL_MODELS_PATH` and configure `YOLO_ENABLED=true` plus +Production images always include the pinned AI dependencies. Mount models +through `GEOINTEL_MODELS_PATH` and configure `YOLO_ENABLED=true` plus `YOLO_MODELS_DIR=/app/models` and `YOLO_MODEL_PATH=/app/models/.pt` only when you have a local model file. The AI-enabled image installs PyTorch/Ultralytics plus the native OpenCV runtime @@ -264,7 +262,8 @@ git reset --hard origin/main bash deploy/unraid/deploy-release.sh ``` -The equivalent low-level build remains available for debugging: +The equivalent low-level GIS-only build remains available only for local +debugging; it is not a production deployment path: ```bash docker build --build-arg GEOINTEL_INSTALL_AI=${GEOINTEL_INSTALL_AI:-false} -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest . @@ -305,20 +304,27 @@ bash scripts/verify_release_upgrade_smoke.sh \ Return to the image that was active immediately before the latest deployment: ```bash -bash deploy/unraid/rollback-dockerman-container.sh +bash deploy/unraid/rollback-dockerman-container.sh \ + --backup-dir /mnt/user/appdata/geointel/backups/ \ + --confirm-production-database-restore ``` For an older retained commit, select its immutable tag explicitly: ```bash GEOINTEL_ROLLBACK_IMAGE=geointel-all-in-one:-ai \ - bash deploy/unraid/rollback-dockerman-container.sh + bash deploy/unraid/rollback-dockerman-container.sh \ + --backup-dir /mnt/user/appdata/geointel/backups/ \ + --confirm-production-database-restore ``` -Rollback reuses the configured PostGIS and storage mounts and never runs an -Alembic downgrade. If a future release has a backward-incompatible migration, -restore its verified pre-release backup instead of forcing an older app -against a newer schema. +Rollback restores and verifies the selected dump in an isolated proof database +before any production replacement. It then swaps database names, retains the +pre-restore production database for operator recovery, reuses the configured +storage mount and never runs an Alembic downgrade or an older app against an +unknown newer schema. Remove the retained recovery database and old backup +directories only in a separately reviewed operator retention step; deployment +never deletes them automatically. The configured upload limit is shared by FastAPI and the generated nginx runtime configuration. Values outside `1..2048` MiB are rejected before the @@ -339,7 +345,9 @@ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py ``` The full backup, confirmation, candidate-limit and apply sequence is in -`docs/DATA_OPERATIONS_RUNBOOK.md`. GeoIntel installs no automatic cleanup +`docs/DATA_OPERATIONS_RUNBOOK.md`. Apply moves bytes to protected, +checksum-bound quarantine rather than deleting them; a separate confirmed +restore command reverses the move. GeoIntel installs no automatic cleanup schedule. ## Safe cleanup diff --git a/deploy/unraid/deploy-release.sh b/deploy/unraid/deploy-release.sh index e606fd7d..2ec08faa 100644 --- a/deploy/unraid/deploy-release.sh +++ b/deploy/unraid/deploy-release.sh @@ -26,7 +26,11 @@ if [ -n "${DEPLOY_GEOINTEL_INSTALL_AI:-}" ]; then GEOINTEL_INSTALL_AI="$DEPLOY_GEOINTEL_INSTALL_AI" fi -GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-false}" +GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}" +if [ "$GEOINTEL_INSTALL_AI" != "true" ]; then + echo "Production release deployment requires the gated AI image (GEOINTEL_INSTALL_AI=true)." >&2 + exit 2 +fi GEOINTEL_APP_VERSION="$(tr -d '[:space:]' < VERSION)" if ! [[ "$GEOINTEL_APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then echo "Invalid semantic version in VERSION: ${GEOINTEL_APP_VERSION}" >&2 @@ -57,7 +61,52 @@ source_tree_hash() { } resolve_build_sha() { - local head="" content="" + local head="" content="" controller_sha="" controller_source="" + + if [ -n "${GITEA_COMMIT_SHA:-}" ]; then + controller_sha="$GITEA_COMMIT_SHA" + controller_source="GITEA_COMMIT_SHA" + fi + if [ -n "${GITHUB_SHA:-}" ]; then + if ! [[ "$GITHUB_SHA" =~ ^[0-9A-Fa-f]{40}$ ]]; then + echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2 + return 2 + fi + if [ -n "$controller_sha" ] && [ "${controller_sha,,}" != "${GITHUB_SHA,,}" ]; then + echo "Controller commit variables disagree." >&2 + return 2 + fi + controller_sha="$GITHUB_SHA" + controller_source="${controller_source:-GITHUB_SHA}" + fi + if [ -n "$controller_sha" ]; then + if ! [[ "$controller_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then + echo "${controller_source} must contain one full 40-character Git commit SHA." >&2 + return 2 + fi + controller_sha="${controller_sha,,}" + if [ -n "${GEOINTEL_BUILD_SHA:-}" ] && [ "${GEOINTEL_BUILD_SHA,,}" != "$controller_sha" ]; then + echo "Explicit build revision differs from the controller revision." >&2 + return 2 + fi + if command -v git >/dev/null 2>&1 && git rev-parse --git-dir >/dev/null 2>&1; then + head="$(git rev-parse HEAD 2>/dev/null || true)" + if [ "${head,,}" != "$controller_sha" ]; then + echo "Prepared Git checkout does not match the controller revision." >&2 + return 2 + fi + if [ -n "$(git status --porcelain 2>/dev/null)" ]; then + echo "Prepared Git checkout contains changes outside the controller revision." >&2 + return 2 + fi + fi + printf '%s' "$controller_sha" + return 0 + fi + if [ -n "${GITEA_REPOSITORY:-}" ] || [ -n "${GITHUB_REPOSITORY:-}" ]; then + echo "Automated deployment context is missing GITEA_COMMIT_SHA/GITHUB_SHA." >&2 + return 2 + fi # 1. Explicit override wins. if [ -n "${GEOINTEL_BUILD_SHA:-}" ]; then @@ -107,7 +156,13 @@ if [ -z "$GEOINTEL_BUILD_SHA" ]; then echo "Could not determine a build revision for this deployment." >&2 exit 2 fi +export GEOINTEL_BUILD_SHA echo "Build revision: ${GEOINTEL_BUILD_SHA}" +GEOINTEL_RELEASE_TOKEN="$(printf '%s' "$GEOINTEL_BUILD_SHA" | tr -c 'A-Za-z0-9._-' '_' | cut -c1-48)" +if [ -z "$GEOINTEL_RELEASE_TOKEN" ]; then + echo "Could not derive a safe release evidence identifier." >&2 + exit 2 +fi GEOINTEL_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" GEOINTEL_IMAGE_REPOSITORY="${GEOINTEL_IMAGE_REPOSITORY:-geointel-all-in-one}" if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then @@ -116,8 +171,23 @@ else GEOINTEL_RELEASE_VARIANT="gis" fi GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}" -GEOINTEL_PREVIOUS_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:previous" FRONTEND_URL="${FRONTEND_URL:-http://127.0.0.1:${GEOINTEL_FRONTEND_PORT:-1202}}" +GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}" +GEOINTEL_STORAGE_PATH="${GEOINTEL_STORAGE_PATH:-/mnt/user/appdata/geointel/storage}" +GEOINTEL_MODELS_PATH="${GEOINTEL_MODELS_PATH:-/mnt/user/appdata/geointel/models}" +GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}" +GEOINTEL_DEPLOY_EVIDENCE_DIR="${GEOINTEL_DEPLOY_EVIDENCE_DIR:-artifacts/release-evidence/deploy/${GEOINTEL_RELEASE_TOKEN}-ai}" +GEOINTEL_PREDEPLOY_BACKUP_DIR="" +GEOINTEL_RELEASE_IMAGE_ID="" +GEOINTEL_BACKUP_LINK_DEST="" +GEOINTEL_PREDEPLOY_ROLLBACK_TAG="" + +case "$GEOINTEL_DEPLOY_EVIDENCE_DIR" in + /*|*..*) + echo "Deployment evidence directory must be repository-relative and must not contain '..'." >&2 + exit 2 + ;; +esac wait_for_geointel_health() { local status="" @@ -141,35 +211,312 @@ wait_for_geointel_health() { start_image() { local image="$1" + local running_image_id="" + local running_revision="" + local running_ai="" GEOINTEL_IMAGE="$image" bash deploy/unraid/run-dockerman-container.sh wait_for_geointel_health + running_image_id="$(docker inspect --format '{{.Image}}' geointel)" + if [ "$running_image_id" != "$image" ]; then + echo "Running container image ${running_image_id} differs from attested image ${image}." >&2 + return 1 + fi + running_revision="$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' geointel)" + running_ai="$(docker inspect --format '{{index .Config.Labels "io.geointel.ai.enabled"}}' geointel)" + if [ "$running_revision" != "$GEOINTEL_BUILD_SHA" ] || [ "$running_ai" != "true" ]; then + echo "Running container labels do not match the attested AI revision." >&2 + return 1 + fi + echo "Running container matches attested image: ${running_image_id}" +} + +scan_release_image() { + local scanned_image_id="" + local current_image_id="" + local inspect_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/image-inspect.json" + local sbom_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-sbom.spdx.json" + local vulnerability_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-container-vulnerabilities.json" + local attestation_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/deployment-attestation.json" + + scanned_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")" + test -n "$scanned_image_id" + mkdir -p "$ROOT/$GEOINTEL_DEPLOY_EVIDENCE_DIR" + docker image inspect "$GEOINTEL_RELEASE_IMAGE" > "$ROOT/$inspect_output" + bash scripts/generate_container_sbom.sh "$GEOINTEL_RELEASE_IMAGE" "$sbom_output" + bash scripts/scan_container_image.sh "$GEOINTEL_RELEASE_IMAGE" "$vulnerability_output" + current_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")" + if [ "$current_image_id" != "$scanned_image_id" ]; then + echo "Release image tag changed while SBOM/scan evidence was being generated." >&2 + return 1 + fi + test -s "$ROOT/$inspect_output" + test -s "$ROOT/$sbom_output" + test -s "$ROOT/$vulnerability_output" + GEOINTEL_RELEASE_IMAGE_ID="$scanned_image_id" + python3 - \ + "$ROOT/$attestation_output" \ + "$GEOINTEL_RELEASE_IMAGE" \ + "$GEOINTEL_RELEASE_IMAGE_ID" \ + "$GEOINTEL_BUILD_SHA" \ + "$inspect_output" \ + "$sbom_output" \ + "$vulnerability_output" <<'PY' +import datetime +import json +import pathlib +import sys + +output, image_tag, image_id, revision, inspect_path, sbom_path, vulnerability_path = sys.argv[1:] +payload = { + "schema_version": 1, + "attested_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "image_tag": image_tag, + "image_id": image_id, + "image_config_digest": image_id, + "revision": revision, + "variant": "ai", + "evidence": { + "image_inspect": inspect_path, + "sbom": sbom_path, + "vulnerabilities": vulnerability_path, + }, +} +path = pathlib.Path(output) +temporary = path.with_suffix(".json.partial") +temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") +temporary.replace(path) +PY + test -s "$ROOT/$attestation_output" + echo "Exact deployment image scanned: ${GEOINTEL_RELEASE_IMAGE_ID}" +} + +preflight_backup_capacity() { + local database_name="" + local database_user="" + local database_size_bytes="" + + database_name="$(docker exec geointel sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"')" + database_user="$(docker exec geointel sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"')" + database_size_bytes="$(docker exec geointel psql -X -v ON_ERROR_STOP=1 \ + -U "$database_user" -d "$database_name" -Atqc \ + 'SELECT pg_database_size(current_database());')" + mkdir -p "$GEOINTEL_BACKUPS_PATH" + python3 - \ + "$GEOINTEL_BACKUPS_PATH" \ + "$GEOINTEL_STORAGE_PATH" \ + "$GEOINTEL_MODELS_PATH" \ + "$database_size_bytes" <<'PY' +import os +import pathlib +import shutil +import stat +import sys + +backup_root = pathlib.Path(sys.argv[1]).expanduser().resolve() +sources = [pathlib.Path(value).expanduser().resolve() for value in sys.argv[2:4]] +database_bytes = int(sys.argv[4]) + +def retained_bytes(root: pathlib.Path) -> int: + if not root.is_dir(): + raise SystemExit(f"Mandatory snapshot source is not a directory: {root}") + total = 0 + for current, directories, files in os.walk(root, topdown=True, followlinks=False): + current_path = pathlib.Path(current) + for name in [*directories, *files]: + path = current_path / name + details = path.lstat() + if stat.S_ISLNK(details.st_mode): + raise SystemExit(f"Mandatory snapshot refuses symlinked content: {path}") + if name in directories and not stat.S_ISDIR(details.st_mode): + raise SystemExit(f"Snapshot directory changed during capacity preflight: {path}") + if name in files: + if not stat.S_ISREG(details.st_mode): + raise SystemExit(f"Mandatory snapshot refuses non-regular content: {path}") + total += details.st_size + return total + +source_bytes = sum(retained_bytes(source) for source in sources) +# Reflink clones are used when the backing filesystem supports them. Budget for +# a complete copy plus two uncompressed database sizes (dump and isolated +# restore/cutover recovery) so fallback still fails before the live backend is +# quiesced rather than midway through the snapshot. +required = source_bytes + (2 * database_bytes) +headroom = max(5 * 1024**3, required // 10) +free = shutil.disk_usage(backup_root).free +if free < required + headroom: + raise SystemExit( + "Insufficient free space for a fail-safe predeploy snapshot: " + f"required={required + headroom} free={free} source={source_bytes} database={database_bytes}" + ) +print( + "Predeploy snapshot capacity: " + f"source_bytes={source_bytes} database_bytes={database_bytes} free_bytes={free}" +) +PY +} + +select_verified_link_dest() { + local candidate="" + GEOINTEL_BACKUP_LINK_DEST="" + while IFS= read -r candidate; do + if ( + cd "$candidate" \ + && sha256sum -c CHECKSUMS.sha256 >/dev/null \ + && python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate" + ); then + GEOINTEL_BACKUP_LINK_DEST="$candidate" + echo "Using verified prior byte snapshot as link-dest: ${candidate}" + return 0 + fi + echo "Skipping unusable prior backup link-dest: ${candidate}" >&2 + done < <( + python3 - "$GEOINTEL_BACKUPS_PATH" <<'PY' +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]).expanduser().resolve() +candidates = sorted( + ( + path + for path in root.iterdir() + if path.is_dir() and not path.name.startswith(".") and (path / "manifest.json").is_file() + ), + key=lambda path: path.stat().st_mtime_ns, + reverse=True, +) +for candidate in candidates: + print(candidate) +PY + ) + echo "No verified prior byte snapshot found; this deployment will create a full first snapshot." +} + +create_predeploy_backup() { + local container_exists="false" + local container_running="false" + local release_id="" + local backup_link_args=() + local current_image_id="" + local existing_rollback_id="" + + if docker ps -a --format '{{.Names}}' | grep -Fxq geointel; then + container_exists="true" + fi + if [ "$(docker inspect -f '{{.State.Running}}' geointel 2>/dev/null || true)" = "true" ]; then + container_running="true" + fi + + if [ "$container_exists" = "false" ]; then + if [ -f "$GEOINTEL_POSTGIS_DATA_PATH/PG_VERSION" ]; then + echo "PostGIS data exists without a running GeoIntel container; refusing an unbacked migration." >&2 + return 1 + fi + echo "No existing GeoIntel state found; pre-deploy backup is not required for this fresh install." + return 0 + fi + if [ "$container_running" != "true" ]; then + echo "Existing GeoIntel container is not running; refusing deployment because a consistent backup cannot be created." >&2 + return 1 + fi + + current_image_id="$(docker inspect --format '{{.Image}}' geointel)" + if ! [[ "$current_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "Running release does not expose one immutable Docker image ID." >&2 + return 1 + fi + release_id="predeploy-${GEOINTEL_RELEASE_TOKEN:0:24}-$(date -u +%Y%m%dT%H%M%SZ)-$$" + GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}" + existing_rollback_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" 2>/dev/null || true)" + if [ -n "$existing_rollback_id" ] && [ "$existing_rollback_id" != "$current_image_id" ]; then + echo "Backup-specific rollback tag already identifies different image bytes." >&2 + return 1 + fi + docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" + + # This conservative full-copy fallback estimate runs while the existing + # release is still healthy. Verified backup-to-backup hardlinks normally + # avoid recopying unchanged bytes, but are never assumed for this fail-closed + # capacity decision. + preflight_backup_capacity + select_verified_link_dest + if [ -n "$GEOINTEL_BACKUP_LINK_DEST" ]; then + backup_link_args=(--link-dest-backup "$GEOINTEL_BACKUP_LINK_DEST") + fi + + echo "Quiescing the current backend so the rollback point cannot miss concurrent writes..." + if ! docker exec -i geointel python - <<'PY' +import os +import pathlib +import signal +import time + +matches = [] +for item in pathlib.Path("/proc").iterdir(): + if not item.name.isdigit() or int(item.name) in {os.getpid(), os.getppid()}: + continue + try: + command = (item / "cmdline").read_bytes().replace(b"\0", b" ") + except (OSError, PermissionError): + continue + if b"uvicorn" in command and b"app.main:app" in command: + matches.append(int(item.name)) +if not matches: + raise SystemExit("Could not identify the running GeoIntel backend") +for process_id in matches: + os.kill(process_id, signal.SIGTERM) +deadline = time.monotonic() + 60 +remaining = matches +while remaining and time.monotonic() < deadline: + time.sleep(0.25) + remaining = [process_id for process_id in remaining if pathlib.Path(f"/proc/{process_id}").exists()] +if remaining: + raise SystemExit(f"Backend did not stop cleanly: {remaining}") +print(f"Stopped {len(matches)} backend process(es)") +PY + then + echo "Could not quiesce the current backend; refusing a potentially inconsistent backup." >&2 + docker restart geointel >/dev/null || true + wait_for_geointel_health || true + return 1 + fi + + GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}" + echo "Creating mandatory pre-deploy backup ${release_id}..." + if ! bash scripts/backup_release_state.sh \ + --container geointel \ + --output-root "$GEOINTEL_BACKUPS_PATH" \ + --release-id "$release_id" \ + --storage-path "$GEOINTEL_STORAGE_PATH" \ + --models-path "$GEOINTEL_MODELS_PATH" \ + --inventory-mode sha256 \ + --rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \ + "${backup_link_args[@]}" \ + || ! bash scripts/verify_release_backup.sh \ + --container geointel \ + --backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then + echo "Pre-deploy backup failed; restarting the unchanged current release." >&2 + docker restart geointel >/dev/null || true + wait_for_geointel_health || true + GEOINTEL_PREDEPLOY_BACKUP_DIR="" + return 1 + fi + echo "Pre-deploy backup verified: ${GEOINTEL_PREDEPLOY_BACKUP_DIR}" } rollback_previous() { - if ! docker image inspect "$GEOINTEL_PREVIOUS_IMAGE" >/dev/null 2>&1; then - echo "Automatic rollback unavailable: ${GEOINTEL_PREVIOUS_IMAGE} does not exist." >&2 + if [ -z "$GEOINTEL_PREDEPLOY_BACKUP_DIR" ]; then + echo "Automatic rollback unavailable: no verified pre-deploy database backup was created." >&2 return 1 fi - echo "Rolling back to ${GEOINTEL_PREVIOUS_IMAGE}..." - start_image "$GEOINTEL_PREVIOUS_IMAGE" + echo "Rolling back database and image to the verified pre-deploy state..." + GEOINTEL_DEPLOY_LOCK_HELD=true \ + bash deploy/unraid/rollback-dockerman-container.sh \ + --backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR" \ + --confirm-production-database-restore } docker compose -f docker-compose.unraid.yml config >/dev/null -current_image_id="$(docker inspect --format '{{.Image}}' geointel 2>/dev/null || true)" -release_image_id="$( - docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE" 2>/dev/null || true -)" -if ( - [ -n "$current_image_id" ] && - [ "$current_image_id" != "$release_image_id" ] && - docker image inspect "$current_image_id" >/dev/null 2>&1 -); then - docker tag "$current_image_id" "$GEOINTEL_PREVIOUS_IMAGE" -elif [ -n "$current_image_id" ] && [ "$current_image_id" = "$release_image_id" ]; then - echo "Current container already uses ${GEOINTEL_RELEASE_IMAGE}; preserving the existing previous image." -fi - if docker image inspect "$GEOINTEL_RELEASE_IMAGE" >/dev/null 2>&1; then stored_revision="$( docker image inspect \ @@ -208,7 +555,10 @@ else . fi -if ! start_image "$GEOINTEL_RELEASE_IMAGE"; then +scan_release_image +create_predeploy_backup + +if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"; then rollback_previous || true exit 1 fi diff --git a/deploy/unraid/geointel-unraid-template.xml b/deploy/unraid/geointel-unraid-template.xml index 6aae8de3..611d8420 100644 --- a/deploy/unraid/geointel-unraid-template.xml +++ b/deploy/unraid/geointel-unraid-template.xml @@ -32,11 +32,17 @@ change-me-before-shared-use http://localhost:1202,http://127.0.0.1:1202,http://192.168.10.150:1202 500 + 64 false 43200 + http://localhost:1202 + + + + true Gast 7200 diff --git a/deploy/unraid/geointel.env.example b/deploy/unraid/geointel.env.example index f9867917..1592d24b 100644 --- a/deploy/unraid/geointel.env.example +++ b/deploy/unraid/geointel.env.example @@ -34,6 +34,8 @@ GEOINTEL_CORS_ORIGINS=https://geointel.itworx.tech,http://geointel.itworx.tech,h # Upload guard in MiB. The same 1-2048 limit is applied by nginx and FastAPI. GEOINTEL_MAX_UPLOAD_MB=500 +# Maximum decompressed vector payload processed fully in memory (1-256 MiB). +GEOINTEL_MAX_IN_MEMORY_VECTOR_MB=64 GEOINTEL_AOI_WORKER_ENABLED=true GEOINTEL_AOI_WORKER_POLL_SECONDS=2 @@ -46,6 +48,14 @@ GEOINTEL_AUTH_PASSWORD_HASH= GEOINTEL_AUTH_SESSION_SECRET= GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200 +# Optional additive Authentik OIDC login. Configure all fields together. The +# local operator credentials above remain the recovery login. +GEOINTEL_PUBLIC_BASE_URL=http://localhost:1202 +GEOINTEL_AUTHENTIK_ISSUER= +GEOINTEL_AUTHENTIK_CLIENT_ID= +GEOINTEL_AUTHENTIK_CLIENT_SECRET= +GEOINTEL_AUTHENTIK_ALLOWED_EMAIL= + # Guest access is enabled by default whenever operator authentication is active. # It opens the seeded GeoIntel demo in a temporary, API-enforced restricted # session. Set this to false on installations containing private project data. @@ -153,8 +163,9 @@ SPW_TERRAIN_ANALYSIS_RESOLUTION_M=5 SPW_TERRAIN_MAX_SIDE_M=20000 SPW_TERRAIN_MAX_PIXELS=12000000 -# Optional configured-YOLO runtime. Keep disabled unless a local model is mounted. -GEOINTEL_INSTALL_AI=false +# Production releases always contain the pinned AI dependencies. Inference may +# remain disabled until an integrity-bound local model is mounted. +GEOINTEL_INSTALL_AI=true YOLO_ENABLED=false YOLO_MODELS_DIR=/app/models YOLO_MODEL_PATH= diff --git a/deploy/unraid/nginx-all-in-one.conf b/deploy/unraid/nginx-all-in-one.conf index cf530a3d..d1067d7e 100644 --- a/deploy/unraid/nginx-all-in-one.conf +++ b/deploy/unraid/nginx-all-in-one.conf @@ -1,3 +1,18 @@ +geo $geointel_trusted_forwarder { + default 0; + 127.0.0.0/8 1; + ::1/128 1; + # The outer Nginx Proxy Manager reaches this container through Docker's + # internal bridge; public/LAN clients are not trusted forwarders. + 172.16.0.0/12 1; +} + +map "$geointel_trusted_forwarder:$http_x_forwarded_proto" $geointel_forwarded_proto { + default $scheme; + "1:https" https; + "1:http" http; +} + server { listen 80; server_name _; @@ -5,21 +20,42 @@ server { proxy_read_timeout 600s; proxy_send_timeout 600s; + add_header Content-Security-Policy "frame-ancestors 'none'" always; + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; + root /usr/share/nginx/html; index index.html; location = /index.html { add_header Cache-Control "no-cache"; + add_header Content-Security-Policy "frame-ancestors 'none'" always; + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; try_files /index.html =404; } location = /geointel-icon.svg { add_header Cache-Control "public, max-age=3600"; + add_header Content-Security-Policy "frame-ancestors 'none'" always; + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; try_files /geointel-icon.svg =404; } location = /geointel-icon.png { add_header Cache-Control "public, max-age=3600"; + add_header Content-Security-Policy "frame-ancestors 'none'" always; + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; # Preserve the stable DockerMan/public URL while the frontend keeps # its explicit 32px and 180px icon variants. try_files /geointel-icon-180.png =404; @@ -27,6 +63,11 @@ server { location /assets/ { add_header Cache-Control "no-cache"; + add_header Content-Security-Policy "frame-ancestors 'none'" always; + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; try_files $uri =404; } @@ -36,7 +77,7 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto; } location = /health { @@ -45,7 +86,7 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto; } location = /health/live { diff --git a/deploy/unraid/restore-predeploy-database.sh b/deploy/unraid/restore-predeploy-database.sh new file mode 100644 index 00000000..742ac8da --- /dev/null +++ b/deploy/unraid/restore-predeploy-database.sh @@ -0,0 +1,323 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Restore the production database from a verified pre-deploy dump while the +# normal GeoIntel container is stopped. This is intentionally a separate, +# explicitly confirmed operation: starting an older image against a schema +# migrated by a newer image is not a safe rollback strategy. + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +GEOINTEL_DEPLOY_LOCK_FILE="${GEOINTEL_DEPLOY_LOCK_FILE:-/tmp/geointel-release-deploy.lock}" +if [ "${GEOINTEL_DEPLOY_LOCK_HELD:-false}" != "true" ]; then + command -v flock >/dev/null 2>&1 || { + echo "GeoIntel database restore requires flock to prevent concurrent deployment." >&2 + exit 2 + } + exec 9>"$GEOINTEL_DEPLOY_LOCK_FILE" + if ! flock -n 9; then + echo "Another GeoIntel deployment or rollback is already running." >&2 + exit 3 + fi +fi + +BACKUP_DIR="" +CONFIRMED="false" +RESTORE_IMAGE="${GEOINTEL_ROLLBACK_IMAGE:-}" +GEOINTEL_CONTAINER_NAME="${GEOINTEL_CONTAINER_NAME:-geointel}" + +usage() { + cat <<'EOF' +Usage: bash deploy/unraid/restore-predeploy-database.sh \ + --backup-dir PATH --confirm-production-database-restore [options] + +Stops the normal GeoIntel container, starts an isolated PostGIS recovery +container on the same persistent database path, restores the checksum-verified +custom-format dump, validates Alembic/table counts, and stops recovery again. +The caller must start the rollback image after this command succeeds. + +Options: + --image IMAGE Recovery image containing PostgreSQL/PostGIS tools + --container NAME Normal application container (default: geointel) +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --backup-dir) BACKUP_DIR="$2"; shift 2 ;; + --confirm-production-database-restore) CONFIRMED="true"; shift ;; + --image) RESTORE_IMAGE="$2"; shift 2 ;; + --container) GEOINTEL_CONTAINER_NAME="$2"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +if [ "$CONFIRMED" != "true" ] || [ -z "$BACKUP_DIR" ]; then + echo "Explicit --confirm-production-database-restore and --backup-dir are required." >&2 + exit 2 +fi +for required in docker python3 sha256sum; do + command -v "$required" >/dev/null 2>&1 || { + echo "Missing required command: $required" >&2 + exit 2 + } +done + +if [ -f .env ]; then + set -a + # shellcheck disable=SC1091 + . ./.env + set +a +fi + +GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}" +GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}" +GEOINTEL_POSTGRES_DB="${GEOINTEL_POSTGRES_DB:-geointel}" +GEOINTEL_POSTGRES_USER="${GEOINTEL_POSTGRES_USER:-geointel}" +GEOINTEL_POSTGRES_PASSWORD="${GEOINTEL_POSTGRES_PASSWORD:-}" + +if ! [[ "$GEOINTEL_POSTGRES_DB" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] \ + || ! [[ "$GEOINTEL_POSTGRES_USER" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + echo "Configured PostGIS database and user names must be simple SQL identifiers." >&2 + exit 2 +fi +case "$GEOINTEL_POSTGRES_PASSWORD" in + ''|geointel|postgres|password|changeme|change-me-before-shared-use) + echo "Refusing database restore with an empty or known-default PostGIS password." >&2 + exit 2 + ;; +esac +test -f "$GEOINTEL_POSTGIS_DATA_PATH/PG_VERSION" || { + echo "Persistent PostGIS data path is not initialized: $GEOINTEL_POSTGIS_DATA_PATH" >&2 + exit 3 +} + +GEOINTEL_BACKUPS_PATH="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$GEOINTEL_BACKUPS_PATH")" +BACKUP_DIR="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$BACKUP_DIR")" +python3 - "$GEOINTEL_BACKUPS_PATH" "$BACKUP_DIR" <<'PY' +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +backup = pathlib.Path(sys.argv[2]) +try: + backup.relative_to(root) +except ValueError as exc: + raise SystemExit(f"Backup directory must be below {root}") from exc +if backup == root: + raise SystemExit("Backup directory must identify one immutable backup") +PY + +for required_file in manifest.json database.dump database.list database-metadata.tsv table-counts.tsv CHECKSUMS.sha256; do + test -s "$BACKUP_DIR/$required_file" || { + echo "Missing or empty backup artifact: $required_file" >&2 + exit 3 + } +done +( + cd "$BACKUP_DIR" + sha256sum -c CHECKSUMS.sha256 +) +python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$BACKUP_DIR" + +IFS=$'\t' read -r BACKUP_DB BACKUP_USER BACKUP_IMAGE_ID BACKUP_RELEASE_ID < <( + python3 - "$BACKUP_DIR/manifest.json" <<'PY' +import json +import pathlib +import sys + +payload = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +if payload.get("schema_version") != 1 or payload.get("read_only_source") is not True: + raise SystemExit("Unsupported or unsafe backup manifest") +print( + f"{payload.get('database_name', '')}\t{payload.get('database_user', '')}\t" + f"{payload.get('image_id', '')}\t{payload.get('release_id', '')}" +) +PY +) +if [ "$BACKUP_DB" != "$GEOINTEL_POSTGRES_DB" ] || [ "$BACKUP_USER" != "$GEOINTEL_POSTGRES_USER" ]; then + echo "Backup database identity does not match the configured production database." >&2 + exit 3 +fi +if ! [[ "$BACKUP_IMAGE_ID" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "Backup manifest does not contain one immutable Docker image ID." >&2 + exit 3 +fi +if [ -z "$RESTORE_IMAGE" ]; then + RESTORE_IMAGE="$BACKUP_IMAGE_ID" +fi +docker image inspect "$RESTORE_IMAGE" >/dev/null +RESTORE_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$RESTORE_IMAGE")" +if [ -z "$BACKUP_IMAGE_ID" ] || [ "$BACKUP_IMAGE_ID" != "$RESTORE_IMAGE_ID" ]; then + echo "Backup image identity does not match the retained rollback image." >&2 + exit 3 +fi +case "$BACKUP_RELEASE_ID" in + predeploy-*) ;; + *) echo "Production rollback requires a predeploy backup." >&2; exit 3 ;; +esac + +if docker ps -a --format '{{.Names}}' | grep -Fxq "$GEOINTEL_CONTAINER_NAME"; then + docker rm -f "$GEOINTEL_CONTAINER_NAME" >/dev/null +fi + +RECOVERY_CONTAINER="geointel-db-restore-$(date -u +%Y%m%d%H%M%S)-$$" +RESTORE_PROOF_DB="geointel_restore_proof_$(date -u +%Y%m%d%H%M%S)_$$" +RECOVERY_DB="geointel_pre_restore_$(date -u +%Y%m%d%H%M%S)_$$" +FAILED_RESTORE_DB="geointel_failed_restore_$(date -u +%Y%m%d%H%M%S)_$$" +SWAP_COMPLETE="false" +cleanup_recovery() { + if [ "$SWAP_COMPLETE" != "true" ] \ + && [ "$(docker inspect -f '{{.State.Running}}' "$RECOVERY_CONTAINER" 2>/dev/null || true)" = "true" ]; then + docker exec "$RECOVERY_CONTAINER" dropdb --if-exists --force \ + -U "$GEOINTEL_POSTGRES_USER" "$RESTORE_PROOF_DB" >/dev/null 2>&1 || true + fi + docker rm -f "$RECOVERY_CONTAINER" >/dev/null 2>&1 || true +} +trap cleanup_recovery EXIT + +docker run -d \ + --name "$RECOVERY_CONTAINER" \ + --restart no \ + -e PGDATA=/var/lib/postgresql/data \ + -e PGPASSWORD="$GEOINTEL_POSTGRES_PASSWORD" \ + -v "$GEOINTEL_POSTGIS_DATA_PATH:/var/lib/postgresql/data" \ + -v "$BACKUP_DIR:/restore:ro" \ + --entrypoint /bin/bash \ + "$RESTORE_IMAGE" \ + -c 'set -euo pipefail; chown postgres:postgres "$PGDATA"; exec gosu postgres postgres' \ + >/dev/null + +for attempt in $(seq 1 180); do + if docker exec "$RECOVERY_CONTAINER" pg_isready -h 127.0.0.1 -U "$GEOINTEL_POSTGRES_USER" -d postgres >/dev/null 2>&1; then + break + fi + if [ "$(docker inspect -f '{{.State.Running}}' "$RECOVERY_CONTAINER" 2>/dev/null || true)" != "true" ]; then + echo "Database recovery container exited before PostGIS became ready." >&2 + docker logs "$RECOVERY_CONTAINER" >&2 || true + exit 4 + fi + if [ "$attempt" -eq 180 ]; then + echo "PostGIS recovery did not become ready within six minutes." >&2 + exit 4 + fi + sleep 2 +done + +RESTORED_LIST="$(mktemp)" +trap 'rm -f -- "$RESTORED_LIST"; cleanup_recovery' EXIT +docker exec "$RECOVERY_CONTAINER" pg_restore --list /restore/database.dump > "$RESTORED_LIST" +cmp -s "$RESTORED_LIST" "$BACKUP_DIR/database.list" || { + echo "Recovery image reads a different PostgreSQL archive listing." >&2 + exit 4 +} + +if ! docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \ + -U "$GEOINTEL_POSTGRES_USER" -d postgres -Atqc \ + "SELECT 1 FROM pg_database WHERE datname = '${GEOINTEL_POSTGRES_DB}';" | grep -Fxq 1; then + echo "Configured production database does not exist; refusing replacement." >&2 + exit 4 +fi +for generated_database in "$RESTORE_PROOF_DB" "$RECOVERY_DB" "$FAILED_RESTORE_DB"; do + if docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \ + -U "$GEOINTEL_POSTGRES_USER" -d postgres -Atqc \ + "SELECT 1 FROM pg_database WHERE datname = '${generated_database}';" | grep -Fxq 1; then + echo "Generated recovery database already exists: ${generated_database}" >&2 + exit 4 + fi +done + +# Prove the complete archive in a separate database before touching production. +docker exec "$RECOVERY_CONTAINER" createdb \ + -U "$GEOINTEL_POSTGRES_USER" "$RESTORE_PROOF_DB" +docker exec "$RECOVERY_CONTAINER" pg_restore \ + --exit-on-error \ + --no-owner \ + --no-privileges \ + -U "$GEOINTEL_POSTGRES_USER" \ + -d "$RESTORE_PROOF_DB" \ + /restore/database.dump + +EXPECTED_HEAD="$(awk -F $'\t' '$1 == "alembic_head" { print $2 }' "$BACKUP_DIR/database-metadata.tsv")" +validate_restored_database() { + local database_name="$1" + local restored_head="" + restored_head="$(docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \ + -U "$GEOINTEL_POSTGRES_USER" -d "$database_name" -Atqc \ + 'SELECT version_num FROM alembic_version;')" + if [ -z "$EXPECTED_HEAD" ] || [ "$restored_head" != "$EXPECTED_HEAD" ]; then + echo "Restored Alembic head '$restored_head' differs from backup head '$EXPECTED_HEAD'." >&2 + return 1 + fi + while IFS=$'\t' read -r table expected; do + [[ "$table" =~ ^[a-z_]+$ ]] || { + echo "Unsafe table name in retained counts: $table" >&2 + return 1 + } + actual="$(docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \ + -U "$GEOINTEL_POSTGRES_USER" -d "$database_name" -Atqc \ + "SELECT count(*) FROM public.${table};")" + if [ "$actual" != "$expected" ]; then + echo "Restored count mismatch for $table: expected $expected, got $actual." >&2 + return 1 + fi + done < "$BACKUP_DIR/table-counts.tsv" +} + +validate_restored_database "$RESTORE_PROOF_DB" +echo "Isolated predeploy restore proof passed: ${RESTORE_PROOF_DB}" + +docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \ + -U "$GEOINTEL_POSTGRES_USER" -d postgres -c \ + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IN ('${GEOINTEL_POSTGRES_DB}', '${RESTORE_PROOF_DB}') AND pid <> pg_backend_pid();" \ + >/dev/null +docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \ + -U "$GEOINTEL_POSTGRES_USER" -d postgres -c \ + "ALTER DATABASE ${GEOINTEL_POSTGRES_DB} RENAME TO ${RECOVERY_DB};" +if ! docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \ + -U "$GEOINTEL_POSTGRES_USER" -d postgres -c \ + "ALTER DATABASE ${RESTORE_PROOF_DB} RENAME TO ${GEOINTEL_POSTGRES_DB};"; then + echo "Restored database cutover failed; restoring the untouched production database name." >&2 + docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \ + -U "$GEOINTEL_POSTGRES_USER" -d postgres -c \ + "ALTER DATABASE ${RECOVERY_DB} RENAME TO ${GEOINTEL_POSTGRES_DB};" + exit 4 +fi +SWAP_COMPLETE="true" + +if ! validate_restored_database "$GEOINTEL_POSTGRES_DB"; then + echo "Post-cutover validation failed; restoring the retained pre-restore database." >&2 + docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \ + -U "$GEOINTEL_POSTGRES_USER" -d postgres -c \ + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${GEOINTEL_POSTGRES_DB}' AND pid <> pg_backend_pid();" \ + >/dev/null + docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \ + -U "$GEOINTEL_POSTGRES_USER" -d postgres -c \ + "ALTER DATABASE ${GEOINTEL_POSTGRES_DB} RENAME TO ${FAILED_RESTORE_DB};" + docker exec "$RECOVERY_CONTAINER" psql -X -v ON_ERROR_STOP=1 \ + -U "$GEOINTEL_POSTGRES_USER" -d postgres -c \ + "ALTER DATABASE ${RECOVERY_DB} RENAME TO ${GEOINTEL_POSTGRES_DB};" + SWAP_COMPLETE="false" + echo "Original production database was restored; failed restore retained as ${FAILED_RESTORE_DB}." >&2 + exit 4 +fi + +if [ -z "$RECOVERY_DB" ]; then + echo "Recovery database identity was not retained." >&2 + exit 4 +fi + +while IFS=$'\t' read -r table expected; do + [[ "$table" =~ ^[a-z_]+$ ]] || { + echo "Unsafe table name in retained counts: $table" >&2 + exit 4 + } +done < "$BACKUP_DIR/table-counts.tsv" + +rm -f -- "$RESTORED_LIST" +cleanup_recovery +trap - EXIT +echo "Production database restored and verified from: $BACKUP_DIR" +echo "Pre-restore production database retained for operator recovery as: $RECOVERY_DB" diff --git a/deploy/unraid/rollback-dockerman-container.sh b/deploy/unraid/rollback-dockerman-container.sh index 987fb0c4..729c47d9 100644 --- a/deploy/unraid/rollback-dockerman-container.sh +++ b/deploy/unraid/rollback-dockerman-container.sh @@ -4,14 +4,78 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$ROOT" -GEOINTEL_ROLLBACK_IMAGE="${GEOINTEL_ROLLBACK_IMAGE:-geointel-all-in-one:previous}" +GEOINTEL_DEPLOY_LOCK_FILE="${GEOINTEL_DEPLOY_LOCK_FILE:-/tmp/geointel-release-deploy.lock}" +if [ "${GEOINTEL_DEPLOY_LOCK_HELD:-false}" != "true" ]; then + command -v flock >/dev/null 2>&1 || { + echo "GeoIntel rollback requires flock to prevent concurrent deployment." >&2 + exit 2 + } + exec 9>"$GEOINTEL_DEPLOY_LOCK_FILE" + if ! flock -n 9; then + echo "Another GeoIntel deployment or rollback is already running." >&2 + exit 3 + fi + GEOINTEL_DEPLOY_LOCK_HELD="true" + export GEOINTEL_DEPLOY_LOCK_HELD +fi -if ! docker image inspect "$GEOINTEL_ROLLBACK_IMAGE" >/dev/null 2>&1; then - echo "Rollback image does not exist: ${GEOINTEL_ROLLBACK_IMAGE}" >&2 +GEOINTEL_ROLLBACK_IMAGE="${GEOINTEL_ROLLBACK_IMAGE:-}" +BACKUP_DIR="${GEOINTEL_ROLLBACK_BACKUP_DIR:-}" +CONFIRM_RESTORE="false" + +usage() { + cat <<'EOF' +Usage: bash deploy/unraid/rollback-dockerman-container.sh \ + --backup-dir PATH --confirm-production-database-restore + +Restores the verified pre-deploy PostgreSQL dump first and only then starts the +retained previous image. Image-only rollback against an unknown migrated +schema is deliberately not supported. +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --backup-dir) BACKUP_DIR="$2"; shift 2 ;; + --confirm-production-database-restore) CONFIRM_RESTORE="true"; shift ;; + --help|-h) usage; exit 0 ;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +if [ -z "$BACKUP_DIR" ] || [ "$CONFIRM_RESTORE" != "true" ]; then + echo "Rollback requires a verified pre-deploy backup and explicit database-restore confirmation." >&2 + usage >&2 exit 2 fi -echo "Starting rollback image ${GEOINTEL_ROLLBACK_IMAGE} without changing persistent volumes..." +restore_image_args=() +if [ -n "$GEOINTEL_ROLLBACK_IMAGE" ]; then + restore_image_args=(--image "$GEOINTEL_ROLLBACK_IMAGE") +fi +echo "Restoring the pre-deploy database before starting its checksum-bound image..." +bash deploy/unraid/restore-predeploy-database.sh \ + --backup-dir "$BACKUP_DIR" \ + "${restore_image_args[@]}" \ + --confirm-production-database-restore + +if [ -z "$GEOINTEL_ROLLBACK_IMAGE" ]; then + GEOINTEL_ROLLBACK_IMAGE="$(python3 - "$BACKUP_DIR/manifest.json" <<'PY' +import json +import pathlib +import re +import sys + +image_id = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")).get("image_id", "") +if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): + raise SystemExit("Backup manifest lacks an immutable rollback image ID") +print(image_id) +PY +)" +fi +docker image inspect "$GEOINTEL_ROLLBACK_IMAGE" >/dev/null + +echo "Starting rollback image ${GEOINTEL_ROLLBACK_IMAGE} with the restored persistent database..." GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE" bash deploy/unraid/run-dockerman-container.sh for attempt in $(seq 1 90); do diff --git a/deploy/unraid/run-dockerman-container.sh b/deploy/unraid/run-dockerman-container.sh index a8992597..dbe287ec 100644 --- a/deploy/unraid/run-dockerman-container.sh +++ b/deploy/unraid/run-dockerman-container.sh @@ -33,6 +33,7 @@ GEOINTEL_POSTGRES_USER="${GEOINTEL_POSTGRES_USER:-geointel}" GEOINTEL_POSTGRES_PASSWORD="${GEOINTEL_POSTGRES_PASSWORD:-}" GEOINTEL_CORS_ORIGINS="${GEOINTEL_CORS_ORIGINS:-http://localhost:${GEOINTEL_FRONTEND_PORT},http://127.0.0.1:${GEOINTEL_FRONTEND_PORT},http://192.168.10.150:${GEOINTEL_FRONTEND_PORT}}" GEOINTEL_MAX_UPLOAD_MB="${GEOINTEL_MAX_UPLOAD_MB:-500}" +GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}" GEOINTEL_AOI_WORKER_ENABLED="${GEOINTEL_AOI_WORKER_ENABLED:-true}" GEOINTEL_AOI_WORKER_POLL_SECONDS="${GEOINTEL_AOI_WORKER_POLL_SECONDS:-2}" GEOINTEL_AUTH_ENABLED="${GEOINTEL_AUTH_ENABLED:-false}" @@ -40,6 +41,11 @@ GEOINTEL_AUTH_USERNAME="${GEOINTEL_AUTH_USERNAME:-}" GEOINTEL_AUTH_PASSWORD_HASH="${GEOINTEL_AUTH_PASSWORD_HASH:-}" GEOINTEL_AUTH_SESSION_SECRET="${GEOINTEL_AUTH_SESSION_SECRET:-}" GEOINTEL_AUTH_SESSION_TTL_SECONDS="${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200}" +GEOINTEL_PUBLIC_BASE_URL="${GEOINTEL_PUBLIC_BASE_URL:-http://localhost:${GEOINTEL_FRONTEND_PORT}}" +GEOINTEL_AUTHENTIK_ISSUER="${GEOINTEL_AUTHENTIK_ISSUER:-}" +GEOINTEL_AUTHENTIK_CLIENT_ID="${GEOINTEL_AUTHENTIK_CLIENT_ID:-}" +GEOINTEL_AUTHENTIK_CLIENT_SECRET="${GEOINTEL_AUTHENTIK_CLIENT_SECRET:-}" +GEOINTEL_AUTHENTIK_ALLOWED_EMAIL="${GEOINTEL_AUTHENTIK_ALLOWED_EMAIL:-}" GEOINTEL_GUEST_ACCESS_ENABLED="${GEOINTEL_GUEST_ACCESS_ENABLED:-true}" GEOINTEL_GUEST_DISPLAY_NAME="${GEOINTEL_GUEST_DISPLAY_NAME:-Gast}" GEOINTEL_GUEST_SESSION_TTL_SECONDS="${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200}" @@ -189,6 +195,17 @@ validate_runtime_config() { return 2 fi + case "$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" in + ''|*[!0-9]*) + echo "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be a whole number." >&2 + return 2 + ;; + esac + if [ "$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" -lt 1 ] || [ "$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" -gt 256 ]; then + echo "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be between 1 and 256." >&2 + return 2 + fi + case "$GEOINTEL_AUTH_ENABLED" in true|false) ;; *) @@ -212,6 +229,36 @@ validate_runtime_config() { esac fi + local authentik_count=0 + local authentik_value + for authentik_value in \ + "$GEOINTEL_AUTHENTIK_ISSUER" \ + "$GEOINTEL_AUTHENTIK_CLIENT_ID" \ + "$GEOINTEL_AUTHENTIK_CLIENT_SECRET" \ + "$GEOINTEL_AUTHENTIK_ALLOWED_EMAIL"; do + if [ -n "$authentik_value" ]; then + authentik_count=$((authentik_count + 1)) + fi + done + if [ "$authentik_count" -ne 0 ] && [ "$authentik_count" -ne 4 ]; then + echo "All GEOINTEL_AUTHENTIK_* values must be configured together." >&2 + return 2 + fi + if [ "$authentik_count" -eq 4 ]; then + if [ "$GEOINTEL_AUTH_ENABLED" != "true" ]; then + echo "GEOINTEL_AUTH_ENABLED must be true when Authentik is configured." >&2 + return 2 + fi + case "$GEOINTEL_AUTHENTIK_ISSUER" in + https://*) ;; + *) echo "GEOINTEL_AUTHENTIK_ISSUER must use HTTPS." >&2; return 2 ;; + esac + case "$GEOINTEL_PUBLIC_BASE_URL" in + https://*) ;; + *) echo "GEOINTEL_PUBLIC_BASE_URL must use HTTPS for Authentik." >&2; return 2 ;; + esac + fi + case "$GEOINTEL_GUEST_ACCESS_ENABLED" in true|false) ;; *) @@ -314,6 +361,7 @@ docker run -d \ -e GEOINTEL_STORAGE_ROOT=/app/storage \ -e GEOINTEL_CORS_ORIGINS="$GEOINTEL_CORS_ORIGINS" \ -e GEOINTEL_MAX_UPLOAD_MB="$GEOINTEL_MAX_UPLOAD_MB" \ + -e GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB" \ -e GEOINTEL_AOI_WORKER_ENABLED="$GEOINTEL_AOI_WORKER_ENABLED" \ -e GEOINTEL_AOI_WORKER_POLL_SECONDS="$GEOINTEL_AOI_WORKER_POLL_SECONDS" \ -e GEOINTEL_AUTH_ENABLED="$GEOINTEL_AUTH_ENABLED" \ @@ -321,6 +369,11 @@ docker run -d \ -e GEOINTEL_AUTH_PASSWORD_HASH="$GEOINTEL_AUTH_PASSWORD_HASH" \ -e GEOINTEL_AUTH_SESSION_SECRET="$GEOINTEL_AUTH_SESSION_SECRET" \ -e GEOINTEL_AUTH_SESSION_TTL_SECONDS="$GEOINTEL_AUTH_SESSION_TTL_SECONDS" \ + -e GEOINTEL_PUBLIC_BASE_URL="$GEOINTEL_PUBLIC_BASE_URL" \ + -e GEOINTEL_AUTHENTIK_ISSUER="$GEOINTEL_AUTHENTIK_ISSUER" \ + -e GEOINTEL_AUTHENTIK_CLIENT_ID="$GEOINTEL_AUTHENTIK_CLIENT_ID" \ + -e GEOINTEL_AUTHENTIK_CLIENT_SECRET="$GEOINTEL_AUTHENTIK_CLIENT_SECRET" \ + -e GEOINTEL_AUTHENTIK_ALLOWED_EMAIL="$GEOINTEL_AUTHENTIK_ALLOWED_EMAIL" \ -e GEOINTEL_GUEST_ACCESS_ENABLED="$GEOINTEL_GUEST_ACCESS_ENABLED" \ -e GEOINTEL_GUEST_DISPLAY_NAME="$GEOINTEL_GUEST_DISPLAY_NAME" \ -e GEOINTEL_GUEST_SESSION_TTL_SECONDS="$GEOINTEL_GUEST_SESSION_TTL_SECONDS" \ diff --git a/docker-compose.unraid.yml b/docker-compose.unraid.yml index be2f3002..55331271 100644 --- a/docker-compose.unraid.yml +++ b/docker-compose.unraid.yml @@ -21,11 +21,17 @@ services: GEOINTEL_AOI_WORKER_POLL_SECONDS: ${GEOINTEL_AOI_WORKER_POLL_SECONDS:-2} GEOINTEL_CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202} GEOINTEL_MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500} + GEOINTEL_MAX_IN_MEMORY_VECTOR_MB: ${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64} GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-false} GEOINTEL_AUTH_USERNAME: ${GEOINTEL_AUTH_USERNAME:-} GEOINTEL_AUTH_PASSWORD_HASH: ${GEOINTEL_AUTH_PASSWORD_HASH:-} GEOINTEL_AUTH_SESSION_SECRET: ${GEOINTEL_AUTH_SESSION_SECRET:-} GEOINTEL_AUTH_SESSION_TTL_SECONDS: ${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200} + GEOINTEL_PUBLIC_BASE_URL: ${GEOINTEL_PUBLIC_BASE_URL:-http://localhost:1202} + GEOINTEL_AUTHENTIK_ISSUER: ${GEOINTEL_AUTHENTIK_ISSUER:-} + GEOINTEL_AUTHENTIK_CLIENT_ID: ${GEOINTEL_AUTHENTIK_CLIENT_ID:-} + GEOINTEL_AUTHENTIK_CLIENT_SECRET: ${GEOINTEL_AUTHENTIK_CLIENT_SECRET:-} + GEOINTEL_AUTHENTIK_ALLOWED_EMAIL: ${GEOINTEL_AUTHENTIK_ALLOWED_EMAIL:-} GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-true} GEOINTEL_GUEST_DISPLAY_NAME: ${GEOINTEL_GUEST_DISPLAY_NAME:-Gast} GEOINTEL_GUEST_SESSION_TTL_SECONDS: ${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200} diff --git a/docker-compose.yml b/docker-compose.yml index 298b60f8..941f277e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,11 +23,17 @@ services: STORAGE_ROOT: /app/storage CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202} MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500} + GEOINTEL_MAX_IN_MEMORY_VECTOR_MB: ${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64} GEOINTEL_AUTH_ENABLED: ${GEOINTEL_AUTH_ENABLED:-false} GEOINTEL_AUTH_USERNAME: ${GEOINTEL_AUTH_USERNAME:-} GEOINTEL_AUTH_PASSWORD_HASH: ${GEOINTEL_AUTH_PASSWORD_HASH:-} GEOINTEL_AUTH_SESSION_SECRET: ${GEOINTEL_AUTH_SESSION_SECRET:-} GEOINTEL_AUTH_SESSION_TTL_SECONDS: ${GEOINTEL_AUTH_SESSION_TTL_SECONDS:-43200} + GEOINTEL_PUBLIC_BASE_URL: ${GEOINTEL_PUBLIC_BASE_URL:-http://localhost:1202} + GEOINTEL_AUTHENTIK_ISSUER: ${GEOINTEL_AUTHENTIK_ISSUER:-} + GEOINTEL_AUTHENTIK_CLIENT_ID: ${GEOINTEL_AUTHENTIK_CLIENT_ID:-} + GEOINTEL_AUTHENTIK_CLIENT_SECRET: ${GEOINTEL_AUTHENTIK_CLIENT_SECRET:-} + GEOINTEL_AUTHENTIK_ALLOWED_EMAIL: ${GEOINTEL_AUTHENTIK_ALLOWED_EMAIL:-} GEOINTEL_GUEST_ACCESS_ENABLED: ${GEOINTEL_GUEST_ACCESS_ENABLED:-true} GEOINTEL_GUEST_DISPLAY_NAME: ${GEOINTEL_GUEST_DISPLAY_NAME:-Gast} GEOINTEL_GUEST_SESSION_TTL_SECONDS: ${GEOINTEL_GUEST_SESSION_TTL_SECONDS:-7200} diff --git a/docs/CI_SUPPLY_CHAIN.md b/docs/CI_SUPPLY_CHAIN.md index 45040b69..32bdffbd 100644 --- a/docs/CI_SUPPLY_CHAIN.md +++ b/docs/CI_SUPPLY_CHAIN.md @@ -5,8 +5,10 @@ GeoIntel uses the same release gates in Gitea Actions and GitHub Actions: - `.gitea/workflows/release-gates.yml` - `.github/workflows/release-gates.yml` -Gitea is the operational source-control platform. The GitHub workflow is kept -equivalent so a mirror or external review does not receive a weaker gate. +Gitea is the operational source-control platform. Its release workflow builds +the production AI variant and is the only workflow that can automatically +deploy. The GitHub mirror builds and scans the same AI variant for external +review but is not a production deployment authority. ## Runner requirements @@ -16,11 +18,11 @@ The `ubuntu-latest` runner must provide: - Python 3.11 and Node 20 through the official setup actions; - Bash and Docker with Compose v2; - permission to build images and mount `/var/run/docker.sock`; -- sufficient disk for the all-in-one GIS image and scanner databases. +- sufficient disk for the all-in-one AI image and scanner databases. -The container job builds the GIS release variant only. PyTorch and -Ultralytics remain in the optional `ai` extra and in the explicit AI image -variant; CI does not silently make them base dependencies. +Both container jobs build the explicit AI image, including the +PyTorch/Ultralytics layers used on Tower. AI packages remain outside the +standard backend lock and are pinned by Docker build arguments. ## Quality gate @@ -33,11 +35,25 @@ cd frontend && npm ci ``` It then validates the lock policy and runs the complete readiness script. The -readiness script covers backend compile/tests, contract audits, Alembic -single-head, frontend typecheck/build and release-script syntax. CI also +readiness script covers Ruff, repository-layout validation, backend +compile/tests, contract audits, Alembic single-head, frontend typecheck/build +and release-script syntax. CI also renders offline migration SQL and resolved Compose configuration as retained evidence. +Pull requests run managed validation against the real `backend/` and +`frontend/` projects plus the complete release gates. On a `main` push, the +Unraid deploy job has explicit `needs` dependencies on quality, dependency and +AI-container jobs. There is no separate deployment workflow: manual validation +uses `workflow_dispatch` on this same release-gates workflow and cannot skip +quality, dependency or AI-container jobs. + +The deploy host requires a full controller commit SHA, builds the AI variant, +records its Docker image/config digest, generates SBOM and Trivy evidence for +that exact local image, and starts the immutable image ID. Deployment fails if +the running container ID, revision label or AI label differs from the retained +attestation. + ## Reproducible Python lock `backend/requirements-runtime.lock` and `backend/requirements-ci.lock` are @@ -73,12 +89,12 @@ The dependency job: - publishes both unfiltered and policy-filtered Python JSON reports plus the npm JSON report, including on failure. -The only current Python/container exceptions are the Starlette 2026 advisories recorded -in `security/pip-audit-exceptions.json`. FastAPI 0.139.2 still constrains -Starlette below 0.53 while patched releases begin at 1.x. GeoIntel applies -request-target, form-content, route-class and Linux-runtime compensating -controls. The exception file has a mandatory review date; readiness and CI -fail automatically after it expires. New advisories are never auto-ignored. +There are currently no Python or container vulnerability exceptions. +`security/pip-audit-exceptions.json` remains as a strict, machine-readable +registry: every future exception must identify one advisory, package, specific +reason and expiry date. Readiness and CI fail on malformed or expired entries; +new advisories are never auto-ignored. GeoIntel requires Starlette 1.3.1 or +newer and therefore no longer suppresses the five 2026 Starlette advisories. The all-in-one image replaces the Go-based base-image `gosu` helper with a small `setpriv` exec wrapper and upgrades packaged setuptools/wheel metadata; the final runtime filesystem no longer exposes the vulnerable Go executable. @@ -89,8 +105,8 @@ verified to contain the audited shell wrapper. This is not a vulnerability exception: the raw evidence remains published and the runtime wrapper is exercised during live release validation. -The container job builds a non-AI all-in-one image and uses digest-pinned -scanner images: +The operational container job builds the production AI all-in-one image and +uses digest-pinned scanner images: - Syft 1.44.0 generates an SPDX JSON SBOM; - Trivy 0.70.0 generates a complete JSON vulnerability report; @@ -103,12 +119,12 @@ Run these controls on a Docker-enabled workstation: ```bash docker build \ -f deploy/unraid/Dockerfile.all-in-one \ - --build-arg GEOINTEL_INSTALL_AI=false \ + --build-arg GEOINTEL_INSTALL_AI=true \ --build-arg GEOINTEL_BUILD_SHA=local \ --build-arg GEOINTEL_BUILD_TIME=local \ - -t geointel-ci:local . -bash scripts/generate_container_sbom.sh geointel-ci:local -bash scripts/scan_container_image.sh geointel-ci:local + -t geointel-ci:local-ai . +bash scripts/generate_container_sbom.sh geointel-ci:local-ai +bash scripts/scan_container_image.sh geointel-ci:local-ai ``` Outputs are written below ignored `artifacts/`; scanner cache is written below diff --git a/docs/DATA_OPERATIONS_RUNBOOK.md b/docs/DATA_OPERATIONS_RUNBOOK.md index b56ecdec..533c8ea2 100644 --- a/docs/DATA_OPERATIONS_RUNBOOK.md +++ b/docs/DATA_OPERATIONS_RUNBOOK.md @@ -89,9 +89,9 @@ The normal project lifecycle archive path does not need destructive confirmation because it changes only `status=archived` and preserves all data. -## Destructive apply gate +## Recoverable quarantine gate -First create a fresh backup with a SHA-256 storage inventory on the host: +First create a fresh backup with a byte-complete SHA-256 storage snapshot on the host: ```bash bash scripts/backup_release_state.sh \ @@ -112,7 +112,8 @@ bash scripts/verify_release_backup.sh \ ``` The Unraid runtime mounts `GEOINTEL_BACKUPS_PATH` read-only at `/app/backups`. -Only after reviewing the dry run may an operator execute: +Only after reviewing the dry run may an operator move the exact candidates to +a protected quarantine: ```bash docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \ @@ -120,14 +121,34 @@ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \ --max-delete \ --backup-dir /app/backups/ \ --backup-max-age-hours 24 \ - --confirm DELETE_STORAGE_ARTIFACTS \ + --confirm QUARANTINE_STORAGE_ARTIFACTS \ --apply ``` -The command re-runs the audit immediately before deletion. It refuses the +The command re-runs the audit immediately before quarantine. It refuses the operation when the exact token is absent, the candidate count exceeds the -operator limit, the backup is stale/incomplete, checksums differ, the storage -inventory is not SHA-256, or the path is outside the cleanup allowlist. +operator limit, the backup is stale/incomplete, snapshot checksums differ, or +the path is outside the cleanup allowlist. +Every moved byte is hashed and retained below +`operator-evidence/cleanup-quarantine//files/`; an atomic +manifest records its original path, quarantine path, size, checksum and +backup identity. Cleanup uses a backup-to-quarantine hard-link state machine: +the manifest records `planned`, `linked` and `quarantined` transitions so each +crash window can be reconciled without losing the retained inode. + +Restore a reviewed quarantine without overwriting any path: + +```bash +docker exec geointel python /app/scripts/restore_storage_quarantine.py \ + --storage-root /app/storage \ + --manifest /app/storage/operator-evidence/cleanup-quarantine//manifest.json \ + --confirm RESTORE_QUARANTINED_ARTIFACTS +``` + +Restore validates every retained checksum and never replaces an existing +destination. It uses an exclusive hard link and reconciles both-file and +one-file interruption states before updating the manifest, so an interrupted +restore can be resumed and verified. The older demo-export cleanup has the same gate and uses confirmation token `DELETE_DEMO_EXPORTS`. diff --git a/docs/RELEASE_RUNBOOK.md b/docs/RELEASE_RUNBOOK.md index ea596ac4..0b9253a7 100644 --- a/docs/RELEASE_RUNBOOK.md +++ b/docs/RELEASE_RUNBOOK.md @@ -15,7 +15,7 @@ The repository version is stored in `VERSION`. The current release is - clean `main` worktree at the commit being released; - secure non-default PostGIS password in the Tower `.env`; - existing local AI model only when the AI image is enabled; -- recent checksum-verified backup with SHA-256 storage inventory; +- recent checksum-verified database dump and byte-complete storage/model snapshot; - Docker, `ssh-keygen`, Python 3.11, Node 20 and Bash available; - one Alembic head and no unsupported metric represented as successful. @@ -39,6 +39,24 @@ docker compose config ## Immutable deployment +`deploy-release.sh` first builds/reuses the candidate while the current release +stays available. Immediately before replacement it quiesces backend writes and +creates and checksum-verifies a database dump plus byte-complete SHA-256 +storage/model snapshots under `/mnt/user/appdata/geointel/backups`. The first +snapshot copies every byte (using CoW reflinks when supported); later snapshots +hard-link only checksum-identical bytes from a verified older backup, never +from live storage. A conservative full-copy/free-space preflight runs before +backend quiescence. Only then may the +candidate start or run Alembic. If backup fails, the unchanged release is +restarted; deployment stops if existing PostGIS state cannot be backed up +consistently. + +The backup manifest separates provenance intentionally: +`backup_tool_revision` is the new candidate source running the backup tool, +while `running_image_revision` is the OCI label of the old release whose data +is being captured. The retained Docker `image_id`, not either descriptive +revision field, is authoritative for rollback. + On the Codex workstation: ```powershell @@ -58,8 +76,9 @@ bash scripts/live_migration_smoke.sh ## Backup and recovery proof -Create an immutable backup. The SHA-256 inventory can take several minutes on -large storage: +Create an immutable byte-complete backup. Initial storage/model copy and +verification can be I/O-heavy; subsequent backups deduplicate unchanged bytes +against the newest verified prior snapshot: ```bash bash scripts/backup_release_state.sh \ @@ -124,17 +143,37 @@ gate fails on reachable fixed HIGH/CRITICAL findings. ## Rollback proof -The rollback command reuses persistent paths and never downgrades Alembic: +Use the exact backup printed by the deployment. Rollback first stops the +candidate, restores and verifies the pre-deploy PostgreSQL dump in a temporary +proof database, then swaps database names while retaining the pre-restore +database as a recovery point. Only then does it start the exact image ID bound +into that backup; it never relies on a mutable global `previous` tag. It never runs an +Alembic downgrade or starts an old image against an unknown newer schema: ```bash -bash deploy/unraid/rollback-dockerman-container.sh +bash deploy/unraid/rollback-dockerman-container.sh \ + --backup-dir /mnt/user/appdata/geointel/backups/ \ + --confirm-production-database-restore curl -fsS http://127.0.0.1:1202/health/ready bash deploy/unraid/deploy-release.sh curl -fsS http://127.0.0.1:1202/health/ready ``` -For a future backward-incompatible migration, restore the verified pre-release -backup instead of running an older image against a newer schema. +Database rollback restores persisted rows and schema. Files newly written by a +failed candidate remain in storage as unreferenced evidence; the protected, +recoverable quarantine flow in `DATA_OPERATIONS_RUNBOOK.md` handles those +files without deleting source data. + +After the rollback has remained healthy and its retained evidence has been +reviewed, list the recovery database printed by the script. Remove it only by +an explicit, separately approved `dropdb` maintenance command; deployment and +rollback never auto-delete recovery databases or backup directories. Retain at +least the current successful predeploy backup and its predecessor. Before +removing an older backup, run `verify_release_backup.sh` on the backups that +remain and confirm no newer snapshot hard-links depend on operator policy for +retention (hard-linked bytes remain allocated while any retained backup names +them). Remove its `rollback-predeploy-*` image tag only in the same explicitly +reviewed retention operation. ## Tag and signed package diff --git a/docs/ROLLBACK_AND_RECOVERY.md b/docs/ROLLBACK_AND_RECOVERY.md index 4b158018..8336256d 100644 --- a/docs/ROLLBACK_AND_RECOVERY.md +++ b/docs/ROLLBACK_AND_RECOVERY.md @@ -28,10 +28,11 @@ The restore smoke may only create databases whose name starts with use `pg_restore --clean` and drops the temporary database unless an operator explicitly asks to retain it. -Storage and model files are inventoried rather than copied into the database -dump. Release backups must therefore be paired with the persistent storage -volume backup policy. Use `--inventory-mode sha256` for final release -evidence. +Storage and model files are retained as byte-complete, SHA-256 verified +snapshots alongside the database dump. Snapshot creation rejects symlinks and +special files. The first snapshot uses CoW reflinks when supported and falls +back to full copies; later snapshots hard-link checksum-identical files only +from a verified older backup, never from live storage. An old persistent volume can also retain glibc collation metadata for the empty `postgres` and `template1` system databases. If `createdb` fails for that diff --git a/frontend/.dockerignore b/frontend/.dockerignore index dfebf087..511c4431 100644 --- a/frontend/.dockerignore +++ b/frontend/.dockerignore @@ -5,3 +5,5 @@ __pycache__ .pytest_cache .vite .env +.env.* +!.env.example diff --git a/frontend/nginx.conf b/frontend/nginx.conf index c2238d7c..5ebaac3f 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -1,3 +1,16 @@ +geo $geointel_trusted_forwarder { + default 0; + 127.0.0.0/8 1; + ::1/128 1; + 172.16.0.0/12 1; +} + +map "$geointel_trusted_forwarder:$http_x_forwarded_proto" $geointel_forwarded_proto { + default $scheme; + "1:https" https; + "1:http" http; +} + server { listen 80; server_name _; @@ -5,16 +18,32 @@ server { proxy_read_timeout 600s; proxy_send_timeout 600s; + add_header Content-Security-Policy "frame-ancestors 'none'" always; + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; + root /usr/share/nginx/html; index index.html; location = /index.html { add_header Cache-Control "no-cache"; + add_header Content-Security-Policy "frame-ancestors 'none'" always; + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; try_files /index.html =404; } location /assets/ { add_header Cache-Control "no-cache"; + add_header Content-Security-Policy "frame-ancestors 'none'" always; + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; try_files $uri =404; } @@ -24,7 +53,7 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto; } location = /health { @@ -33,7 +62,7 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto; } location = /health/live { diff --git a/scripts/README.md b/scripts/README.md index de3b072a..0d5a194a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2142,11 +2142,17 @@ bash scripts/backup_release_state.sh \ ``` The backup is written atomically and contains a PostgreSQL custom-format dump, -archive listing, Alembic/PostGIS metadata, critical table counts, optional -storage/model inventories and SHA-256 checksums. An empty or known-default -database password leaves the release gate failed. For an emergency backup -before rotating that password, add `--allow-insecure-password`; the manifest -still records the insecure state. +archive listing, Alembic/PostGIS metadata, critical table counts and +byte-complete SHA-256-verified storage/model snapshots. The first snapshot is a +full copy; a later deployment may hard-link only checksum-identical files from +another completed, fully verified backup with `--link-dest-backup`. It never +hard-links a live source file and never deletes an older backup. An empty or +known-default database password leaves the release gate failed. For an +emergency backup before rotating that password, add +`--allow-insecure-password`; the manifest still records the insecure state. +`backup_tool_revision` identifies the candidate source that executed the +backup; `running_image_revision` identifies the currently running old image. +Rollback is always bound to the retained immutable Docker `image_id`. Verify without changing any database: @@ -2190,7 +2196,7 @@ docker exec geointel python /app/scripts/audit_data_operations.py \ --output /app/storage/release-evidence/rc-current/data-operations.json ``` -Preview old unreferenced derived/cache/export candidates without deletion: +Preview old unreferenced derived/cache/export candidates without mutation: ```bash docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \ @@ -2199,10 +2205,14 @@ docker exec geointel python /app/scripts/cleanup_storage_artifacts.py \ ``` Apply requires a reviewed candidate count, the exact -`DELETE_STORAGE_ARTIFACTS` token and a backup no older than 24 hours with a -checksum-verified database dump and SHA-256 storage inventory. The host backup +`QUARANTINE_STORAGE_ARTIFACTS` token and a backup no older than 24 hours with a +checksum-verified database dump and byte-complete storage snapshot. The host backup root is mounted read-only at `/app/backups`. See -`docs/DATA_OPERATIONS_RUNBOOK.md`. No cleanup is scheduled by GeoIntel. +`docs/DATA_OPERATIONS_RUNBOOK.md`. Candidates enter protected +`operator-evidence/cleanup-quarantine` storage through an interruption-safe +hard-link/unlink state machine. `restore_storage_quarantine.py` reverses that +move with the exact `RESTORE_QUARANTINED_ARTIFACTS` token and refuses to +overwrite an existing original path. No cleanup is scheduled by GeoIntel. ## RC-8 Belgium/North Sea release journeys diff --git a/scripts/audit_data_operations.py b/scripts/audit_data_operations.py index ee0f3369..62dc2bae 100644 --- a/scripts/audit_data_operations.py +++ b/scripts/audit_data_operations.py @@ -14,17 +14,26 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Iterable +from sqlalchemy import func + ROOT = Path(__file__).resolve().parents[1] BACKEND_ROOT = ROOT / "backend" if (ROOT / "backend" / "app").is_dir() else ROOT if str(BACKEND_ROOT) not in sys.path: sys.path.insert(0, str(BACKEND_ROOT)) -from app.core.config import get_settings -from app.db.session import SessionLocal -from sqlalchemy import func - -from app.models import AnalysisRun, Dataset, DatasetVersion, Detection, Export, Job, Project, Segmentation +from app.core.config import get_settings # noqa: E402 - imported after backend path bootstrap +from app.db.session import SessionLocal # noqa: E402 - imported after backend path bootstrap +from app.models import ( # noqa: E402 - imported after backend path bootstrap + AnalysisRun, + Dataset, + DatasetVersion, + Detection, + Export, + Job, + Project, + Segmentation, +) NATIONAL_PROJECT_NAME = "Belgium and North Sea Workbench" diff --git a/scripts/backup_release_state.sh b/scripts/backup_release_state.sh index 6b359f5f..32d26aa3 100644 --- a/scripts/backup_release_state.sh +++ b/scripts/backup_release_state.sh @@ -3,12 +3,14 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" CONTAINER="geointel" -OUTPUT_ROOT="backups" +OUTPUT_ROOT="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}" RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)" STORAGE_PATH="" MODELS_PATH="" INVENTORY_MODE="metadata" ALLOW_INSECURE_PASSWORD="false" +LINK_DEST_BACKUP="" +ROLLBACK_IMAGE_TAG="" usage() { cat <<'EOF' @@ -19,11 +21,16 @@ GeoIntel container. It never deletes or restores application data. Options: --container NAME Docker container (default: geointel) - --output-root PATH Host backup root (default: backups) + --output-root PATH Host backup root (default: + /mnt/user/appdata/geointel/backups) --release-id ID Safe backup directory name - --storage-path PATH Optional host storage path to inventory - --models-path PATH Optional host model path to inventory - --inventory-mode metadata|sha256 Hash all inventoried files only with sha256 + --storage-path PATH Host storage path to snapshot byte-for-byte + --models-path PATH Host model path to snapshot byte-for-byte + --inventory-mode metadata|sha256 Retained manifest compatibility setting + --link-dest-backup PATH Verified older backup used only to hard-link + checksum-identical backup-to-backup files + --rollback-image-tag TAG Immutable backup-specific tag bound to the + running image ID --allow-insecure-password Complete emergency backup despite an empty/default production DB password EOF @@ -37,6 +44,8 @@ while [ "$#" -gt 0 ]; do --storage-path) STORAGE_PATH="$2"; shift 2 ;; --models-path) MODELS_PATH="$2"; shift 2 ;; --inventory-mode) INVENTORY_MODE="$2"; shift 2 ;; + --link-dest-backup) LINK_DEST_BACKUP="$2"; shift 2 ;; + --rollback-image-tag) ROLLBACK_IMAGE_TAG="$2"; shift 2 ;; --allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;; --help|-h) usage; exit 0 ;; *) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;; @@ -51,12 +60,93 @@ if [ "$INVENTORY_MODE" != "metadata" ] && [ "$INVENTORY_MODE" != "sha256" ]; the echo "--inventory-mode must be metadata or sha256" >&2 exit 2 fi -for required in docker python3 sha256sum git; do +for required in docker python3 sha256sum; do if ! command -v "$required" >/dev/null 2>&1; then echo "Missing required command: $required" >&2 exit 2 fi done + +resolve_source_revision() { + local controller_sha="" explicit_sha="${GEOINTEL_BUILD_SHA:-}" + local gitea_sha="${GITEA_COMMIT_SHA:-}" github_sha="${GITHUB_SHA:-}" + local git_head="" git_dirty="false" source="" + + if [ -n "$gitea_sha" ]; then + if ! [[ "$gitea_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then + echo "GITEA_COMMIT_SHA must contain one full 40-character Git commit SHA." >&2 + return 2 + fi + controller_sha="${gitea_sha,,}" + source="GITEA_COMMIT_SHA" + fi + if [ -n "$github_sha" ]; then + if ! [[ "$github_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then + echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2 + return 2 + fi + github_sha="${github_sha,,}" + if [ -n "$controller_sha" ] && [ "$controller_sha" != "$github_sha" ]; then + echo "Controller commit variables disagree." >&2 + return 2 + fi + controller_sha="$github_sha" + source="${source:-GITHUB_SHA}" + fi + if [ -n "$explicit_sha" ]; then + if ! [[ "$explicit_sha" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]]; then + echo "GEOINTEL_BUILD_SHA contains an unsafe release revision." >&2 + return 2 + fi + explicit_sha="${explicit_sha,,}" + fi + if [ -n "$controller_sha" ]; then + if [ -n "$explicit_sha" ] && [ "$explicit_sha" != "$controller_sha" ]; then + echo "GEOINTEL_BUILD_SHA differs from the controller revision." >&2 + return 2 + fi + explicit_sha="$controller_sha" + elif [ -n "${GITEA_REPOSITORY:-}${GITHUB_REPOSITORY:-}" ]; then + if ! [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "Automated backup requires a full controller or GEOINTEL_BUILD_SHA revision." >&2 + return 2 + fi + source="GEOINTEL_BUILD_SHA" + fi + + if command -v git >/dev/null 2>&1 && git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1; then + git_head="$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)" + git_head="${git_head,,}" + if ! [[ "$git_head" =~ ^[0-9a-f]{40}$ ]]; then + echo "Could not resolve a full Git revision from the source checkout." >&2 + return 2 + fi + if [ -n "$explicit_sha" ] && [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]] && [ "$git_head" != "$explicit_sha" ]; then + echo "Source checkout does not match the supplied release revision." >&2 + return 2 + fi + if [ -n "$(git -C "$ROOT" status --porcelain=v1 2>/dev/null)" ]; then + git_dirty="true" + fi + if [ -z "$explicit_sha" ]; then + explicit_sha="$git_head" + source="git" + fi + fi + + if [ -z "$explicit_sha" ]; then + echo "Cannot bind backup to a source revision; provide GEOINTEL_BUILD_SHA or a controller SHA." >&2 + return 2 + fi + SOURCE_REVISION="$explicit_sha" + SOURCE_REVISION_SOURCE="${source:-GEOINTEL_BUILD_SHA}" + SOURCE_GIT_DIRTY="$git_dirty" +} + +SOURCE_REVISION="" +SOURCE_REVISION_SOURCE="" +SOURCE_GIT_DIRTY="false" +resolve_source_revision if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then echo "Container '$CONTAINER' is not running." >&2 exit 3 @@ -72,6 +162,27 @@ if [ -e "$PARTIAL" ] || [ -e "$FINAL" ]; then fi mkdir -p "$PARTIAL" +if [ -n "$LINK_DEST_BACKUP" ]; then + LINK_DEST_BACKUP="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve(strict=True))' "$LINK_DEST_BACKUP")" + python3 - "$OUTPUT_ROOT" "$LINK_DEST_BACKUP" <<'PY' +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +candidate = pathlib.Path(sys.argv[2]) +try: + candidate.relative_to(root) +except ValueError as exc: + raise SystemExit(f"Link-dest backup must remain below {root}") from exc +if candidate == root or candidate.name.startswith("."): + raise SystemExit("Link-dest backup must identify one completed immutable backup") +PY + ( + cd "$LINK_DEST_BACKUP" + sha256sum -c CHECKSUMS.sha256 >/dev/null + ) +fi + cleanup_partial() { if [ -d "$PARTIAL" ]; then rm -rf -- "$PARTIAL" @@ -113,10 +224,21 @@ test -s "$PARTIAL/database.list" IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")" IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")" -GIT_COMMIT="$(git -C "$ROOT" rev-parse HEAD)" -GIT_DIRTY="false" -if [ -n "$(git -C "$ROOT" status --porcelain=v1)" ]; then - GIT_DIRTY="true" +RUNNING_IMAGE_REVISION="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CONTAINER")" +RUNNING_IMAGE_REVISION_IS_FULL_SHA="false" +if [[ "$RUNNING_IMAGE_REVISION" =~ ^[0-9A-Fa-f]{40}$ ]]; then + RUNNING_IMAGE_REVISION="${RUNNING_IMAGE_REVISION,,}" + RUNNING_IMAGE_REVISION_IS_FULL_SHA="true" +elif ! [[ "$RUNNING_IMAGE_REVISION" =~ ^[A-Za-z0-9._-]{1,128}$ ]]; then + echo "Running image has an unsafe or missing OCI revision label." >&2 + exit 3 +fi +if [ -n "$ROLLBACK_IMAGE_TAG" ]; then + TAGGED_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$ROLLBACK_IMAGE_TAG" 2>/dev/null || true)" + if [ "$TAGGED_IMAGE_ID" != "$IMAGE_ID" ]; then + echo "Backup-specific rollback tag does not resolve to the running image ID." >&2 + exit 3 + fi fi docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \ @@ -132,48 +254,32 @@ for table in projects areas datasets dataset_versions vector_features jobs analy printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv" done -inventory_path() { +snapshot_path() { local source_path="$1" - local output_path="$2" + local label="$2" + local manifest_path="$PARTIAL/${label}-manifest.tsv" + local snapshot_path="$PARTIAL/${label}-snapshot" + local link_args=() if [ -z "$source_path" ]; then - printf 'not_requested\n' > "$output_path" + printf 'not_requested\n' > "$manifest_path" return fi - python3 - "$source_path" "$output_path" "$INVENTORY_MODE" <<'PY' -import hashlib -import os -import pathlib -import sys - -root = pathlib.Path(sys.argv[1]).expanduser().resolve() -output = pathlib.Path(sys.argv[2]) -mode = sys.argv[3] -if not root.is_dir(): - raise SystemExit(f"Inventory root is not a directory: {root}") - -def digest(path: pathlib.Path) -> str: - value = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - value.update(chunk) - return value.hexdigest() - -with output.open("w", encoding="utf-8", newline="\n") as handle: - handle.write("relative_path\tsize_bytes\tmtime_ns\tsha256\n") - for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()): - if path.is_symlink() or not path.is_file(): - continue - stat = path.stat() - checksum = digest(path) if mode == "sha256" else "" - relative = path.relative_to(root).as_posix() - if "\t" in relative or "\n" in relative: - raise SystemExit(f"Unsupported inventory path: {relative!r}") - handle.write(f"{relative}\t{stat.st_size}\t{stat.st_mtime_ns}\t{checksum}\n") -PY + if [ -n "$LINK_DEST_BACKUP" ]; then + link_args=( + --link-dest-snapshot "$LINK_DEST_BACKUP/${label}-snapshot" + --link-dest-manifest "$LINK_DEST_BACKUP/${label}-manifest.tsv" + ) + fi + python3 "$ROOT/scripts/release_backup_snapshot.py" create \ + --source "$source_path" \ + --snapshot "$snapshot_path" \ + --manifest "$manifest_path" \ + --label "$label" \ + "${link_args[@]}" } -inventory_path "$STORAGE_PATH" "$PARTIAL/storage-manifest.tsv" -inventory_path "$MODELS_PATH" "$PARTIAL/models-manifest.tsv" +snapshot_path "$STORAGE_PATH" storage +snapshot_path "$MODELS_PATH" models python3 - "$PARTIAL/manifest.json" < str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_manifest(path: Path, payload: dict[str, object]) -> None: + temporary = path.with_suffix(".json.partial") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path) def parse_args() -> argparse.Namespace: @@ -26,6 +44,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--confirm") parser.add_argument("--backup-dir", type=Path) parser.add_argument("--backup-max-age-hours", type=float, default=24.0) + parser.add_argument( + "--quarantine-root", + type=Path, + help="Protected destination below the storage root (default: operator-evidence/cleanup-quarantine)", + ) return parser.parse_args() @@ -44,7 +67,8 @@ def main() -> int: blocked_reason = None backup = None - deleted: list[str] = [] + quarantined: list[dict[str, object]] = [] + quarantine_manifest: Path | None = None if args.apply: require_confirmation(args.confirm, CONFIRMATION) if args.backup_dir is None: @@ -59,9 +83,80 @@ def main() -> int: "review the dry run and raise the explicit limit" ) else: + quarantine_root = ( + args.quarantine_root + or storage_root / "operator-evidence" / "cleanup-quarantine" + ).resolve() + protected_quarantine_root = ( + storage_root / "operator-evidence" / "cleanup-quarantine" + ).resolve() + try: + quarantine_root.relative_to(protected_quarantine_root) + except ValueError as exc: + raise RuntimeError( + "--quarantine-root must remain below " + "operator-evidence/cleanup-quarantine in --storage-root" + ) from exc + operation_id = f"cleanup-{datetime.now(timezone.utc):%Y%m%dT%H%M%SZ}-{uuid4().hex[:12]}" + operation_root = quarantine_root / operation_id + operation_root.mkdir(parents=True, exist_ok=False) + quarantine_manifest = operation_root / "manifest.json" + entries: list[dict[str, object]] = [] for candidate in candidates: + if candidate.path.is_symlink(): + raise RuntimeError(f"Cleanup candidate became a symlink: {candidate.relative_path}") + try: + candidate.path.resolve().relative_to(storage_root) + except ValueError as exc: + raise RuntimeError( + f"Cleanup candidate escaped storage: {candidate.relative_path}" + ) from exc + destination = operation_root / "files" / candidate.relative_path + current_size = candidate.path.stat().st_size + if current_size != candidate.size_bytes: + raise RuntimeError(f"Cleanup candidate changed size: {candidate.relative_path}") + entries.append( + { + "relative_path": candidate.relative_path, + "size_bytes": current_size, + "sha256": sha256(candidate.path), + "status": "planned", + "quarantine_relative_path": destination.relative_to(storage_root).as_posix(), + } + ) + manifest: dict[str, object] = { + "schema_version": 1, + "operation_id": operation_id, + "created_at": datetime.now(timezone.utc).isoformat(), + "state": "in_progress", + "storage_root": str(storage_root), + "backup_release_id": backup.release_id, + "entries": entries, + } + write_manifest(quarantine_manifest, manifest) + for candidate, entry in zip(candidates, entries, strict=True): + destination = storage_root / str(entry["quarantine_relative_path"]) + destination.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(candidate.path, destination, follow_symlinks=False) + except FileExistsError as exc: + raise RuntimeError(f"Quarantine destination already exists: {destination}") from exc + if ( + not destination.is_file() + or destination.stat().st_size != entry["size_bytes"] + or sha256(destination) != entry["sha256"] + ): + destination.unlink(missing_ok=True) + raise RuntimeError(f"Quarantine link verification failed: {candidate.relative_path}") + entry["status"] = "linked" + write_manifest(quarantine_manifest, manifest) candidate.path.unlink() - deleted.append(candidate.relative_path) + entry["status"] = "quarantined" + quarantined.append(dict(entry)) + write_manifest(quarantine_manifest, manifest) + manifest["state"] = "complete" + manifest["completed_at"] = datetime.now(timezone.utc).isoformat() + write_manifest(quarantine_manifest, manifest) payload = { "schema_version": 1, @@ -72,8 +167,11 @@ def main() -> int: "candidate_count": len(candidates), "candidate_bytes": sum(item.size_bytes for item in candidates), "candidates": [item.relative_path for item in candidates], - "deleted_count": len(deleted), - "deleted": deleted, + "deleted_count": 0, + "deleted": [], + "quarantined_count": len(quarantined), + "quarantined": quarantined, + "quarantine_manifest": str(quarantine_manifest) if quarantine_manifest else None, "blocked_reason": blocked_reason, "protected_prefixes": report["cleanup"]["protected_prefixes"], "backup": ( @@ -81,7 +179,7 @@ def main() -> int: "release_id": backup.release_id, "created_at": backup.created_at.isoformat(), "age_hours": round(backup.age_hours, 3), - "git_commit": backup.git_commit, + "backup_tool_revision": backup.backup_tool_revision, } if backup else None diff --git a/scripts/release_backup_guard.py b/scripts/release_backup_guard.py index 3d98ead0..ac456782 100644 --- a/scripts/release_backup_guard.py +++ b/scripts/release_backup_guard.py @@ -9,6 +9,8 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path +from release_backup_snapshot import verify_backup as verify_byte_snapshots + @dataclass(frozen=True) class VerifiedBackup: @@ -16,7 +18,7 @@ class VerifiedBackup: release_id: str created_at: datetime age_hours: float - git_commit: str + backup_tool_revision: str def _sha256(path: Path) -> str: @@ -62,6 +64,8 @@ def verify_current_backup( missing = sorted(name for name in required if not (root / name).is_file()) if missing: raise RuntimeError(f"Backup is incomplete; missing: {', '.join(missing)}") + if not (root / "storage-snapshot").is_dir(): + raise RuntimeError("Backup is incomplete; missing: storage-snapshot") checksum_lines = (root / "CHECKSUMS.sha256").read_text(encoding="utf-8").splitlines() checked: set[str] = set() @@ -93,6 +97,9 @@ def verify_current_backup( raise RuntimeError("Backup was made from an insecure database configuration") if manifest.get("inventory_mode") != "sha256" or manifest.get("storage_inventory_requested") is not True: raise RuntimeError("Destructive maintenance requires a SHA-256 storage inventory backup") + if manifest.get("storage_snapshot_requested") is not True: + raise RuntimeError("Destructive maintenance requires a byte-complete storage snapshot") + verify_byte_snapshots(root) created = _created_at(manifest.get("created_at")) current = now or datetime.now(timezone.utc) @@ -107,17 +114,17 @@ def verify_current_backup( ) release_id = manifest.get("release_id") - git_commit = manifest.get("git_commit") + backup_tool_revision = manifest.get("backup_tool_revision", manifest.get("git_commit")) if not isinstance(release_id, str) or not release_id: raise RuntimeError("Backup release id is missing") - if not isinstance(git_commit, str) or len(git_commit) < 7: - raise RuntimeError("Backup Git commit is missing") + if not isinstance(backup_tool_revision, str) or len(backup_tool_revision) < 7: + raise RuntimeError("Backup tool revision is missing") return VerifiedBackup( backup_dir=root, release_id=release_id, created_at=created, age_hours=age_hours, - git_commit=git_commit, + backup_tool_revision=backup_tool_revision, ) diff --git a/scripts/release_backup_snapshot.py b/scripts/release_backup_snapshot.py new file mode 100644 index 00000000..6cf845e0 --- /dev/null +++ b/scripts/release_backup_snapshot.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +"""Create and verify byte-complete, symlink-safe release backup snapshots.""" + +from __future__ import annotations + +import argparse +import errno +import hashlib +import json +import os +import re +import stat +from dataclasses import dataclass +from pathlib import Path, PurePosixPath + + +MANIFEST_HEADER = "relative_path\tsize_bytes\tmtime_ns\tsha256" +SAFE_LABEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") +FICLONE = 0x40049409 +FICLONE_FALLBACK_ERRORS = { + errno.EXDEV, + errno.EOPNOTSUPP, + errno.ENOTTY, + errno.EINVAL, + errno.ENOSYS, +} + + +@dataclass(frozen=True) +class SourceEntry: + path: Path + relative_path: str + stat_result: os.stat_result + is_directory: bool + + +@dataclass(frozen=True) +class ManifestEntry: + relative_path: str + size_bytes: int + mtime_ns: int + sha256: str + + +def _safe_relative(value: str) -> str: + if not value or "\t" in value or "\n" in value or "\r" in value: + raise RuntimeError(f"Unsupported snapshot path: {value!r}") + candidate = PurePosixPath(value) + if candidate.is_absolute() or any(part in {"", ".", ".."} for part in candidate.parts): + raise RuntimeError(f"Unsafe snapshot path: {value!r}") + return candidate.as_posix() + + +def _collect(root: Path) -> list[SourceEntry]: + entries: list[SourceEntry] = [] + for current, directory_names, file_names in os.walk(root, topdown=True, followlinks=False): + directory_names.sort() + file_names.sort() + current_path = Path(current) + for name, is_directory in [ + *((name, True) for name in directory_names), + *((name, False) for name in file_names), + ]: + path = current_path / name + details = path.lstat() + relative = _safe_relative(path.relative_to(root).as_posix()) + if stat.S_ISLNK(details.st_mode): + raise RuntimeError(f"Release snapshot refuses symlinked content: {relative}") + if is_directory and not stat.S_ISDIR(details.st_mode): + raise RuntimeError(f"Snapshot directory changed during inventory: {relative}") + if not is_directory and not stat.S_ISREG(details.st_mode): + raise RuntimeError(f"Release snapshot refuses non-regular content: {relative}") + entries.append(SourceEntry(path, relative, details, is_directory)) + return entries + + +def _same_file_state(before: os.stat_result, after: os.stat_result) -> bool: + return ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + ) == ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _copy_all(source_descriptor: int, destination_descriptor: int) -> None: + while True: + value = os.read(source_descriptor, 1024 * 1024) + if not value: + return + view = memoryview(value) + while view: + written = os.write(destination_descriptor, view) + if written <= 0: + raise RuntimeError("Snapshot copy stopped before writing all bytes") + view = view[written:] + + +def _clone_or_copy(entry: SourceEntry, destination: Path) -> ManifestEntry: + source_flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + destination_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0) + source_descriptor = os.open(entry.path, source_flags) + destination_descriptor = -1 + try: + opened = os.fstat(source_descriptor) + if not stat.S_ISREG(opened.st_mode) or not _same_file_state(entry.stat_result, opened): + raise RuntimeError(f"Snapshot file changed before copying: {entry.relative_path}") + destination.parent.mkdir(parents=True, exist_ok=True) + destination_descriptor = os.open(destination, destination_flags, stat.S_IMODE(opened.st_mode)) + cloned = False + if os.name == "posix": + try: + import fcntl + + fcntl.ioctl(destination_descriptor, FICLONE, source_descriptor) + cloned = True + except OSError as exc: + if exc.errno not in FICLONE_FALLBACK_ERRORS: + raise + if not cloned: + os.lseek(source_descriptor, 0, os.SEEK_SET) + os.ftruncate(destination_descriptor, 0) + _copy_all(source_descriptor, destination_descriptor) + os.fsync(destination_descriptor) + after = os.fstat(source_descriptor) + if not _same_file_state(opened, after): + raise RuntimeError(f"Snapshot file changed while copying: {entry.relative_path}") + except BaseException: + if destination_descriptor >= 0: + os.close(destination_descriptor) + destination_descriptor = -1 + destination.unlink(missing_ok=True) + raise + finally: + if destination_descriptor >= 0: + os.close(destination_descriptor) + os.close(source_descriptor) + + os.chmod(destination, stat.S_IMODE(entry.stat_result.st_mode) & ~0o222, follow_symlinks=False) + retained_times = (entry.stat_result.st_atime_ns, entry.stat_result.st_mtime_ns) + try: + os.utime(destination, ns=retained_times, follow_symlinks=False) + except NotImplementedError: + # Windows does not expose no-follow utime. The destination was created + # exclusively above; recheck it before using the portable call. + if destination.is_symlink(): + destination.unlink(missing_ok=True) + raise RuntimeError(f"Snapshot destination became a symlink: {entry.relative_path}") + os.utime(destination, ns=retained_times) + source_checksum = _sha256(entry.path) + snapshot_checksum = _sha256(destination) + final_source = entry.path.lstat() + if not _same_file_state(entry.stat_result, final_source): + raise RuntimeError(f"Snapshot file changed during checksum verification: {entry.relative_path}") + if source_checksum != snapshot_checksum: + raise RuntimeError(f"Snapshot checksum differs from source: {entry.relative_path}") + return ManifestEntry( + relative_path=entry.relative_path, + size_bytes=entry.stat_result.st_size, + mtime_ns=entry.stat_result.st_mtime_ns, + sha256=snapshot_checksum, + ) + + +def _link_verified_prior( + entry: SourceEntry, + destination: Path, + prior_root: Path, + prior_manifest: dict[str, ManifestEntry], +) -> ManifestEntry | None: + retained = prior_manifest.get(entry.relative_path) + if retained is None or retained.size_bytes != entry.stat_result.st_size: + return None + source_checksum = _sha256(entry.path) + final_source = entry.path.lstat() + if not _same_file_state(entry.stat_result, final_source): + raise RuntimeError(f"Snapshot file changed during prior comparison: {entry.relative_path}") + if source_checksum != retained.sha256: + return None + prior_path = prior_root / entry.relative_path + try: + prior_details = prior_path.lstat() + except FileNotFoundError: + return None + if not stat.S_ISREG(prior_details.st_mode) or prior_details.st_size != retained.size_bytes: + raise RuntimeError(f"Prior snapshot file is not reusable: {entry.relative_path}") + if _sha256(prior_path) != retained.sha256: + raise RuntimeError(f"Prior snapshot checksum changed: {entry.relative_path}") + destination.parent.mkdir(parents=True, exist_ok=True) + os.link(prior_path, destination, follow_symlinks=False) + if destination.stat().st_size != retained.size_bytes or _sha256(destination) != retained.sha256: + destination.unlink(missing_ok=True) + raise RuntimeError(f"Hard-linked snapshot verification failed: {entry.relative_path}") + return ManifestEntry( + relative_path=entry.relative_path, + size_bytes=entry.stat_result.st_size, + mtime_ns=entry.stat_result.st_mtime_ns, + sha256=source_checksum, + ) + + +def read_manifest(path: Path) -> dict[str, ManifestEntry]: + lines = path.read_text(encoding="utf-8").splitlines() + if not lines or lines[0] != MANIFEST_HEADER: + raise RuntimeError(f"Snapshot inventory has an invalid header: {path}") + entries: dict[str, ManifestEntry] = {} + for line in lines[1:]: + fields = line.split("\t") + if len(fields) != 4: + raise RuntimeError(f"Snapshot inventory has an invalid row: {line!r}") + relative_path, size_text, mtime_text, checksum = fields + relative_path = _safe_relative(relative_path) + if relative_path in entries: + raise RuntimeError(f"Snapshot inventory repeats a path: {relative_path}") + try: + size_bytes = int(size_text) + mtime_ns = int(mtime_text) + except ValueError as exc: + raise RuntimeError(f"Snapshot inventory has invalid metadata: {relative_path}") from exc + if size_bytes < 0 or mtime_ns < 0 or not re.fullmatch(r"[0-9a-f]{64}", checksum): + raise RuntimeError(f"Snapshot inventory has invalid retained state: {relative_path}") + entries[relative_path] = ManifestEntry(relative_path, size_bytes, mtime_ns, checksum) + return entries + + +def verify_snapshot(snapshot_path: Path, manifest_path: Path) -> None: + root = snapshot_path.expanduser().resolve(strict=True) + if not root.is_dir(): + raise RuntimeError(f"Snapshot path is not a directory: {root}") + expected = read_manifest(manifest_path) + observed_entries = _collect(root) + observed_files = {item.relative_path: item for item in observed_entries if not item.is_directory} + extra = sorted(set(observed_files) - set(expected)) + missing = sorted(set(expected) - set(observed_files)) + if extra: + raise RuntimeError(f"Snapshot contains unmanifested files: {', '.join(extra[:10])}") + if missing: + raise RuntimeError(f"Snapshot omits manifested files: {', '.join(missing[:10])}") + for relative, retained in expected.items(): + current = observed_files[relative] + if current.stat_result.st_size != retained.size_bytes: + raise RuntimeError(f"Snapshot size differs for: {relative}") + if _sha256(current.path) != retained.sha256: + raise RuntimeError(f"Snapshot checksum differs for: {relative}") + + +def create_snapshot( + source: Path, + snapshot_path: Path, + manifest_path: Path, + *, + label: str, + link_dest_snapshot: Path | None = None, + link_dest_manifest: Path | None = None, +) -> None: + if not SAFE_LABEL.fullmatch(label): + raise RuntimeError(f"Unsafe snapshot label: {label!r}") + root = source.expanduser() + if root.is_symlink(): + raise RuntimeError(f"Release snapshot refuses a symlinked root: {root}") + root = root.resolve(strict=True) + if not root.is_dir(): + raise RuntimeError(f"Snapshot source is not a directory: {root}") + snapshot_path = snapshot_path.expanduser().resolve() + manifest_path = manifest_path.expanduser().resolve() + for output in (snapshot_path, manifest_path): + try: + output.relative_to(root) + except ValueError: + pass + else: + raise RuntimeError("Release snapshot output must not be inside its source tree") + if snapshot_path.exists(): + raise RuntimeError(f"Snapshot destination already exists: {snapshot_path}") + snapshot_path.mkdir(parents=True, exist_ok=False) + manifest_path.parent.mkdir(parents=True, exist_ok=True) + + prior_root: Path | None = None + prior_manifest: dict[str, ManifestEntry] = {} + if (link_dest_snapshot is None) != (link_dest_manifest is None): + raise RuntimeError("Prior snapshot and manifest must be supplied together") + if link_dest_snapshot is not None and link_dest_manifest is not None: + prior_root = link_dest_snapshot.expanduser().resolve(strict=True) + prior_manifest = read_manifest(link_dest_manifest.expanduser().resolve(strict=True)) + + initial = _collect(root) + retained: list[ManifestEntry] = [] + for entry in initial: + destination = snapshot_path / entry.relative_path + if entry.is_directory: + destination.mkdir(parents=True, exist_ok=False) + os.chmod(destination, stat.S_IMODE(entry.stat_result.st_mode), follow_symlinks=False) + continue + linked = ( + _link_verified_prior(entry, destination, prior_root, prior_manifest) + if prior_root is not None + else None + ) + retained.append(linked or _clone_or_copy(entry, destination)) + final = _collect(root) + if [(item.relative_path, item.is_directory) for item in initial] != [ + (item.relative_path, item.is_directory) for item in final + ]: + raise RuntimeError(f"Snapshot source contents changed while backup was running: {root}") + with manifest_path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(f"{MANIFEST_HEADER}\n") + for entry in retained: + handle.write( + f"{entry.relative_path}\t{entry.size_bytes}\t{entry.mtime_ns}\t{entry.sha256}\n" + ) + verify_snapshot(snapshot_path, manifest_path) + + +def verify_backup(backup_dir: Path) -> None: + root = backup_dir.expanduser().resolve(strict=True) + payload = json.loads((root / "manifest.json").read_text(encoding="utf-8")) + for label in ("storage", "models"): + requested = payload.get(f"{label}_inventory_requested") is True + snapshotted = payload.get(f"{label}_snapshot_requested") is True + if requested != snapshotted: + raise RuntimeError(f"Backup manifest does not bind the {label} inventory to a snapshot") + if requested: + verify_snapshot(root / f"{label}-snapshot", root / f"{label}-manifest.tsv") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + create = subparsers.add_parser("create") + create.add_argument("--source", type=Path, required=True) + create.add_argument("--snapshot", type=Path, required=True) + create.add_argument("--manifest", type=Path, required=True) + create.add_argument("--label", required=True) + create.add_argument("--link-dest-snapshot", type=Path) + create.add_argument("--link-dest-manifest", type=Path) + verify = subparsers.add_parser("verify") + verify.add_argument("--snapshot", type=Path, required=True) + verify.add_argument("--manifest", type=Path, required=True) + verify_backup_parser = subparsers.add_parser("verify-backup") + verify_backup_parser.add_argument("--backup-dir", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.command == "create": + create_snapshot( + args.source, + args.snapshot, + args.manifest, + label=args.label, + link_dest_snapshot=args.link_dest_snapshot, + link_dest_manifest=args.link_dest_manifest, + ) + elif args.command == "verify": + verify_snapshot(args.snapshot, args.manifest) + else: + verify_backup(args.backup_dir) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/restore_storage_quarantine.py b/scripts/restore_storage_quarantine.py new file mode 100644 index 00000000..5f8c7299 --- /dev/null +++ b/scripts/restore_storage_quarantine.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Restore a traceable GeoIntel cleanup quarantine without overwriting data.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from datetime import datetime, timezone +from pathlib import Path + + +CONFIRMATION = "RESTORE_QUARANTINED_ARTIFACTS" +CLEANUP_PREFIXES = ("exports", "previews", "tiles", "masks", "derived", "rasters/derived") + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_manifest(path: Path, payload: dict[str, object]) -> None: + temporary = path.with_suffix(".json.partial") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--storage-root", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--confirm", required=True) + return parser.parse_args() + + +def _within(path: Path, root: Path, *, label: str) -> Path: + resolved = path.resolve() + try: + resolved.relative_to(root) + except ValueError as exc: + raise RuntimeError(f"{label} escapes the storage root") from exc + return resolved + + +def main() -> int: + args = parse_args() + if args.confirm != CONFIRMATION: + raise RuntimeError(f"Refusing restore; pass --confirm {CONFIRMATION}") + storage_root = args.storage_root.expanduser().resolve() + manifest_path = _within(args.manifest.expanduser(), storage_root, label="Manifest") + protected_quarantine_root = storage_root / "operator-evidence" / "cleanup-quarantine" + try: + manifest_path.relative_to(protected_quarantine_root.resolve()) + except ValueError as exc: + raise RuntimeError("Manifest is outside the protected cleanup quarantine") from exc + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + if payload.get("schema_version") != 1: + raise RuntimeError("Unsupported quarantine manifest") + if payload.get("state") not in {"complete", "in_progress", "restore_in_progress", "restored"}: + raise RuntimeError("Quarantine manifest is not in a restorable state") + raw_entries = payload.get("entries") + if not isinstance(raw_entries, list): + raise RuntimeError("Quarantine manifest entries are invalid") + + plans: list[tuple[str, dict[str, object], Path, Path]] = [] + for raw_entry in raw_entries: + if not isinstance(raw_entry, dict): + raise RuntimeError("Quarantine manifest entry is invalid") + status = raw_entry.get("status") + if status not in {"planned", "linked", "quarantined", "restore_linked", "restored"}: + raise RuntimeError(f"Quarantine manifest entry has an invalid status: {status!r}") + relative_path = raw_entry.get("relative_path") + quarantine_relative_path = raw_entry.get("quarantine_relative_path") + expected_hash = raw_entry.get("sha256") + expected_size = raw_entry.get("size_bytes") + if ( + not isinstance(relative_path, str) + or not isinstance(expected_hash, str) + or not isinstance(expected_size, int) + ): + raise RuntimeError("Quarantine manifest entry lacks recovery metadata") + if not any( + relative_path == prefix or relative_path.startswith(f"{prefix}/") + for prefix in CLEANUP_PREFIXES + ): + raise RuntimeError(f"Original path is outside the cleanup allowlist: {relative_path}") + original = _within(storage_root / relative_path, storage_root, label="Original path") + if not isinstance(quarantine_relative_path, str): + if status != "planned": + raise RuntimeError("Quarantine manifest entry lacks its retained path") + quarantine_relative_path = ( + manifest_path.parent / "files" / relative_path + ).relative_to(storage_root).as_posix() + raw_entry["quarantine_relative_path"] = quarantine_relative_path + quarantined = _within( + storage_root / quarantine_relative_path, + storage_root, + label="Quarantine path", + ) + try: + quarantined.relative_to(manifest_path.parent.resolve()) + except ValueError as exc: + raise RuntimeError("Quarantine entry escapes its operation directory") from exc + original_exists = original.exists() + quarantined_exists = quarantined.exists() + if original_exists: + if not original.is_file() or original.stat().st_size != expected_size or sha256(original) != expected_hash: + raise RuntimeError(f"Restore destination already exists with different bytes: {relative_path}") + if quarantined_exists: + if ( + not quarantined.is_file() + or quarantined.stat().st_size != expected_size + or sha256(quarantined) != expected_hash + ): + raise RuntimeError(f"Quarantined artifact checksum mismatch: {quarantine_relative_path}") + if original_exists and quarantined_exists: + if not os.path.samefile(original, quarantined): + raise RuntimeError(f"Restore destination already exists: {relative_path}") + plans.append(("remove_duplicate_link", raw_entry, quarantined, original)) + elif original_exists: + plans.append(("mark_restored", raw_entry, quarantined, original)) + elif quarantined_exists: + plans.append(("restore", raw_entry, quarantined, original)) + else: + raise RuntimeError(f"Both original and quarantined artifacts are missing: {relative_path}") + + payload["state"] = "restore_in_progress" + write_manifest(manifest_path, payload) + for action, entry, quarantined, original in plans: + if action == "restore": + original.parent.mkdir(parents=True, exist_ok=True) + _within(original, storage_root, label="Original path") + try: + os.link(quarantined, original, follow_symlinks=False) + except FileExistsError as exc: + raise RuntimeError(f"Restore destination was created concurrently: {original}") from exc + if not os.path.samefile(quarantined, original): + original.unlink(missing_ok=True) + raise RuntimeError(f"Restore link verification failed: {original}") + entry["status"] = "restore_linked" + write_manifest(manifest_path, payload) + quarantined.unlink() + elif action == "remove_duplicate_link": + quarantined.unlink() + entry["status"] = "restored" + entry["restored_at"] = datetime.now(timezone.utc).isoformat() + write_manifest(manifest_path, payload) + payload["state"] = "restored" + payload["restored_at"] = datetime.now(timezone.utc).isoformat() + write_manifest(manifest_path, payload) + print(json.dumps({"state": "restored", "restored_count": len(plans)}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 9cc10a07..77d7a24f 100644 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -43,6 +43,8 @@ echo "== GeoIntel run readiness check ==" "$PYTHON_BIN" -m py_compile scripts/build_release_package.py "$PYTHON_BIN" -m py_compile scripts/verify_python_lock.py "$PYTHON_BIN" scripts/verify_python_lock.py +"$PYTHON_BIN" -m py_compile scripts/verify_repository_layout.py +"$PYTHON_BIN" scripts/verify_repository_layout.py "$PYTHON_BIN" -m py_compile scripts/verify_security_exceptions.py "$PYTHON_BIN" scripts/verify_security_exceptions.py bash -n scripts/backup_release_state.sh @@ -55,6 +57,7 @@ bash -n scripts/scan_container_image.sh bash -n scripts/audit_python_dependencies.sh echo "Using Python: ${PYTHON_BIN}" bash scripts/check_repo_structure.sh +"$PYTHON_BIN" -m ruff check backend scripts tests ${PYTHON_BIN} scripts/smoke_docs.py ${PYTHON_BIN} scripts/validate_fixtures.py ${PYTHON_BIN} scripts/smoke_contracts.py @@ -129,8 +132,10 @@ ${PYTHON_BIN} -m py_compile scripts/migrate_runtime_model_provenance.py ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m py_compile scripts/archive_technical_projects.py ${PYTHON_BIN} -m py_compile scripts/release_backup_guard.py +${PYTHON_BIN} -m py_compile scripts/release_backup_snapshot.py ${PYTHON_BIN} -m py_compile scripts/audit_data_operations.py ${PYTHON_BIN} -m py_compile scripts/cleanup_storage_artifacts.py +${PYTHON_BIN} -m py_compile scripts/restore_storage_quarantine.py ${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m compileall backend/app (cd backend && ${PYTHON_BIN} -m pytest -W error::DeprecationWarning) @@ -149,6 +154,7 @@ bash -n deploy/unraid/gosu-setpriv bash -n deploy/unraid/run-dockerman-container.sh bash -n deploy/unraid/deploy-release.sh bash -n deploy/unraid/rollback-dockerman-container.sh +bash -n deploy/unraid/restore-predeploy-database.sh bash -n scripts/verify_browser_runtime.sh bash -n scripts/verify_demo_export_workflow.sh bash -n scripts/verify_demo_raster_workflow.sh diff --git a/scripts/scan_container_image.sh b/scripts/scan_container_image.sh index 87b5406a..fdd443d3 100644 --- a/scripts/scan_container_image.sh +++ b/scripts/scan_container_image.sh @@ -21,8 +21,17 @@ esac docker image inspect "$TARGET_IMAGE" >/dev/null mkdir -p "$ROOT/$(dirname "$OUTPUT")" "$CACHE_DIR" "$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" -"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" \ - --print-container-ids > "$IGNORE_FILE" +mapfile -t ignored_container_ids < <( + "$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" \ + --print-container-ids | tr -d '\r' +) +ignore_args=() +trivy_ignore_args=() +if [ "${#ignored_container_ids[@]}" -gt 0 ]; then + printf '%s\n' "${ignored_container_ids[@]}" > "$IGNORE_FILE" + ignore_args=(-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro") + trivy_ignore_args=(--ignorefile "$CONTAINER_IGNORE_FILE") +fi # Keep the complete report, including vulnerabilities without an available fix. docker run --rm \ @@ -45,14 +54,14 @@ docker run --rm \ docker run --rm \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$CACHE_DIR:/root/.cache/trivy" \ - -v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro" \ + "${ignore_args[@]}" \ "$TRIVY_IMAGE" \ image \ --scanners vuln \ --timeout 20m \ --skip-version-check \ --ignore-unfixed \ - --ignorefile "$CONTAINER_IGNORE_FILE" \ + "${trivy_ignore_args[@]}" \ --skip-files /usr/local/bin/gosu \ --severity HIGH,CRITICAL \ --exit-code 1 \ diff --git a/scripts/verify_release_backup.sh b/scripts/verify_release_backup.sh index 93b91ccf..a5fdeba4 100644 --- a/scripts/verify_release_backup.sh +++ b/scripts/verify_release_backup.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" CONTAINER="geointel" BACKUP_DIR="" @@ -59,15 +60,22 @@ required = { "database_name", "database_user", "image_id", - "git_commit", } missing = sorted(required - payload.keys()) if missing: raise SystemExit(f"Backup manifest misses: {', '.join(missing)}") if payload["schema_version"] != 1 or payload["read_only_source"] is not True: raise SystemExit("Unsupported or unsafe backup manifest") +tool_revision = payload.get("backup_tool_revision", payload.get("git_commit")) +if not isinstance(tool_revision, str) or len(tool_revision) < 7: + raise SystemExit("Backup manifest lacks its backup-tool revision") +running_revision = payload.get("running_image_revision") +if running_revision is not None and not isinstance(running_revision, str): + raise SystemExit("Backup manifest has an invalid running-image revision") PY +python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$BACKUP_DIR" + if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then echo "Container '$CONTAINER' is required to run pg_restore --list." >&2 exit 3 diff --git a/scripts/verify_security_exceptions.py b/scripts/verify_security_exceptions.py index 898c1293..0a7348c6 100644 --- a/scripts/verify_security_exceptions.py +++ b/scripts/verify_security_exceptions.py @@ -17,29 +17,39 @@ EXCEPTIONS_PATH = ROOT / "security" / "pip-audit-exceptions.json" def load_and_validate() -> tuple[dict[str, object], list[str]]: payload = json.loads(EXCEPTIONS_PATH.read_text(encoding="utf-8")) errors: list[str] = [] - try: - review_by = dt.date.fromisoformat(str(payload["review_by"])) - except (KeyError, ValueError): - errors.append("review_by must be an ISO date") - review_by = dt.date.min - if review_by < dt.date.today(): - errors.append(f"dependency exception review expired on {review_by.isoformat()}") - if payload.get("package") != "starlette": - errors.append("only the documented Starlette compatibility exception is allowed") - controls = payload.get("compensating_controls") - if not isinstance(controls, list) or len(controls) < 3: - errors.append("at least three compensating controls are required") + if set(payload) != {"schema_version", "advisories"}: + errors.append("exception policy must contain only schema_version and advisories") + if payload.get("schema_version") != 1: + errors.append("schema_version must be 1") advisories = payload.get("advisories") - if not isinstance(advisories, list) or not advisories: - errors.append("at least one advisory exception is required") + if not isinstance(advisories, list): + errors.append("advisories must be a list") else: ids = [str(item.get("id", "")) for item in advisories if isinstance(item, dict)] if len(ids) != len(set(ids)) or any(not item.startswith("PYSEC-") for item in ids): errors.append("advisory IDs must be unique PYSEC identifiers") for item in advisories: - if not isinstance(item, dict) or len(str(item.get("reason", ""))) < 30: + if not isinstance(item, dict): + errors.append("every advisory must be an object") + continue + required = {"id", "package", "review_by", "reason"} + allowed = required | {"aliases"} + if not required.issubset(item) or not set(item).issubset(allowed): + errors.append("every advisory must match the documented exception schema") + if not str(item.get("package", "")).strip(): + errors.append("every advisory requires a package") + if len(str(item.get("reason", ""))) < 30: errors.append("every advisory requires a specific reason") - break + try: + review_by = dt.date.fromisoformat(str(item["review_by"])) + except (KeyError, ValueError): + errors.append("every advisory review_by must be an ISO date") + else: + if review_by < dt.date.today(): + errors.append( + f"dependency exception {item.get('id', '')} expired on " + f"{review_by.isoformat()}" + ) aliases = [ str(alias) for item in advisories @@ -72,8 +82,7 @@ def main() -> int: print(alias) else: print( - "Dependency exceptions valid through " - f"{payload['review_by']} with documented compensating controls." + f"Dependency exception policy valid; {len(payload['advisories'])} active exception(s)." ) return 0 diff --git a/security/pip-audit-exceptions.json b/security/pip-audit-exceptions.json index 291731f8..38a5bcd4 100644 --- a/security/pip-audit-exceptions.json +++ b/security/pip-audit-exceptions.json @@ -1,35 +1,4 @@ { - "review_by": "2026-08-31", - "package": "starlette", - "constraint": "FastAPI 0.139.2 requires Starlette below 0.53; patched Starlette releases start at 1.0.1 or later.", - "compensating_controls": [ - "GeoIntel rejects missing or ambiguous Host headers and request targets before request.url is accessed.", - "Application code logs the raw ASGI path and does not use request.url for authorization or routing.", - "application/x-www-form-urlencoded is rejected; supported uploads use bounded multipart requests behind nginx.", - "The release image is Linux, no HTTPEndpoint route class is used, and the Windows StaticFiles advisory is not applicable." - ], - "advisories": [ - { - "id": "PYSEC-2026-161", - "reason": "Host/path ambiguity is rejected at the outer request middleware and request.url is not a security boundary." - }, - { - "id": "PYSEC-2026-248", - "reason": "Non-slash request paths and ambiguous Host values are rejected before URL reconstruction." - }, - { - "id": "PYSEC-2026-249", - "aliases": ["CVE-2026-54283"], - "reason": "GeoIntel rejects application/x-www-form-urlencoded before Starlette form parsing." - }, - { - "id": "PYSEC-2026-2280", - "reason": "GeoIntel registers FastAPI APIRouter functions and has no Starlette HTTPEndpoint routes." - }, - { - "id": "PYSEC-2026-2281", - "aliases": ["CVE-2026-48818"], - "reason": "The supported all-in-one production runtime is Linux; the advisory affects Windows StaticFiles." - } - ] + "schema_version": 1, + "advisories": [] }