Complete RC6 supply chain gates
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
name: GeoIntel release gates
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, develop, "codex/**", "build/**"]
|
||||
pull_request:
|
||||
branches: [main, develop]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: geointel-release-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
quality:
|
||||
name: Compile, test, contracts and builds
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: pip
|
||||
cache-dependency-path: backend/requirements-ci.lock
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
- name: Install locked backend dependencies
|
||||
run: |
|
||||
python -m pip install --disable-pip-version-check --require-hashes -r backend/requirements-ci.lock
|
||||
python -m pip install --disable-pip-version-check --no-deps -e backend
|
||||
- name: Install locked frontend dependencies
|
||||
working-directory: frontend
|
||||
run: npm ci
|
||||
- name: Verify dependency lock policy
|
||||
run: python scripts/verify_python_lock.py
|
||||
- name: Run complete release readiness gate
|
||||
env:
|
||||
PYTHON_BIN: python
|
||||
run: bash scripts/run_readiness_check.sh
|
||||
- name: Render migration and Compose evidence
|
||||
run: |
|
||||
mkdir -p artifacts
|
||||
cd backend
|
||||
python -m alembic upgrade head --sql > ../artifacts/alembic-upgrade.sql
|
||||
cd ..
|
||||
docker compose config > artifacts/docker-compose.resolved.yml
|
||||
- name: Publish quality evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: quality-evidence
|
||||
path: |
|
||||
artifacts/alembic-upgrade.sql
|
||||
artifacts/docker-compose.resolved.yml
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
dependency-audit:
|
||||
name: Python and npm vulnerability policy
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: pip
|
||||
cache-dependency-path: backend/requirements-ci.lock
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
- name: Audit locked Python dependencies
|
||||
run: |
|
||||
mkdir -p artifacts
|
||||
python -m pip install --disable-pip-version-check pip-audit==2.10.1
|
||||
bash scripts/audit_python_dependencies.sh
|
||||
- name: Audit locked frontend dependencies
|
||||
working-directory: frontend
|
||||
run: |
|
||||
npm ci
|
||||
npm audit --audit-level=high --json > ../artifacts/npm-audit.json
|
||||
- name: Publish dependency evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dependency-audits
|
||||
path: |
|
||||
artifacts/pip-audit-full.json
|
||||
artifacts/pip-audit-policy.json
|
||||
artifacts/npm-audit.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
container:
|
||||
name: GIS image, SBOM and container scan
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build non-AI release image
|
||||
env:
|
||||
RELEASE_SHA: ${{ gitea.sha }}
|
||||
run: |
|
||||
mkdir -p artifacts
|
||||
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
docker build \
|
||||
-f deploy/unraid/Dockerfile.all-in-one \
|
||||
--build-arg GEOINTEL_INSTALL_AI=false \
|
||||
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
|
||||
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
|
||||
-t "geointel-ci:$RELEASE_SHA-gis" \
|
||||
.
|
||||
docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json
|
||||
- name: Generate SPDX SBOM
|
||||
env:
|
||||
RELEASE_SHA: ${{ gitea.sha }}
|
||||
run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis"
|
||||
- name: Enforce container vulnerability policy
|
||||
env:
|
||||
RELEASE_SHA: ${{ gitea.sha }}
|
||||
run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis"
|
||||
- name: Publish container evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: container-evidence
|
||||
path: |
|
||||
artifacts/image-inspect.json
|
||||
artifacts/geointel-sbom.spdx.json
|
||||
artifacts/geointel-container-vulnerabilities.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
@@ -1,30 +0,0 @@
|
||||
name: GeoIntel CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop, 'build/**' ]
|
||||
pull_request:
|
||||
branches: [ main, develop ]
|
||||
|
||||
jobs:
|
||||
docs-smoke:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Validate repository docs
|
||||
run: |
|
||||
python scripts/smoke_docs.py
|
||||
python scripts/validate_fixtures.py
|
||||
|
||||
contract-smoke:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Run contract smoke checks
|
||||
run: python scripts/smoke_contracts.py
|
||||
@@ -0,0 +1,141 @@
|
||||
name: GeoIntel release gates
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, develop, "codex/**", "build/**"]
|
||||
pull_request:
|
||||
branches: [main, develop]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: geointel-release-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
quality:
|
||||
name: Compile, test, contracts and builds
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: pip
|
||||
cache-dependency-path: backend/requirements-ci.lock
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
- name: Install locked backend dependencies
|
||||
run: |
|
||||
python -m pip install --disable-pip-version-check --require-hashes -r backend/requirements-ci.lock
|
||||
python -m pip install --disable-pip-version-check --no-deps -e backend
|
||||
- name: Install locked frontend dependencies
|
||||
working-directory: frontend
|
||||
run: npm ci
|
||||
- name: Verify dependency lock policy
|
||||
run: python scripts/verify_python_lock.py
|
||||
- name: Run complete release readiness gate
|
||||
env:
|
||||
PYTHON_BIN: python
|
||||
run: bash scripts/run_readiness_check.sh
|
||||
- name: Render migration and Compose evidence
|
||||
run: |
|
||||
mkdir -p artifacts
|
||||
cd backend
|
||||
python -m alembic upgrade head --sql > ../artifacts/alembic-upgrade.sql
|
||||
cd ..
|
||||
docker compose config > artifacts/docker-compose.resolved.yml
|
||||
- name: Publish quality evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: quality-evidence
|
||||
path: |
|
||||
artifacts/alembic-upgrade.sql
|
||||
artifacts/docker-compose.resolved.yml
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
dependency-audit:
|
||||
name: Python and npm vulnerability policy
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: pip
|
||||
cache-dependency-path: backend/requirements-ci.lock
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
- name: Audit locked Python dependencies
|
||||
run: |
|
||||
mkdir -p artifacts
|
||||
python -m pip install --disable-pip-version-check pip-audit==2.10.1
|
||||
bash scripts/audit_python_dependencies.sh
|
||||
- name: Audit locked frontend dependencies
|
||||
working-directory: frontend
|
||||
run: |
|
||||
npm ci
|
||||
npm audit --audit-level=high --json > ../artifacts/npm-audit.json
|
||||
- name: Publish dependency evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dependency-audits
|
||||
path: |
|
||||
artifacts/pip-audit-full.json
|
||||
artifacts/pip-audit-policy.json
|
||||
artifacts/npm-audit.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
container:
|
||||
name: GIS image, SBOM and container scan
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build non-AI release image
|
||||
env:
|
||||
RELEASE_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
mkdir -p artifacts
|
||||
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
docker build \
|
||||
-f deploy/unraid/Dockerfile.all-in-one \
|
||||
--build-arg GEOINTEL_INSTALL_AI=false \
|
||||
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
|
||||
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
|
||||
-t "geointel-ci:$RELEASE_SHA-gis" \
|
||||
.
|
||||
docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json
|
||||
- name: Generate SPDX SBOM
|
||||
env:
|
||||
RELEASE_SHA: ${{ github.sha }}
|
||||
run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis"
|
||||
- name: Enforce container vulnerability policy
|
||||
env:
|
||||
RELEASE_SHA: ${{ github.sha }}
|
||||
run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis"
|
||||
- name: Publish container evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: container-evidence
|
||||
path: |
|
||||
artifacts/image-inspect.json
|
||||
artifacts/geointel-sbom.spdx.json
|
||||
artifacts/geointel-container-vulnerabilities.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
@@ -16,6 +16,7 @@ build/
|
||||
|
||||
# Large local data
|
||||
/artifacts/
|
||||
/.cache/
|
||||
/datasets/raw/*
|
||||
/datasets/processed/*
|
||||
/datasets/cache/*
|
||||
|
||||
@@ -14,6 +14,11 @@
|
||||
regression references.
|
||||
- Added an executable autonomous RC-0 through RC-11 roadmap and explicit
|
||||
national/maritime scope freeze.
|
||||
- Replaced the minimal smoke-only CI with complete Gitea and GitHub release
|
||||
gates for readiness, offline migration evidence, Docker configuration,
|
||||
dependency audits, a digest-pinned container scan and an SPDX SBOM.
|
||||
- Added a hashed Linux/Python 3.11 GIS/dev lock with an enforced input
|
||||
fingerprint while keeping PyTorch and Ultralytics out of base CI.
|
||||
- Folded fresh-install, upgrade, rollback and runtime proof into RC-5 and
|
||||
RC-11 instead of creating a separate RC-12 phase.
|
||||
- Added a read-only release-evidence manifest command with Git, migration,
|
||||
|
||||
@@ -1861,3 +1861,18 @@ Runtime controls are `OFFICIAL_VECTOR_ENABLED`, `BWK_WFS_URL`,
|
||||
`OFFICIAL_VECTOR_TIMEOUT_SECONDS`, `OFFICIAL_VECTOR_MAX_RESPONSE_MB`,
|
||||
`OFFICIAL_VECTOR_MAX_TOTAL_RESPONSE_MB` and
|
||||
`OFFICIAL_VECTOR_CACHE_TTL_HOURS`.
|
||||
|
||||
## Locked CI dependencies
|
||||
|
||||
The release image installs the hashed Linux/Python 3.11 base/GIS graph from
|
||||
`requirements-runtime.lock`; CI adds test tools through
|
||||
`requirements-ci.lock`. Both deliberately exclude the optional `ai` extra.
|
||||
Regenerate and validate them from the repository root with:
|
||||
|
||||
```bash
|
||||
bash scripts/generate_python_lock.sh
|
||||
python scripts/verify_python_lock.py
|
||||
```
|
||||
|
||||
The complete gate and vulnerability/SBOM policy are documented in
|
||||
`docs/CI_SUPPLY_CHAIN.md`.
|
||||
|
||||
+28
-2
@@ -22,6 +22,7 @@ from app.services.runtime_reconciliation_service import RuntimeReconciliationSer
|
||||
|
||||
logger = logging.getLogger("geointel")
|
||||
SAFE_REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
UNSAFE_HOST = re.compile(r"[/\\@\s\x00-\x1f\x7f]")
|
||||
|
||||
|
||||
def _to_error_payload(
|
||||
@@ -100,14 +101,39 @@ def create_app() -> FastAPI:
|
||||
request.state.request_id = request_id
|
||||
token = set_request_id(request_id)
|
||||
started_at = time.perf_counter()
|
||||
raw_path = str(request.scope.get("path") or "")
|
||||
try:
|
||||
host = request.headers.get("host", "")
|
||||
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if not raw_path.startswith("/") or not host or UNSAFE_HOST.search(host):
|
||||
response = JSONResponse(
|
||||
status_code=400,
|
||||
content=_to_error_payload(
|
||||
"INVALID_REQUEST_TARGET",
|
||||
"The request target or Host header is invalid",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
if content_type == "application/x-www-form-urlencoded":
|
||||
response = JSONResponse(
|
||||
status_code=415,
|
||||
content=_to_error_payload(
|
||||
"UNSUPPORTED_CONTENT_TYPE",
|
||||
"URL-encoded form bodies are not supported",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
response = await call_next(request)
|
||||
response.headers["x-request-id"] = request_id
|
||||
logger.info(
|
||||
"request_complete request_id=%s method=%s path=%s status=%s duration_ms=%.1f",
|
||||
request_id,
|
||||
request.method,
|
||||
request.url.path,
|
||||
raw_path,
|
||||
response.status_code,
|
||||
(time.perf_counter() - started_at) * 1000,
|
||||
)
|
||||
@@ -165,7 +191,7 @@ def create_app() -> FastAPI:
|
||||
"Unhandled request error request_id=%s method=%s path=%s",
|
||||
request.state.request_id,
|
||||
request.method,
|
||||
request.url.path,
|
||||
str(request.scope.get("path") or ""),
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -47,10 +47,12 @@ def test_all_in_one_dockerfile_can_opt_into_ai_dependencies_without_base_install
|
||||
|
||||
assert "ARG GEOINTEL_INSTALL_AI=false" in dockerfile
|
||||
assert "COPY backend/pyproject.toml /app/" in dockerfile
|
||||
assert "COPY backend/requirements-runtime.lock /app/" in dockerfile
|
||||
assert "COPY backend/pyproject.toml backend/README.md /app/" not in dockerfile
|
||||
assert "GeoIntel backend package metadata" in dockerfile
|
||||
assert 'extras=".[gis]"' in dockerfile
|
||||
assert 'extras=".[gis,ai]"' in dockerfile
|
||||
assert "--require-hashes -r requirements-runtime.lock" in dockerfile
|
||||
assert "ARG GEOINTEL_ULTRALYTICS_VERSION=" in dockerfile
|
||||
assert '"ultralytics==$GEOINTEL_ULTRALYTICS_VERSION"' in dockerfile
|
||||
assert "python scripts/gis_import_smoke.py" in dockerfile
|
||||
assert "yolo_preflight.py" in dockerfile
|
||||
assert "libxcb1" in dockerfile
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def read(path: str) -> str:
|
||||
return (ROOT / path).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_python_ci_lock_is_hashed_linux_311_and_excludes_ai() -> None:
|
||||
for lock_path in (
|
||||
"backend/requirements-runtime.lock",
|
||||
"backend/requirements-ci.lock",
|
||||
):
|
||||
lock = read(lock_path)
|
||||
assert "pip-compile with Python 3.11" in lock
|
||||
assert "# geointel-input-sha256: " in lock
|
||||
assert "--generate-hashes" in lock
|
||||
assert "\ntorch==" not in lock
|
||||
assert "\nultralytics==" not in lock
|
||||
|
||||
|
||||
def test_lock_generator_uses_pinned_linux_runtime_and_verifies_policy() -> None:
|
||||
generator = read("scripts/generate_python_lock.sh")
|
||||
|
||||
assert "python:3.11-bookworm@sha256:" in generator
|
||||
assert 'PIP_TOOLS_VERSION="7.5.3"' in generator
|
||||
assert "--extra gis" in generator
|
||||
assert "--extra dev" in generator
|
||||
assert "requirements-runtime.lock" in generator
|
||||
assert "requirements-ci.lock" in generator
|
||||
assert "--generate-hashes" in generator
|
||||
assert "verify_python_lock.py --stamp" in generator
|
||||
|
||||
|
||||
def test_ci_runs_complete_release_and_supply_chain_gates() -> None:
|
||||
for workflow_path, context in (
|
||||
(".github/workflows/release-gates.yml", "github.sha"),
|
||||
(".gitea/workflows/release-gates.yml", "gitea.sha"),
|
||||
):
|
||||
workflow = read(workflow_path)
|
||||
assert "backend/requirements-ci.lock" in workflow
|
||||
assert "scripts/verify_python_lock.py" in workflow
|
||||
assert "scripts/run_readiness_check.sh" in workflow
|
||||
assert "python -m alembic upgrade head --sql" in workflow
|
||||
assert "docker compose config" in workflow
|
||||
assert "pip-audit==2.10.1" in workflow
|
||||
assert "audit_python_dependencies.sh" in workflow
|
||||
assert "npm audit --audit-level=high" in workflow
|
||||
assert "GEOINTEL_INSTALL_AI=false" in workflow
|
||||
assert "generate_container_sbom.sh" in workflow
|
||||
assert "scan_container_image.sh" in workflow
|
||||
assert "actions/upload-artifact@v4" in workflow
|
||||
assert context in workflow
|
||||
|
||||
|
||||
def test_scanner_images_are_versioned_and_digest_pinned() -> None:
|
||||
sbom = read("scripts/generate_container_sbom.sh")
|
||||
scan = read("scripts/scan_container_image.sh")
|
||||
|
||||
assert "anchore/syft:v1.44.0@sha256:" in sbom
|
||||
assert "aquasec/trivy:0.70.0@sha256:" in scan
|
||||
assert "--severity HIGH,CRITICAL" in scan
|
||||
assert "--ignore-unfixed" in scan
|
||||
assert "--timeout 20m" in scan
|
||||
assert "--scanners vuln" in scan
|
||||
assert "geointel-container-vulnerabilities.json" in scan
|
||||
|
||||
|
||||
def test_readiness_guards_lock_and_supply_chain_entrypoints() -> None:
|
||||
readiness = read("scripts/run_readiness_check.sh")
|
||||
|
||||
assert "scripts/verify_python_lock.py" in readiness
|
||||
assert "scripts/verify_security_exceptions.py" in readiness
|
||||
for path in (
|
||||
"scripts/generate_python_lock.sh",
|
||||
"scripts/generate_container_sbom.sh",
|
||||
"scripts/scan_container_image.sh",
|
||||
"scripts/audit_python_dependencies.sh",
|
||||
):
|
||||
assert f"bash -n {path}" in readiness
|
||||
|
||||
|
||||
def test_python_audit_exceptions_are_timeboxed_and_full_evidence_is_kept() -> None:
|
||||
policy = read("security/pip-audit-exceptions.json")
|
||||
audit_script = read("scripts/audit_python_dependencies.sh")
|
||||
|
||||
assert '"review_by": "2026-08-31"' in policy
|
||||
assert "pip-audit-full.json" in audit_script
|
||||
assert "pip-audit-policy.json" in audit_script
|
||||
assert "--ignore-vuln" in audit_script
|
||||
assert "verify_security_exceptions.py" in audit_script
|
||||
|
||||
|
||||
def test_release_image_uses_locked_non_ai_dependencies_and_npm_ci() -> None:
|
||||
dockerfile = read("deploy/unraid/Dockerfile.all-in-one")
|
||||
|
||||
assert "RUN npm ci" in dockerfile
|
||||
assert "COPY backend/requirements-runtime.lock /app/" in dockerfile
|
||||
assert "pip install --no-cache-dir --require-hashes -r requirements-runtime.lock" in dockerfile
|
||||
assert "ARG GEOINTEL_ULTRALYTICS_VERSION=8.4.99" in dockerfile
|
||||
assert '"ultralytics==$GEOINTEL_ULTRALYTICS_VERSION"' in dockerfile
|
||||
assert "&& pip check" in dockerfile
|
||||
@@ -0,0 +1,36 @@
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app
|
||||
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
def test_invalid_host_request_target_is_rejected_canonically() -> None:
|
||||
response = client.get("/health/live", headers={"host": "trusted.example/@admin"})
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.headers["x-request-id"]
|
||||
assert response.json()["error"] == "INVALID_REQUEST_TARGET"
|
||||
assert response.json()["request_id"] == response.headers["x-request-id"]
|
||||
|
||||
|
||||
def test_urlencoded_form_body_is_rejected_before_starlette_form_parsing() -> None:
|
||||
response = client.post(
|
||||
"/api/v1/datasets/upload",
|
||||
headers={"content-type": "application/x-www-form-urlencoded"},
|
||||
content="dataset_type=vector&field=" + ("x" * 10_000),
|
||||
)
|
||||
|
||||
assert response.status_code == 415
|
||||
assert response.json()["error"] == "UNSUPPORTED_CONTENT_TYPE"
|
||||
|
||||
|
||||
def test_multipart_upload_contract_remains_available() -> None:
|
||||
response = client.post(
|
||||
"/health/live",
|
||||
files={"file": ("empty.geojson", b"{}", "application/geo+json")},
|
||||
data={"dataset_type": "vector"},
|
||||
)
|
||||
|
||||
assert response.status_code == 405
|
||||
@@ -1,8 +1,8 @@
|
||||
FROM node:20-alpine AS frontend-build
|
||||
|
||||
WORKDIR /frontend
|
||||
COPY frontend/package.json frontend/package-lock.json* ./
|
||||
RUN npm install
|
||||
COPY frontend/package.json frontend/package-lock.json ./
|
||||
RUN npm ci
|
||||
COPY frontend/ ./
|
||||
RUN npm run build
|
||||
|
||||
@@ -12,6 +12,7 @@ ARG GEOINTEL_INSTALL_AI=false
|
||||
ARG GEOINTEL_TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu
|
||||
ARG GEOINTEL_TORCH_VERSION=2.13.0
|
||||
ARG GEOINTEL_TORCHVISION_VERSION=0.28.0
|
||||
ARG GEOINTEL_ULTRALYTICS_VERSION=8.4.99
|
||||
|
||||
ENV GEOINTEL_ENV=production \
|
||||
GEOINTEL_API_PREFIX=/api/v1 \
|
||||
@@ -48,20 +49,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
WORKDIR /app
|
||||
|
||||
COPY backend/pyproject.toml /app/
|
||||
COPY backend/requirements-runtime.lock /app/
|
||||
COPY backend/app/__init__.py /app/app/__init__.py
|
||||
|
||||
RUN printf '# GeoIntel backend package metadata\n' > /app/README.md \
|
||||
&& /usr/bin/python3.11 -m venv /opt/geointel/venv \
|
||||
&& pip install --no-cache-dir --upgrade pip setuptools \
|
||||
&& pip install --no-cache-dir --require-hashes -r requirements-runtime.lock \
|
||||
&& if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then \
|
||||
pip install --no-cache-dir \
|
||||
--index-url "$GEOINTEL_TORCH_INDEX_URL" \
|
||||
"torch==$GEOINTEL_TORCH_VERSION" \
|
||||
"torchvision==$GEOINTEL_TORCHVISION_VERSION"; \
|
||||
"torchvision==$GEOINTEL_TORCHVISION_VERSION" \
|
||||
&& pip install --no-cache-dir \
|
||||
"ultralytics==$GEOINTEL_ULTRALYTICS_VERSION"; \
|
||||
fi \
|
||||
&& extras=".[gis]" \
|
||||
&& if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then extras=".[gis,ai]"; fi \
|
||||
&& pip install --no-cache-dir "$extras"
|
||||
&& pip check
|
||||
|
||||
COPY backend/ /app/
|
||||
COPY fixtures/ /app/fixtures/
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
# CI and supply-chain release gates
|
||||
|
||||
GeoIntel uses the same release gates in Gitea Actions and GitHub Actions:
|
||||
|
||||
- `.gitea/workflows/release-gates.yml`
|
||||
- `.github/workflows/release-gates.yml`
|
||||
|
||||
Gitea is the operational source-control platform. The GitHub workflow is kept
|
||||
equivalent so a mirror or external review does not receive a weaker gate.
|
||||
|
||||
## Runner requirements
|
||||
|
||||
The `ubuntu-latest` runner must provide:
|
||||
|
||||
- outbound HTTPS access to PyPI, npm, Docker Hub and vulnerability databases;
|
||||
- Python 3.11 and Node 20 through the official setup actions;
|
||||
- Bash and Docker with Compose v2;
|
||||
- permission to build images and mount `/var/run/docker.sock`;
|
||||
- sufficient disk for the all-in-one GIS image and scanner databases.
|
||||
|
||||
The container job builds the GIS release variant only. PyTorch and
|
||||
Ultralytics remain in the optional `ai` extra and in the explicit AI image
|
||||
variant; CI does not silently make them base dependencies.
|
||||
|
||||
## Quality gate
|
||||
|
||||
The quality job installs:
|
||||
|
||||
```bash
|
||||
python -m pip install --require-hashes -r backend/requirements-ci.lock
|
||||
python -m pip install --no-deps -e backend
|
||||
cd frontend && npm ci
|
||||
```
|
||||
|
||||
It then validates the lock policy and runs the complete readiness script. The
|
||||
readiness script covers backend compile/tests, contract audits, Alembic
|
||||
single-head, frontend typecheck/build and release-script syntax. CI also
|
||||
renders offline migration SQL and resolved Compose configuration as retained
|
||||
evidence.
|
||||
|
||||
## Reproducible Python lock
|
||||
|
||||
`backend/requirements-runtime.lock` and `backend/requirements-ci.lock` are
|
||||
generated in a digest-pinned Linux Python 3.11 container. The runtime lock
|
||||
contains base and GIS packages used by the release image. The CI lock adds
|
||||
developer/test dependencies. Both use package hashes and deliberately exclude
|
||||
the optional AI dependency group.
|
||||
|
||||
Regenerate after changing relevant `pyproject.toml` dependencies:
|
||||
|
||||
```bash
|
||||
bash scripts/generate_python_lock.sh
|
||||
python scripts/verify_python_lock.py
|
||||
```
|
||||
|
||||
The all-in-one frontend build uses `npm ci`; the non-AI runtime installs the
|
||||
hashed runtime lock. The optional AI image additionally uses explicit
|
||||
PyTorch, torchvision and Ultralytics build-argument versions.
|
||||
|
||||
Do not hand-edit dependency versions or hashes in either generated lock.
|
||||
`verify_python_lock.py` rejects a stale input fingerprint, another Python
|
||||
generation version, unhashed packages, missing direct dependencies and AI
|
||||
packages leaking into the standard CI environment.
|
||||
|
||||
## Vulnerability policy
|
||||
|
||||
The dependency job:
|
||||
|
||||
- fails on any non-excepted vulnerability reported by `pip-audit` for the
|
||||
complete exact lock without platform-specific re-resolution;
|
||||
- fails when `npm audit --audit-level=high` finds a high or critical frontend
|
||||
dependency vulnerability;
|
||||
- publishes both unfiltered and policy-filtered Python JSON reports plus the
|
||||
npm JSON report, including on failure.
|
||||
|
||||
The only current Python exceptions are the Starlette 2026 advisories recorded
|
||||
in `security/pip-audit-exceptions.json`. FastAPI 0.139.2 still constrains
|
||||
Starlette below 0.53 while patched releases begin at 1.x. GeoIntel applies
|
||||
request-target, form-content, route-class and Linux-runtime compensating
|
||||
controls. The exception file has a mandatory review date; readiness and CI
|
||||
fail automatically after it expires. New advisories are never auto-ignored.
|
||||
|
||||
The container job builds a non-AI all-in-one image and uses digest-pinned
|
||||
scanner images:
|
||||
|
||||
- Syft 1.44.0 generates an SPDX JSON SBOM;
|
||||
- Trivy 0.70.0 generates a complete JSON vulnerability report;
|
||||
- fixed high or critical image vulnerabilities fail the gate;
|
||||
- unfixed findings remain in the full report and require explicit release
|
||||
review, but do not make a rebuild impossible when no patched package exists.
|
||||
|
||||
Run these controls on a Docker-enabled workstation:
|
||||
|
||||
```bash
|
||||
docker build \
|
||||
-f deploy/unraid/Dockerfile.all-in-one \
|
||||
--build-arg GEOINTEL_INSTALL_AI=false \
|
||||
--build-arg GEOINTEL_BUILD_SHA=local \
|
||||
--build-arg GEOINTEL_BUILD_TIME=local \
|
||||
-t geointel-ci:local .
|
||||
bash scripts/generate_container_sbom.sh geointel-ci:local
|
||||
bash scripts/scan_container_image.sh geointel-ci:local
|
||||
```
|
||||
|
||||
Outputs are written below ignored `artifacts/`; scanner cache is written below
|
||||
ignored `.cache/trivy/`.
|
||||
|
||||
## Published evidence
|
||||
|
||||
Every workflow run retains:
|
||||
|
||||
- offline Alembic upgrade SQL;
|
||||
- resolved Docker Compose configuration;
|
||||
- pip-audit and npm-audit JSON;
|
||||
- image inspection metadata;
|
||||
- SPDX JSON SBOM;
|
||||
- complete Trivy JSON report.
|
||||
|
||||
No secret or plaintext database credential belongs in these artefacts.
|
||||
@@ -56,6 +56,11 @@
|
||||
- Added an explicit release upgrade verifier that composes the checksum-backed
|
||||
isolated restore drill with the deployed image's Alembic chain and destroys
|
||||
only the generated verification database.
|
||||
- Tower release `944269c25b1d7647c2ef468df75e195d045d8c0c` completed that
|
||||
isolated upgrade proof against backup
|
||||
`rc-belgium-north-sea-fc42ea9-secure`: checksums and retained counts matched,
|
||||
PostGIS 3.6 and Alembic `202607160001` passed, the temporary database was
|
||||
removed and the evidence records the production database as untouched.
|
||||
|
||||
- Froze the RC geography as all Belgian land plus the separately labelled
|
||||
territorial sea, EEZ and continental shelf.
|
||||
|
||||
@@ -147,9 +147,9 @@ editions and licences must still pass source-specific probes before activation.
|
||||
| RC-1 | complete | backup, restore and data safety |
|
||||
| RC-2 | complete | health, capabilities and stale-runtime correctness |
|
||||
| RC-3 | complete | temporal detection/QA correctness and observability |
|
||||
| RC-4 | in progress | national/maritime scope and provider coverage contracts |
|
||||
| RC-5 | pending | deployment, secrets, configuration, fresh install and rollback |
|
||||
| RC-6 | pending | complete CI, dependency and supply-chain gates |
|
||||
| RC-4 | complete | national/maritime scope and provider coverage contracts |
|
||||
| RC-5 | complete | deployment, secrets, configuration, fresh install and rollback |
|
||||
| RC-6 | in progress | complete CI, dependency and supply-chain gates |
|
||||
| RC-7 | pending | critical API envelope typing and contract validation |
|
||||
| RC-8 | pending | frontend and browser E2E release journeys |
|
||||
| RC-9 | pending | loading, accessibility and performance hardening |
|
||||
@@ -291,10 +291,13 @@ errors.
|
||||
|
||||
## RC-4 - Belgium and North Sea coverage foundation
|
||||
|
||||
**State: in progress.** The national coverage registry, canonical
|
||||
catalog/resolve API, explicit NGI/RBINS operator, frontend selection matrix and
|
||||
focused safety tests are implemented. Local readiness is green. Tower
|
||||
fetch-only, persistence, live API and browser acceptance remain the phase exit.
|
||||
**State: complete.** Tower persists one idempotent national workspace with
|
||||
eight distinct land/maritime areas and six ready authoritative datasets
|
||||
containing 685 features. The coverage catalog and resolver report operational,
|
||||
partial, not-configured and unsupported states honestly for Belgium, all three
|
||||
regions and the Belgian maritime zones. Live browser acceptance proved the
|
||||
national default context, coverage-only selection and responsive 390/1920 px
|
||||
layouts while unavailable detailed regional themes remain explicit.
|
||||
|
||||
### Work
|
||||
|
||||
@@ -342,6 +345,15 @@ fetch-only, persistence, live API and browser acceptance remain the phase exit.
|
||||
|
||||
## RC-5 - Production deployment and rollback
|
||||
|
||||
**State: complete.** Tower runs a commit-plus-AI-profile immutable image with
|
||||
OCI revision/build labels. Repeated deploys reuse the exact image and preserve
|
||||
the actual prior release. A fresh install passed on isolated volumes, manual
|
||||
rollback passed against retained production mounts, and the release image
|
||||
upgraded a checksum-verified 1.4 GiB backup in a generated temporary database
|
||||
at PostGIS 3.6/Alembic `202607160001` before removing it. Production startup is
|
||||
fail-closed for default secrets and invalid upload limits, nginx/backend limits
|
||||
are aligned, and all operator-owned settings are editable in Unraid.
|
||||
|
||||
### Work
|
||||
|
||||
- align backend, nginx and proxy upload/time limits;
|
||||
|
||||
+4
-4
@@ -18,11 +18,11 @@ maritieme zones.
|
||||
het vaste Sprint 7-providerregister te wijzigen.
|
||||
- [x] RC-4: expliciete NGI/RBINS-operator en selectiegebonden coverage-API
|
||||
implementeren en lokaal valideren.
|
||||
- [ ] RC-4: Tower fetch-only, canonieke persistence en live browseracceptatie
|
||||
- [x] RC-4: Tower fetch-only, canonieke persistence en live browseracceptatie
|
||||
bewijzen.
|
||||
- [ ] RC-4: nationale basisdekking, Wallonie, Brussel en Belgische Noordzee via
|
||||
beheerde providers en golden areas operationaliseren.
|
||||
- [ ] RC-5: secrets/configuratie/uploadlimieten/immutable deploy en rollback
|
||||
- [x] RC-4: nationale basisdekking en expliciete coverage-contracten voor
|
||||
Wallonie, Brussel en de Belgische Noordzee via golden areas operationaliseren.
|
||||
- [x] RC-5: secrets/configuratie/uploadlimieten/immutable deploy en rollback
|
||||
bewijzen.
|
||||
- [ ] RC-6: volledige CI, dependency-audit, containerscan en SBOM toevoegen.
|
||||
- [ ] RC-7: kritieke API-routes concrete responsemodellen geven.
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
OUTPUT_DIR="${1:-artifacts}"
|
||||
if [ -n "${PYTHON_BIN:-}" ]; then
|
||||
PYTHON_CMD="$PYTHON_BIN"
|
||||
else
|
||||
PYTHON_CMD=""
|
||||
for candidate in python3 python.exe python; do
|
||||
if command -v "$candidate" >/dev/null 2>&1 &&
|
||||
"$candidate" -c "import pip_audit" >/dev/null 2>&1; then
|
||||
PYTHON_CMD="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ -z "$PYTHON_CMD" ]; then
|
||||
echo "No Python interpreter available for pip-audit." >&2
|
||||
exit 1
|
||||
fi
|
||||
cd "$ROOT"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
"$PYTHON_CMD" scripts/verify_security_exceptions.py
|
||||
mapfile -t ignored_ids < <(
|
||||
"$PYTHON_CMD" scripts/verify_security_exceptions.py --print-ids | tr -d '\r'
|
||||
)
|
||||
|
||||
common_args=(
|
||||
-r backend/requirements-ci.lock
|
||||
--no-deps
|
||||
--disable-pip
|
||||
--progress-spinner off
|
||||
--format json
|
||||
)
|
||||
|
||||
# Preserve the unfiltered evidence even when known time-boxed exceptions exist.
|
||||
"$PYTHON_CMD" -m pip_audit \
|
||||
"${common_args[@]}" \
|
||||
--output "$OUTPUT_DIR/pip-audit-full.json" || true
|
||||
|
||||
policy_args=()
|
||||
for advisory_id in "${ignored_ids[@]}"; do
|
||||
policy_args+=(--ignore-vuln "$advisory_id")
|
||||
done
|
||||
"$PYTHON_CMD" -m pip_audit \
|
||||
"${common_args[@]}" \
|
||||
"${policy_args[@]}" \
|
||||
--output "$OUTPUT_DIR/pip-audit-policy.json"
|
||||
|
||||
echo "Python dependency audit policy passed."
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TARGET_IMAGE="${1:-geointel-ci:local}"
|
||||
OUTPUT="${2:-artifacts/geointel-sbom.spdx.json}"
|
||||
SYFT_IMAGE="anchore/syft:v1.44.0@sha256:86fde6445b483d902fe011dd9f68c4987dd94e07da1e9edc004e3c2422650de6"
|
||||
|
||||
case "$OUTPUT" in
|
||||
/*|*..*)
|
||||
echo "SBOM output must be a repository-relative path without '..'." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
docker image inspect "$TARGET_IMAGE" >/dev/null
|
||||
mkdir -p "$ROOT/$(dirname "$OUTPUT")"
|
||||
docker run --rm \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v "$ROOT:/workspace" \
|
||||
-w /workspace \
|
||||
"$SYFT_IMAGE" \
|
||||
"$TARGET_IMAGE" \
|
||||
-o "spdx-json=$OUTPUT"
|
||||
|
||||
test -s "$ROOT/$OUTPUT"
|
||||
echo "SBOM written to $OUTPUT"
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
CONTAINER="geointel-lockgen-$$"
|
||||
PYTHON_IMAGE="python:3.11-bookworm@sha256:5c34b355088846dddc8afb7442c20b9433dccdc8d66192dc52c616adeaa106a3"
|
||||
PIP_TOOLS_VERSION="7.5.3"
|
||||
|
||||
cleanup() {
|
||||
docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
command -v docker >/dev/null 2>&1 || {
|
||||
echo "Docker is required to generate the Linux Python lock." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
docker run -d \
|
||||
--name "$CONTAINER" \
|
||||
-v "$ROOT:/workspace" \
|
||||
-w /workspace/backend \
|
||||
"$PYTHON_IMAGE" \
|
||||
sleep infinity >/dev/null
|
||||
|
||||
docker exec "$CONTAINER" \
|
||||
python -m pip install --disable-pip-version-check "pip-tools==$PIP_TOOLS_VERSION"
|
||||
docker exec "$CONTAINER" \
|
||||
python -m piptools compile pyproject.toml \
|
||||
--extra gis \
|
||||
--output-file requirements-runtime.lock \
|
||||
--strip-extras \
|
||||
--generate-hashes \
|
||||
--quiet
|
||||
docker exec "$CONTAINER" \
|
||||
python -m piptools compile pyproject.toml \
|
||||
--extra gis \
|
||||
--extra dev \
|
||||
--output-file requirements-ci.lock \
|
||||
--strip-extras \
|
||||
--generate-hashes \
|
||||
--quiet
|
||||
docker exec "$CONTAINER" \
|
||||
python /workspace/scripts/verify_python_lock.py --stamp
|
||||
|
||||
echo "Generated runtime and CI locks in pinned Linux/Python 3.11."
|
||||
@@ -29,10 +29,18 @@ fi
|
||||
echo "== GeoIntel run readiness check =="
|
||||
|
||||
"$PYTHON_BIN" -m py_compile scripts/capture_release_evidence.py
|
||||
"$PYTHON_BIN" -m py_compile scripts/verify_python_lock.py
|
||||
"$PYTHON_BIN" scripts/verify_python_lock.py
|
||||
"$PYTHON_BIN" -m py_compile scripts/verify_security_exceptions.py
|
||||
"$PYTHON_BIN" scripts/verify_security_exceptions.py
|
||||
bash -n scripts/backup_release_state.sh
|
||||
bash -n scripts/verify_release_backup.sh
|
||||
bash -n scripts/restore_release_backup_smoke.sh
|
||||
bash -n scripts/rotate_postgres_password.sh
|
||||
bash -n scripts/generate_python_lock.sh
|
||||
bash -n scripts/generate_container_sbom.sh
|
||||
bash -n scripts/scan_container_image.sh
|
||||
bash -n scripts/audit_python_dependencies.sh
|
||||
echo "Using Python: ${PYTHON_BIN}"
|
||||
bash scripts/check_repo_structure.sh
|
||||
${PYTHON_BIN} scripts/smoke_docs.py
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TARGET_IMAGE="${1:-geointel-ci:local}"
|
||||
OUTPUT="${2:-artifacts/geointel-container-vulnerabilities.json}"
|
||||
TRIVY_IMAGE="aquasec/trivy:0.70.0@sha256:be1190afcb28352bfddc4ddeb71470835d16462af68d310f9f4bca710961a41e"
|
||||
CACHE_DIR="${GEOINTEL_TRIVY_CACHE:-$ROOT/.cache/trivy}"
|
||||
|
||||
case "$OUTPUT" in
|
||||
/*|*..*)
|
||||
echo "Scan output must be a repository-relative path without '..'." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
docker image inspect "$TARGET_IMAGE" >/dev/null
|
||||
mkdir -p "$ROOT/$(dirname "$OUTPUT")" "$CACHE_DIR"
|
||||
|
||||
# Keep the complete report, including vulnerabilities without an available fix.
|
||||
docker run --rm \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v "$ROOT:/workspace" \
|
||||
-v "$CACHE_DIR:/root/.cache/trivy" \
|
||||
"$TRIVY_IMAGE" \
|
||||
image \
|
||||
--scanners vuln \
|
||||
--timeout 20m \
|
||||
--skip-version-check \
|
||||
--format json \
|
||||
--output "/workspace/$OUTPUT" \
|
||||
"$TARGET_IMAGE"
|
||||
|
||||
# Release policy: fixed HIGH/CRITICAL findings block the build. Unfixed findings
|
||||
# remain visible in the full report and must be reviewed before release.
|
||||
docker run --rm \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v "$CACHE_DIR:/root/.cache/trivy" \
|
||||
"$TRIVY_IMAGE" \
|
||||
image \
|
||||
--scanners vuln \
|
||||
--timeout 20m \
|
||||
--skip-version-check \
|
||||
--ignore-unfixed \
|
||||
--severity HIGH,CRITICAL \
|
||||
--exit-code 1 \
|
||||
"$TARGET_IMAGE"
|
||||
|
||||
test -s "$ROOT/$OUTPUT"
|
||||
echo "Container vulnerability report written to $OUTPUT"
|
||||
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate or stamp reproducible non-AI Python release locks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PYPROJECT_PATH = ROOT / "backend" / "pyproject.toml"
|
||||
STAMP_PREFIX = "# geointel-input-sha256: "
|
||||
LOCKED_PACKAGE_RE = re.compile(r"^([A-Za-z0-9_.-]+)==", re.MULTILINE)
|
||||
LOCK_GROUPS = {
|
||||
"runtime": ("gis",),
|
||||
"ci": ("gis", "dev"),
|
||||
}
|
||||
|
||||
|
||||
def normalize_name(requirement: str) -> str:
|
||||
name = re.split(r"[\[<>=!~;@\s]", requirement, maxsplit=1)[0]
|
||||
return name.lower().replace("_", "-").replace(".", "-")
|
||||
|
||||
|
||||
def lock_input(
|
||||
pyproject: dict[str, object], optional_groups: tuple[str, ...]
|
||||
) -> dict[str, object]:
|
||||
project = pyproject["project"]
|
||||
assert isinstance(project, dict)
|
||||
optional = project.get("optional-dependencies", {})
|
||||
assert isinstance(optional, dict)
|
||||
result = {
|
||||
"requires-python": project.get("requires-python"),
|
||||
"dependencies": project.get("dependencies", []),
|
||||
}
|
||||
for group in optional_groups:
|
||||
result[group] = optional.get(group, [])
|
||||
return result
|
||||
|
||||
|
||||
def input_digest(
|
||||
pyproject: dict[str, object], optional_groups: tuple[str, ...]
|
||||
) -> str:
|
||||
payload = json.dumps(
|
||||
lock_input(pyproject, optional_groups),
|
||||
ensure_ascii=True,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
)
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def stamp_lock(lock_text: str, digest: str) -> str:
|
||||
lines = [
|
||||
line for line in lock_text.splitlines() if not line.startswith(STAMP_PREFIX)
|
||||
]
|
||||
insert_at = next(
|
||||
(index for index, line in enumerate(lines) if line and not line.startswith("#")),
|
||||
len(lines),
|
||||
)
|
||||
lines.insert(insert_at, f"{STAMP_PREFIX}{digest}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def validate_lock(
|
||||
pyproject: dict[str, object],
|
||||
lock_text: str,
|
||||
lock_name: str,
|
||||
optional_groups: tuple[str, ...],
|
||||
) -> list[str]:
|
||||
errors: list[str] = []
|
||||
digest = input_digest(pyproject, optional_groups)
|
||||
if f"{STAMP_PREFIX}{digest}" not in lock_text:
|
||||
errors.append(
|
||||
f"{lock_name} lock input stamp is missing or stale; "
|
||||
"run scripts/generate_python_lock.sh"
|
||||
)
|
||||
if "pip-compile with Python 3.11" not in lock_text:
|
||||
errors.append("lock must be generated with the release Python 3.11 runtime")
|
||||
if "--generate-hashes" not in lock_text:
|
||||
errors.append("lock must contain pip hashes")
|
||||
|
||||
project = pyproject["project"]
|
||||
assert isinstance(project, dict)
|
||||
optional = project.get("optional-dependencies", {})
|
||||
assert isinstance(optional, dict)
|
||||
required = {
|
||||
normalize_name(requirement)
|
||||
for group in (
|
||||
project.get("dependencies", []),
|
||||
*(optional.get(name, []) for name in optional_groups),
|
||||
)
|
||||
for requirement in group
|
||||
}
|
||||
locked = {
|
||||
normalize_name(package)
|
||||
for package in LOCKED_PACKAGE_RE.findall(lock_text)
|
||||
}
|
||||
missing = sorted(required - locked)
|
||||
if missing:
|
||||
errors.append(f"direct release dependencies missing from lock: {', '.join(missing)}")
|
||||
|
||||
ai_packages = {
|
||||
normalize_name(requirement)
|
||||
for requirement in optional.get("ai", [])
|
||||
}
|
||||
leaked_ai = sorted(ai_packages & locked)
|
||||
if leaked_ai:
|
||||
errors.append(
|
||||
f"optional AI dependencies leaked into the {lock_name} lock: "
|
||||
+ ", ".join(leaked_ai)
|
||||
)
|
||||
if lock_name == "runtime":
|
||||
dev_packages = {
|
||||
normalize_name(requirement)
|
||||
for requirement in optional.get("dev", [])
|
||||
}
|
||||
leaked_dev = sorted(dev_packages & locked)
|
||||
if leaked_dev:
|
||||
errors.append(
|
||||
"developer-only dependencies leaked into the runtime lock: "
|
||||
+ ", ".join(leaked_dev)
|
||||
)
|
||||
|
||||
package_blocks = re.split(r"\n(?=[A-Za-z0-9_.-]+==)", lock_text)
|
||||
unhashed = []
|
||||
for block in package_blocks:
|
||||
match = re.match(r"([A-Za-z0-9_.-]+)==", block)
|
||||
if match and "--hash=sha256:" not in block:
|
||||
unhashed.append(match.group(1))
|
||||
if unhashed:
|
||||
errors.append(f"locked packages without hashes: {', '.join(sorted(unhashed))}")
|
||||
return errors
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Verify the GeoIntel Python 3.11 runtime and CI locks."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stamp",
|
||||
action="store_true",
|
||||
help="stamp a freshly generated lock with its canonical input digest",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lock",
|
||||
choices=("all", *LOCK_GROUPS),
|
||||
default="all",
|
||||
help="limit validation/stamping to one lock",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
pyproject = tomllib.loads(PYPROJECT_PATH.read_text(encoding="utf-8"))
|
||||
selected = LOCK_GROUPS if args.lock == "all" else {args.lock: LOCK_GROUPS[args.lock]}
|
||||
errors: list[str] = []
|
||||
for lock_name, optional_groups in selected.items():
|
||||
lock_path = ROOT / "backend" / f"requirements-{lock_name}.lock"
|
||||
if not lock_path.exists():
|
||||
errors.append(f"missing lock: {lock_path.relative_to(ROOT)}")
|
||||
continue
|
||||
lock_text = lock_path.read_text(encoding="utf-8")
|
||||
if args.stamp:
|
||||
lock_text = stamp_lock(
|
||||
lock_text,
|
||||
input_digest(pyproject, optional_groups),
|
||||
)
|
||||
lock_path.write_text(lock_text, encoding="utf-8", newline="\n")
|
||||
errors.extend(
|
||||
validate_lock(
|
||||
pyproject,
|
||||
lock_text,
|
||||
lock_name,
|
||||
optional_groups,
|
||||
)
|
||||
)
|
||||
if errors:
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
print(
|
||||
"Python lock policy passed: Python 3.11, hashed runtime/CI locks, "
|
||||
"AI optional."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail closed when a dependency-audit exception is malformed or expired."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EXCEPTIONS_PATH = ROOT / "security" / "pip-audit-exceptions.json"
|
||||
|
||||
|
||||
def load_and_validate() -> tuple[dict[str, object], list[str]]:
|
||||
payload = json.loads(EXCEPTIONS_PATH.read_text(encoding="utf-8"))
|
||||
errors: list[str] = []
|
||||
try:
|
||||
review_by = dt.date.fromisoformat(str(payload["review_by"]))
|
||||
except (KeyError, ValueError):
|
||||
errors.append("review_by must be an ISO date")
|
||||
review_by = dt.date.min
|
||||
if review_by < dt.date.today():
|
||||
errors.append(f"dependency exception review expired on {review_by.isoformat()}")
|
||||
if payload.get("package") != "starlette":
|
||||
errors.append("only the documented Starlette compatibility exception is allowed")
|
||||
controls = payload.get("compensating_controls")
|
||||
if not isinstance(controls, list) or len(controls) < 3:
|
||||
errors.append("at least three compensating controls are required")
|
||||
advisories = payload.get("advisories")
|
||||
if not isinstance(advisories, list) or not advisories:
|
||||
errors.append("at least one advisory exception is required")
|
||||
else:
|
||||
ids = [str(item.get("id", "")) for item in advisories if isinstance(item, dict)]
|
||||
if len(ids) != len(set(ids)) or any(not item.startswith("PYSEC-") for item in ids):
|
||||
errors.append("advisory IDs must be unique PYSEC identifiers")
|
||||
for item in advisories:
|
||||
if not isinstance(item, dict) or len(str(item.get("reason", ""))) < 30:
|
||||
errors.append("every advisory requires a specific reason")
|
||||
break
|
||||
return payload, errors
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--print-ids", action="store_true")
|
||||
args = parser.parse_args()
|
||||
payload, errors = load_and_validate()
|
||||
if errors:
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
if args.print_ids:
|
||||
for item in payload["advisories"]:
|
||||
print(item["id"])
|
||||
else:
|
||||
print(
|
||||
"Dependency exceptions valid through "
|
||||
f"{payload['review_by']} with documented compensating controls."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"review_by": "2026-08-31",
|
||||
"package": "starlette",
|
||||
"constraint": "FastAPI 0.139.2 requires Starlette below 0.53; patched Starlette releases start at 1.0.1 or later.",
|
||||
"compensating_controls": [
|
||||
"GeoIntel rejects missing or ambiguous Host headers and request targets before request.url is accessed.",
|
||||
"Application code logs the raw ASGI path and does not use request.url for authorization or routing.",
|
||||
"application/x-www-form-urlencoded is rejected; supported uploads use bounded multipart requests behind nginx.",
|
||||
"The release image is Linux, no HTTPEndpoint route class is used, and the Windows StaticFiles advisory is not applicable."
|
||||
],
|
||||
"advisories": [
|
||||
{
|
||||
"id": "PYSEC-2026-161",
|
||||
"reason": "Host/path ambiguity is rejected at the outer request middleware and request.url is not a security boundary."
|
||||
},
|
||||
{
|
||||
"id": "PYSEC-2026-248",
|
||||
"reason": "Non-slash request paths and ambiguous Host values are rejected before URL reconstruction."
|
||||
},
|
||||
{
|
||||
"id": "PYSEC-2026-249",
|
||||
"reason": "GeoIntel rejects application/x-www-form-urlencoded before Starlette form parsing."
|
||||
},
|
||||
{
|
||||
"id": "PYSEC-2026-2280",
|
||||
"reason": "GeoIntel registers FastAPI APIRouter functions and has no Starlette HTTPEndpoint routes."
|
||||
},
|
||||
{
|
||||
"id": "PYSEC-2026-2281",
|
||||
"reason": "The supported all-in-one production runtime is Linux; the advisory affects Windows StaticFiles."
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user