Complete RC6 supply chain gates

scripts/audit_python_dependencies.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+OUTPUT_DIR="${1:-artifacts}"
+if [ -n "${PYTHON_BIN:-}" ]; then
+  PYTHON_CMD="$PYTHON_BIN"
+else
+  PYTHON_CMD=""
+  for candidate in python3 python.exe python; do
+    if command -v "$candidate" >/dev/null 2>&1 &&
+      "$candidate" -c "import pip_audit" >/dev/null 2>&1; then
+      PYTHON_CMD="$candidate"
+      break
+    fi
+  done
+fi
+if [ -z "$PYTHON_CMD" ]; then
+  echo "No Python interpreter available for pip-audit." >&2
+  exit 1
+fi
+cd "$ROOT"
+mkdir -p "$OUTPUT_DIR"
+
+"$PYTHON_CMD" scripts/verify_security_exceptions.py
+mapfile -t ignored_ids < <(
+  "$PYTHON_CMD" scripts/verify_security_exceptions.py --print-ids | tr -d '\r'
+)
+
+common_args=(
+  -r backend/requirements-ci.lock
+  --no-deps
+  --disable-pip
+  --progress-spinner off
+  --format json
+)
+
+# Preserve the unfiltered evidence even when known time-boxed exceptions exist.
+"$PYTHON_CMD" -m pip_audit \
+  "${common_args[@]}" \
+  --output "$OUTPUT_DIR/pip-audit-full.json" || true
+
+policy_args=()
+for advisory_id in "${ignored_ids[@]}"; do
+  policy_args+=(--ignore-vuln "$advisory_id")
+done
+"$PYTHON_CMD" -m pip_audit \
+  "${common_args[@]}" \
+  "${policy_args[@]}" \
+  --output "$OUTPUT_DIR/pip-audit-policy.json"
+
+echo "Python dependency audit policy passed."

scripts/generate_container_sbom.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+TARGET_IMAGE="${1:-geointel-ci:local}"
+OUTPUT="${2:-artifacts/geointel-sbom.spdx.json}"
+SYFT_IMAGE="anchore/syft:v1.44.0@sha256:86fde6445b483d902fe011dd9f68c4987dd94e07da1e9edc004e3c2422650de6"
+
+case "$OUTPUT" in
+  /*|*..*)
+    echo "SBOM output must be a repository-relative path without '..'." >&2
+    exit 1
+    ;;
+esac
+
+docker image inspect "$TARGET_IMAGE" >/dev/null
+mkdir -p "$ROOT/$(dirname "$OUTPUT")"
+docker run --rm \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v "$ROOT:/workspace" \
+  -w /workspace \
+  "$SYFT_IMAGE" \
+  "$TARGET_IMAGE" \
+  -o "spdx-json=$OUTPUT"
+
+test -s "$ROOT/$OUTPUT"
+echo "SBOM written to $OUTPUT"

scripts/generate_python_lock.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+CONTAINER="geointel-lockgen-$$"
+PYTHON_IMAGE="python:3.11-bookworm@sha256:5c34b355088846dddc8afb7442c20b9433dccdc8d66192dc52c616adeaa106a3"
+PIP_TOOLS_VERSION="7.5.3"
+
+cleanup() {
+  docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+command -v docker >/dev/null 2>&1 || {
+  echo "Docker is required to generate the Linux Python lock." >&2
+  exit 1
+}
+
+docker run -d \
+  --name "$CONTAINER" \
+  -v "$ROOT:/workspace" \
+  -w /workspace/backend \
+  "$PYTHON_IMAGE" \
+  sleep infinity >/dev/null
+
+docker exec "$CONTAINER" \
+  python -m pip install --disable-pip-version-check "pip-tools==$PIP_TOOLS_VERSION"
+docker exec "$CONTAINER" \
+  python -m piptools compile pyproject.toml \
+    --extra gis \
+    --output-file requirements-runtime.lock \
+    --strip-extras \
+    --generate-hashes \
+    --quiet
+docker exec "$CONTAINER" \
+  python -m piptools compile pyproject.toml \
+    --extra gis \
+    --extra dev \
+    --output-file requirements-ci.lock \
+    --strip-extras \
+    --generate-hashes \
+    --quiet
+docker exec "$CONTAINER" \
+  python /workspace/scripts/verify_python_lock.py --stamp
+
+echo "Generated runtime and CI locks in pinned Linux/Python 3.11."

scripts/run_readiness_check.sh

@@ -29,10 +29,18 @@ fi
 echo "== GeoIntel run readiness check =="
 
 "$PYTHON_BIN" -m py_compile scripts/capture_release_evidence.py
+"$PYTHON_BIN" -m py_compile scripts/verify_python_lock.py
+"$PYTHON_BIN" scripts/verify_python_lock.py
+"$PYTHON_BIN" -m py_compile scripts/verify_security_exceptions.py
+"$PYTHON_BIN" scripts/verify_security_exceptions.py
 bash -n scripts/backup_release_state.sh
 bash -n scripts/verify_release_backup.sh
 bash -n scripts/restore_release_backup_smoke.sh
 bash -n scripts/rotate_postgres_password.sh
+bash -n scripts/generate_python_lock.sh
+bash -n scripts/generate_container_sbom.sh
+bash -n scripts/scan_container_image.sh
+bash -n scripts/audit_python_dependencies.sh
 echo "Using Python: ${PYTHON_BIN}"
 bash scripts/check_repo_structure.sh
 ${PYTHON_BIN} scripts/smoke_docs.py

scripts/scan_container_image.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+TARGET_IMAGE="${1:-geointel-ci:local}"
+OUTPUT="${2:-artifacts/geointel-container-vulnerabilities.json}"
+TRIVY_IMAGE="aquasec/trivy:0.70.0@sha256:be1190afcb28352bfddc4ddeb71470835d16462af68d310f9f4bca710961a41e"
+CACHE_DIR="${GEOINTEL_TRIVY_CACHE:-$ROOT/.cache/trivy}"
+
+case "$OUTPUT" in
+  /*|*..*)
+    echo "Scan output must be a repository-relative path without '..'." >&2
+    exit 1
+    ;;
+esac
+
+docker image inspect "$TARGET_IMAGE" >/dev/null
+mkdir -p "$ROOT/$(dirname "$OUTPUT")" "$CACHE_DIR"
+
+# Keep the complete report, including vulnerabilities without an available fix.
+docker run --rm \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v "$ROOT:/workspace" \
+  -v "$CACHE_DIR:/root/.cache/trivy" \
+  "$TRIVY_IMAGE" \
+  image \
+  --scanners vuln \
+  --timeout 20m \
+  --skip-version-check \
+  --format json \
+  --output "/workspace/$OUTPUT" \
+  "$TARGET_IMAGE"
+
+# Release policy: fixed HIGH/CRITICAL findings block the build. Unfixed findings
+# remain visible in the full report and must be reviewed before release.
+docker run --rm \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v "$CACHE_DIR:/root/.cache/trivy" \
+  "$TRIVY_IMAGE" \
+  image \
+  --scanners vuln \
+  --timeout 20m \
+  --skip-version-check \
+  --ignore-unfixed \
+  --severity HIGH,CRITICAL \
+  --exit-code 1 \
+  "$TARGET_IMAGE"
+
+test -s "$ROOT/$OUTPUT"
+echo "Container vulnerability report written to $OUTPUT"

scripts/verify_python_lock.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+"""Validate or stamp reproducible non-AI Python release locks."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import sys
+import tomllib
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+PYPROJECT_PATH = ROOT / "backend" / "pyproject.toml"
+STAMP_PREFIX = "# geointel-input-sha256: "
+LOCKED_PACKAGE_RE = re.compile(r"^([A-Za-z0-9_.-]+)==", re.MULTILINE)
+LOCK_GROUPS = {
+    "runtime": ("gis",),
+    "ci": ("gis", "dev"),
+}
+
+
+def normalize_name(requirement: str) -> str:
+    name = re.split(r"[\[<>=!~;@\s]", requirement, maxsplit=1)[0]
+    return name.lower().replace("_", "-").replace(".", "-")
+
+
+def lock_input(
+    pyproject: dict[str, object], optional_groups: tuple[str, ...]
+) -> dict[str, object]:
+    project = pyproject["project"]
+    assert isinstance(project, dict)
+    optional = project.get("optional-dependencies", {})
+    assert isinstance(optional, dict)
+    result = {
+        "requires-python": project.get("requires-python"),
+        "dependencies": project.get("dependencies", []),
+    }
+    for group in optional_groups:
+        result[group] = optional.get(group, [])
+    return result
+
+
+def input_digest(
+    pyproject: dict[str, object], optional_groups: tuple[str, ...]
+) -> str:
+    payload = json.dumps(
+        lock_input(pyproject, optional_groups),
+        ensure_ascii=True,
+        separators=(",", ":"),
+        sort_keys=True,
+    )
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def stamp_lock(lock_text: str, digest: str) -> str:
+    lines = [
+        line for line in lock_text.splitlines() if not line.startswith(STAMP_PREFIX)
+    ]
+    insert_at = next(
+        (index for index, line in enumerate(lines) if line and not line.startswith("#")),
+        len(lines),
+    )
+    lines.insert(insert_at, f"{STAMP_PREFIX}{digest}")
+    return "\n".join(lines) + "\n"
+
+
+def validate_lock(
+    pyproject: dict[str, object],
+    lock_text: str,
+    lock_name: str,
+    optional_groups: tuple[str, ...],
+) -> list[str]:
+    errors: list[str] = []
+    digest = input_digest(pyproject, optional_groups)
+    if f"{STAMP_PREFIX}{digest}" not in lock_text:
+        errors.append(
+            f"{lock_name} lock input stamp is missing or stale; "
+            "run scripts/generate_python_lock.sh"
+        )
+    if "pip-compile with Python 3.11" not in lock_text:
+        errors.append("lock must be generated with the release Python 3.11 runtime")
+    if "--generate-hashes" not in lock_text:
+        errors.append("lock must contain pip hashes")
+
+    project = pyproject["project"]
+    assert isinstance(project, dict)
+    optional = project.get("optional-dependencies", {})
+    assert isinstance(optional, dict)
+    required = {
+        normalize_name(requirement)
+        for group in (
+            project.get("dependencies", []),
+            *(optional.get(name, []) for name in optional_groups),
+        )
+        for requirement in group
+    }
+    locked = {
+        normalize_name(package)
+        for package in LOCKED_PACKAGE_RE.findall(lock_text)
+    }
+    missing = sorted(required - locked)
+    if missing:
+        errors.append(f"direct release dependencies missing from lock: {', '.join(missing)}")
+
+    ai_packages = {
+        normalize_name(requirement)
+        for requirement in optional.get("ai", [])
+    }
+    leaked_ai = sorted(ai_packages & locked)
+    if leaked_ai:
+        errors.append(
+            f"optional AI dependencies leaked into the {lock_name} lock: "
+            + ", ".join(leaked_ai)
+        )
+    if lock_name == "runtime":
+        dev_packages = {
+            normalize_name(requirement)
+            for requirement in optional.get("dev", [])
+        }
+        leaked_dev = sorted(dev_packages & locked)
+        if leaked_dev:
+            errors.append(
+                "developer-only dependencies leaked into the runtime lock: "
+                + ", ".join(leaked_dev)
+            )
+
+    package_blocks = re.split(r"\n(?=[A-Za-z0-9_.-]+==)", lock_text)
+    unhashed = []
+    for block in package_blocks:
+        match = re.match(r"([A-Za-z0-9_.-]+)==", block)
+        if match and "--hash=sha256:" not in block:
+            unhashed.append(match.group(1))
+    if unhashed:
+        errors.append(f"locked packages without hashes: {', '.join(sorted(unhashed))}")
+    return errors
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Verify the GeoIntel Python 3.11 runtime and CI locks."
+    )
+    parser.add_argument(
+        "--stamp",
+        action="store_true",
+        help="stamp a freshly generated lock with its canonical input digest",
+    )
+    parser.add_argument(
+        "--lock",
+        choices=("all", *LOCK_GROUPS),
+        default="all",
+        help="limit validation/stamping to one lock",
+    )
+    args = parser.parse_args()
+
+    pyproject = tomllib.loads(PYPROJECT_PATH.read_text(encoding="utf-8"))
+    selected = LOCK_GROUPS if args.lock == "all" else {args.lock: LOCK_GROUPS[args.lock]}
+    errors: list[str] = []
+    for lock_name, optional_groups in selected.items():
+        lock_path = ROOT / "backend" / f"requirements-{lock_name}.lock"
+        if not lock_path.exists():
+            errors.append(f"missing lock: {lock_path.relative_to(ROOT)}")
+            continue
+        lock_text = lock_path.read_text(encoding="utf-8")
+        if args.stamp:
+            lock_text = stamp_lock(
+                lock_text,
+                input_digest(pyproject, optional_groups),
+            )
+            lock_path.write_text(lock_text, encoding="utf-8", newline="\n")
+        errors.extend(
+            validate_lock(
+                pyproject,
+                lock_text,
+                lock_name,
+                optional_groups,
+            )
+        )
+    if errors:
+        for error in errors:
+            print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    print(
+        "Python lock policy passed: Python 3.11, hashed runtime/CI locks, "
+        "AI optional."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/verify_security_exceptions.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Fail closed when a dependency-audit exception is malformed or expired."""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import sys
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+EXCEPTIONS_PATH = ROOT / "security" / "pip-audit-exceptions.json"
+
+
+def load_and_validate() -> tuple[dict[str, object], list[str]]:
+    payload = json.loads(EXCEPTIONS_PATH.read_text(encoding="utf-8"))
+    errors: list[str] = []
+    try:
+        review_by = dt.date.fromisoformat(str(payload["review_by"]))
+    except (KeyError, ValueError):
+        errors.append("review_by must be an ISO date")
+        review_by = dt.date.min
+    if review_by < dt.date.today():
+        errors.append(f"dependency exception review expired on {review_by.isoformat()}")
+    if payload.get("package") != "starlette":
+        errors.append("only the documented Starlette compatibility exception is allowed")
+    controls = payload.get("compensating_controls")
+    if not isinstance(controls, list) or len(controls) < 3:
+        errors.append("at least three compensating controls are required")
+    advisories = payload.get("advisories")
+    if not isinstance(advisories, list) or not advisories:
+        errors.append("at least one advisory exception is required")
+    else:
+        ids = [str(item.get("id", "")) for item in advisories if isinstance(item, dict)]
+        if len(ids) != len(set(ids)) or any(not item.startswith("PYSEC-") for item in ids):
+            errors.append("advisory IDs must be unique PYSEC identifiers")
+        for item in advisories:
+            if not isinstance(item, dict) or len(str(item.get("reason", ""))) < 30:
+                errors.append("every advisory requires a specific reason")
+                break
+    return payload, errors
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--print-ids", action="store_true")
+    args = parser.parse_args()
+    payload, errors = load_and_validate()
+    if errors:
+        for error in errors:
+            print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    if args.print_ids:
+        for item in payload["advisories"]:
+            print(item["id"])
+    else:
+        print(
+            "Dependency exceptions valid through "
+            f"{payload['review_by']} with documented compensating controls."
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())