fix(release): make deployment backup and rollback immutable

backend/tests/test_docker_runtime_config.py

@@ -172,6 +172,25 @@ def test_walloon_runtime_settings_are_editable_in_compose_and_unraid() -> None:
         assert "WALOUS_MAX_PIXELS" in content
 
 
+def test_in_memory_vector_limit_is_propagated_and_validated_in_every_runtime() -> None:
+    expected = "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB"
+    for path in (
+        ROOT / ".env.example",
+        ROOT / "docker-compose.yml",
+        ROOT / "docker-compose.unraid.yml",
+        ROOT / "deploy" / "unraid" / "geointel.env.example",
+        ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml",
+    ):
+        assert expected in path.read_text(encoding="utf-8"), path
+
+    run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(
+        encoding="utf-8"
+    )
+    assert 'GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="${GEOINTEL_MAX_IN_MEMORY_VECTOR_MB:-64}"' in run_script
+    assert "GEOINTEL_MAX_IN_MEMORY_VECTOR_MB must be between 1 and 256." in run_script
+    assert '-e GEOINTEL_MAX_IN_MEMORY_VECTOR_MB="$GEOINTEL_MAX_IN_MEMORY_VECTOR_MB"' in run_script
+
+
 def test_frontend_uses_same_origin_api_proxy_by_default() -> None:
     api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
     nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
@@ -208,6 +227,45 @@ def test_nginx_runtime_allows_long_ai_and_qa_requests() -> None:
         assert "proxy_send_timeout 600s;" in config
 
 
+def test_nginx_preserves_outer_https_scheme_for_secure_session_cookies() -> None:
+    configs = (
+        (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"),
+        (ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8"),
+    )
+    for config in configs:
+        assert "geo $geointel_trusted_forwarder" in config
+        assert "default 0;" in config
+        assert "172.16.0.0/12 1;" in config
+        assert 'map "$geointel_trusted_forwarder:$http_x_forwarded_proto"' in config
+        assert '"1:https" https;' in config
+        assert "proxy_set_header X-Forwarded-Proto $geointel_forwarded_proto;" in config
+        assert "proxy_set_header X-Forwarded-Proto $scheme;" not in config
+
+
+def test_nginx_runtime_sets_security_headers_on_all_cached_locations() -> None:
+    configs = (
+        (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8"),
+        (ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(
+            encoding="utf-8"
+        ),
+    )
+    required = (
+        'Content-Security-Policy "frame-ancestors \'none\'" always;',
+        'X-Frame-Options "DENY" always;',
+        'X-Content-Type-Options "nosniff" always;',
+        'Referrer-Policy "strict-origin-when-cross-origin" always;',
+        'Permissions-Policy "camera=(), microphone=(), geolocation=()" always;',
+    )
+
+    for config in configs:
+        cached_locations = config.count("add_header Cache-Control")
+        assert cached_locations >= 2
+        for header in required:
+            # Nginx 1.27 locations with Cache-Control do not inherit server-level
+            # add_header directives, so every cached location repeats the policy.
+            assert config.count(f"add_header {header}") == cached_locations + 1
+
+
 def test_compose_does_not_publish_postgis_on_default_host_port() -> None:
     compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
 

backend/tests/test_rc10_data_operations.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 import hashlib
 import importlib.util
 import json
+import os
+import subprocess
 import sys
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
@@ -18,6 +20,8 @@ SCRIPTS = ROOT / "scripts"
 
 def load_script(name: str):
     path = SCRIPTS / name
+    if str(SCRIPTS) not in sys.path:
+        sys.path.insert(0, str(SCRIPTS))
     spec = importlib.util.spec_from_file_location(f"rc10_{path.stem}", path)
     assert spec is not None and spec.loader is not None
     module = importlib.util.module_from_spec(spec)
@@ -36,6 +40,9 @@ def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha
         "database_password_secure": True,
         "inventory_mode": inventory_mode,
         "storage_inventory_requested": True,
+        "storage_snapshot_requested": True,
+        "models_inventory_requested": False,
+        "models_snapshot_requested": False,
         "git_commit": "0123456789abcdef",
     }
     files = {
@@ -48,6 +55,7 @@ def write_backup(root: Path, *, created_at: datetime, inventory_mode: str = "sha
     }
     for name, content in files.items():
         (root / name).write_text(content, encoding="utf-8")
+    (root / "storage-snapshot").mkdir()
     checksums = []
     for name in sorted(files):
         digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
@@ -337,8 +345,11 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
     readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
     live_audit = (SCRIPTS / "run_rc10_data_operations_audit.sh").read_text(encoding="utf-8")
 
-    assert "DELETE_STORAGE_ARTIFACTS" in generic
+    assert "QUARANTINE_STORAGE_ARTIFACTS" in generic
     assert "verify_current_backup" in generic
+    assert "os.link" in generic
+    assert 'entry["status"] = "linked"' in generic
+    assert "cleanup-quarantine" in generic
     assert "DELETE_DEMO_EXPORTS" in demo
     assert "verify_current_backup" in demo
     assert "/app/backups:ro" in compose
@@ -347,6 +358,8 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
         "release_backup_guard.py",
         "audit_data_operations.py",
         "cleanup_storage_artifacts.py",
+        "restore_storage_quarantine.py",
+        "release_backup_snapshot.py",
     ):
         assert f"COPY scripts/{name}" in dockerfile
         assert f"py_compile scripts/{name}" in readiness
@@ -356,3 +369,201 @@ def test_cleanup_commands_require_backup_confirmation_and_read_only_mount() -> N
     assert "table-counts-after.tsv" in live_audit
     assert "deleted_count" in live_audit
     assert "missing_manifest_artifact_count" in live_audit
+
+
+def test_cleanup_apply_moves_bytes_to_protected_traceable_quarantine(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    monkeypatch.syspath_prepend(str(SCRIPTS))
+    cleanup = load_script("cleanup_storage_artifacts.py")
+    storage = tmp_path / "storage"
+    source = storage / "derived" / "orphan.bin"
+    source.parent.mkdir(parents=True)
+    source.write_bytes(b"recoverable-derived-artifact")
+    candidate = SimpleNamespace(
+        path=source.resolve(),
+        relative_path="derived/orphan.bin",
+        size_bytes=source.stat().st_size,
+    )
+    now = datetime.now(timezone.utc)
+
+    class SessionContext:
+        def __enter__(self):
+            return SimpleNamespace()
+
+        def __exit__(self, *_args):
+            return False
+
+    monkeypatch.setattr(
+        cleanup,
+        "parse_args",
+        lambda: SimpleNamespace(
+            storage_root=storage,
+            minimum_age_days=7,
+            max_delete=1,
+            apply=True,
+            confirm="QUARANTINE_STORAGE_ARTIFACTS",
+            backup_dir=tmp_path / "backup",
+            backup_max_age_hours=24.0,
+            quarantine_root=None,
+        ),
+    )
+    monkeypatch.setattr(cleanup, "SessionLocal", lambda: SessionContext())
+    monkeypatch.setattr(
+        cleanup,
+        "build_report",
+        lambda *_args, **_kwargs: ({"cleanup": {"protected_prefixes": []}}, [candidate]),
+    )
+    monkeypatch.setattr(
+        cleanup,
+        "verify_current_backup",
+        lambda *_args, **_kwargs: SimpleNamespace(
+            release_id="predeploy-test",
+            created_at=now,
+            age_hours=0.1,
+            backup_tool_revision="0123456789abcdef",
+        ),
+    )
+
+    assert cleanup.main() == 0
+    payload = json.loads(capsys.readouterr().out)
+    manifest_path = Path(payload["quarantine_manifest"])
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    quarantined_path = storage / payload["quarantined"][0]["quarantine_relative_path"]
+
+    assert not source.exists()
+    assert quarantined_path.read_bytes() == b"recoverable-derived-artifact"
+    assert manifest["state"] == "complete"
+    assert manifest["backup_release_id"] == "predeploy-test"
+    assert manifest["entries"][0]["status"] == "quarantined"
+    assert payload["deleted_count"] == 0
+
+    restore = subprocess.run(
+        [
+            sys.executable,
+            str(SCRIPTS / "restore_storage_quarantine.py"),
+            "--storage-root",
+            str(storage),
+            "--manifest",
+            str(manifest_path),
+            "--confirm",
+            "RESTORE_QUARANTINED_ARTIFACTS",
+        ],
+        cwd=ROOT,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert restore.returncode == 0, restore.stderr
+    assert source.read_bytes() == b"recoverable-derived-artifact"
+    assert not quarantined_path.exists()
+    restored_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    assert restored_manifest["state"] == "restored"
+    assert restored_manifest["entries"][0]["status"] == "restored"
+
+
+def _write_interrupted_quarantine(
+    storage: Path,
+    *,
+    original_exists: bool,
+    quarantine_exists: bool,
+    hard_linked: bool = False,
+) -> tuple[Path, Path, Path]:
+    original = storage / "derived" / "interrupted.bin"
+    operation = storage / "operator-evidence" / "cleanup-quarantine" / "cleanup-interrupted"
+    quarantined = operation / "files" / "derived" / "interrupted.bin"
+    original.parent.mkdir(parents=True, exist_ok=True)
+    quarantined.parent.mkdir(parents=True, exist_ok=True)
+    retained = b"interrupted-retained-bytes"
+    if original_exists:
+        original.write_bytes(retained)
+    if quarantine_exists:
+        if hard_linked:
+            os.link(original, quarantined)
+        else:
+            quarantined.write_bytes(retained)
+    manifest = operation / "manifest.json"
+    manifest.write_text(
+        json.dumps(
+            {
+                "schema_version": 1,
+                "state": "in_progress",
+                "entries": [
+                    {
+                        "relative_path": "derived/interrupted.bin",
+                        "quarantine_relative_path": quarantined.relative_to(storage).as_posix(),
+                        "size_bytes": len(retained),
+                        "sha256": hashlib.sha256(retained).hexdigest(),
+                        "status": "linked" if hard_linked else "planned",
+                    }
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+    return manifest, original, quarantined
+
+
+@pytest.mark.parametrize(
+    ("original_exists", "quarantine_exists", "hard_linked"),
+    ((False, True, False), (True, True, True)),
+)
+def test_quarantine_restore_recovers_each_interrupted_move_window(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    original_exists: bool,
+    quarantine_exists: bool,
+    hard_linked: bool,
+) -> None:
+    restore = load_script("restore_storage_quarantine.py")
+    storage = tmp_path / "storage"
+    manifest, original, quarantined = _write_interrupted_quarantine(
+        storage,
+        original_exists=original_exists,
+        quarantine_exists=quarantine_exists,
+        hard_linked=hard_linked,
+    )
+    monkeypatch.setattr(
+        restore,
+        "parse_args",
+        lambda: SimpleNamespace(
+            storage_root=storage,
+            manifest=manifest,
+            confirm="RESTORE_QUARANTINED_ARTIFACTS",
+        ),
+    )
+
+    assert restore.main() == 0
+    assert original.read_bytes() == b"interrupted-retained-bytes"
+    assert not quarantined.exists()
+    assert json.loads(manifest.read_text(encoding="utf-8"))["state"] == "restored"
+
+
+def test_quarantine_restore_never_clobbers_recreated_destination(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    restore = load_script("restore_storage_quarantine.py")
+    storage = tmp_path / "storage"
+    manifest, original, quarantined = _write_interrupted_quarantine(
+        storage,
+        original_exists=False,
+        quarantine_exists=True,
+    )
+    original.write_bytes(b"new-runtime-bytes")
+    monkeypatch.setattr(
+        restore,
+        "parse_args",
+        lambda: SimpleNamespace(
+            storage_root=storage,
+            manifest=manifest,
+            confirm="RESTORE_QUARANTINED_ARTIFACTS",
+        ),
+    )
+
+    with pytest.raises(RuntimeError, match="different bytes"):
+        restore.main()
+    assert original.read_bytes() == b"new-runtime-bytes"
+    assert quarantined.read_bytes() == b"interrupted-retained-bytes"

backend/tests/test_rc5_release_deployment.py

@@ -18,17 +18,16 @@ def test_build_identity_does_not_invalidate_dependency_layers() -> None:
     assert 'io.geointel.ai.enabled="${GEOINTEL_INSTALL_AI}"' in dockerfile
 
 
-def test_release_deploy_preserves_immutable_and_previous_images() -> None:
+def test_release_deploy_preserves_immutable_and_backup_specific_rollback_images() -> None:
     script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
 
     assert 'GEOINTEL_RELEASE_VARIANT="ai"' in script
     assert 'GEOINTEL_RELEASE_VARIANT="gis"' in script
     assert 'GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"' in script
-    assert 'GEOINTEL_PREVIOUS_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:previous"' in script
-    assert 'release_image_id="$(' in script
-    assert '[ "$current_image_id" != "$release_image_id" ]' in script
-    assert 'docker tag "$current_image_id" "$GEOINTEL_PREVIOUS_IMAGE"' in script
-    assert "preserving the existing previous image" in script
+    assert 'GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"' in script
+    assert 'docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
+    assert '--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
+    assert "GEOINTEL_PREVIOUS_IMAGE" not in script
     assert 'if docker image inspect "$GEOINTEL_RELEASE_IMAGE"' in script
     assert "Immutable release tag has conflicting metadata" in script
     assert "Reusing existing immutable image" in script
@@ -36,12 +35,62 @@ def test_release_deploy_preserves_immutable_and_previous_images() -> None:
     assert "Deployed immutable image" in script
 
 
+def test_release_creates_verified_backup_before_candidate_migrations() -> None:
+    script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
+
+    backup_index = script.index("create_predeploy_backup\n")
+    scan_index = script.index("scan_release_image\n")
+    candidate_start_index = script.index('if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"')
+    assert scan_index < backup_index
+    assert backup_index < candidate_start_index
+    assert script.index("docker build") < backup_index
+    assert "/mnt/user/appdata/geointel/backups" in script
+    assert "--inventory-mode sha256" in script
+    assert "scripts/verify_release_backup.sh" in script
+    assert "refusing an unbacked migration" in script
+    assert "Quiescing the current backend" in script
+    assert "restarting the unchanged current release" in script
+    assert "preflight_backup_capacity" in script
+    assert "select_verified_link_dest" in script
+    assert "release_backup_snapshot.py" in script
+
+
+def test_release_starts_only_the_locally_attested_ai_image() -> None:
+    script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
+
+    assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"' in script
+    assert "Production release deployment requires the gated AI image" in script
+    assert 'bash scripts/generate_container_sbom.sh "$GEOINTEL_RELEASE_IMAGE"' in script
+    assert 'bash scripts/scan_container_image.sh "$GEOINTEL_RELEASE_IMAGE"' in script
+    assert 'running_image_id="$(docker inspect --format \'{{.Image}}\' geointel)"' in script
+    assert 'if [ "$running_image_id" != "$image" ]' in script
+    assert "deployment-attestation.json" in script
+    assert "artifacts/release-evidence/deploy" in script
+    assert "GITEA_COMMIT_SHA" in script
+    assert "GITHUB_SHA" in script
+    assert "must contain one full 40-character Git commit SHA" in script
+    assert 'running_revision" != "$GEOINTEL_BUILD_SHA"' in script
+    assert 'running_ai" != "true"' in script
+    assert '"revision": revision' in script
+
+
 def test_release_and_container_replacement_are_serialized() -> None:
     release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
     run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
+    rollback_script = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
+        encoding="utf-8"
+    )
+    restore_script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
+        encoding="utf-8"
+    )
 
     assert "GEOINTEL_DEPLOY_LOCK_FILE" in release_script
     assert "flock -n 9" in release_script
+    assert "GEOINTEL_DEPLOY_LOCK_FILE" in rollback_script
+    assert "flock -n 9" in rollback_script
+    assert "GEOINTEL_DEPLOY_LOCK_FILE" in restore_script
+    assert "flock -n 9" in restore_script
+    assert "GEOINTEL_DEPLOY_LOCK_HELD=true" in release_script
     assert "GEOINTEL_CONTAINER_LOCK_FILE" in run_script
     assert "flock -w 300 8" in run_script
     assert "GeoIntel container removal did not complete within 60 seconds" in run_script
@@ -89,13 +138,35 @@ def test_manual_rollback_reuses_persistent_paths_and_requires_existing_image() -
     rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(encoding="utf-8")
     run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
 
-    assert "geointel-all-in-one:previous" in rollback
+    assert "Backup manifest lacks an immutable rollback image ID" in rollback
+    assert 'get("image_id", "")' in rollback
     assert 'docker image inspect "$GEOINTEL_ROLLBACK_IMAGE"' in rollback
     assert 'GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE"' in rollback
+    assert "restore-predeploy-database.sh" in rollback
+    assert "--confirm-production-database-restore" in rollback
+    assert "Image-only rollback" in rollback
     assert '-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data"' in run_script
     assert '-v "${GEOINTEL_STORAGE_PATH}:/app/storage"' in run_script
 
 
+def test_same_revision_redeploy_rolls_back_by_backup_bound_image_id() -> None:
+    release = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
+    rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
+        encoding="utf-8"
+    )
+    restore = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
+        encoding="utf-8"
+    )
+    backup = (ROOT / "scripts" / "backup_release_state.sh").read_text(encoding="utf-8")
+
+    assert "current_image_id" in release
+    assert "release_image_id" not in release
+    assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in backup
+    assert 'if [ -z "$RESTORE_IMAGE" ]; then\n  RESTORE_IMAGE="$BACKUP_IMAGE_ID"' in restore
+    assert 'GEOINTEL_ROLLBACK_IMAGE="$(python3 - "$BACKUP_DIR/manifest.json"' in rollback
+    assert "geointel-all-in-one:previous" not in release + rollback + restore
+
+
 def test_readiness_checks_all_release_shell_entrypoints() -> None:
     readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
 
@@ -107,6 +178,7 @@ def test_readiness_checks_all_release_shell_entrypoints() -> None:
         "deploy/unraid/run-dockerman-container.sh",
         "deploy/unraid/deploy-release.sh",
         "deploy/unraid/rollback-dockerman-container.sh",
+        "deploy/unraid/restore-predeploy-database.sh",
     ):
         assert f"bash -n {path}" in readiness
 

backend/tests/test_rc6_supply_chain.py

@@ -48,13 +48,54 @@ def test_ci_runs_complete_release_and_supply_chain_gates() -> None:
         assert "pip-audit==2.10.1" in workflow
         assert "audit_python_dependencies.sh" in workflow
         assert "npm audit --audit-level=high" in workflow
-        assert "GEOINTEL_INSTALL_AI=false" in workflow
+        assert "GEOINTEL_INSTALL_AI=true" in workflow
+        assert "geointel-ci:$RELEASE_SHA-ai" in workflow
+        assert "artifacts/image-id.txt" in workflow
+        assert 'scan_container_image.sh "$IMAGE_ID"' in workflow
         assert "generate_container_sbom.sh" in workflow
         assert "scan_container_image.sh" in workflow
-        assert "actions/upload-artifact@v4" in workflow
+        assert "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" in workflow
         assert context in workflow
 
 
+def test_gitea_production_deploy_depends_on_every_release_gate() -> None:
+    release = read(".gitea/workflows/release-gates.yml")
+    legacy_deploy = ROOT / ".gitea" / "workflows" / "unraid-deploy.yml"
+
+    assert "pull_request:" in release
+    assert "needs: [quality, dependency-audit, container]" in release
+    assert "gitea.event_name == 'push'" in release
+    assert "gitea.ref == 'refs/heads/main'" in release
+    assert "/opt/gitea-deploy/deploy.py deploy" in release
+    assert not legacy_deploy.exists()
+    assert "workflow_dispatch:" in release
+
+
+def test_release_workflows_pin_third_party_actions_to_reviewed_commits() -> None:
+    expected = (
+        "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683",
+        "actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065",
+        "actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020",
+        "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02",
+    )
+    for path in (".gitea/workflows/release-gates.yml", ".github/workflows/release-gates.yml"):
+        workflow = read(path)
+        for action in expected:
+            assert action in workflow
+
+
+def test_managed_validation_targets_the_actual_backend_and_frontend_projects() -> None:
+    workflow = read(".gitea/workflows/managed-validation.yml")
+
+    assert "backend/requirements-ci.lock" in workflow
+    assert "frontend/package-lock.json" in workflow
+    assert "python -m pytest -W error::DeprecationWarning" in workflow
+    assert "cd frontend && npm run test:unit" in workflow
+    assert "python -m ruff check backend scripts tests" in workflow
+    assert "python scripts/verify_repository_layout.py" in workflow
+    assert "[[ -f pyproject.toml" not in workflow
+
+
 def test_scanner_images_are_versioned_and_digest_pinned() -> None:
     sbom = read("scripts/generate_container_sbom.sh")
     scan = read("scripts/scan_container_image.sh")
@@ -65,8 +106,9 @@ def test_scanner_images_are_versioned_and_digest_pinned() -> None:
     assert "--ignore-unfixed" in scan
     assert "--timeout 20m" in scan
     assert "--scanners vuln" in scan
-    assert '-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro"' in scan
-    assert '--ignorefile "$CONTAINER_IGNORE_FILE"' in scan
+    assert 'ignored_container_ids' in scan
+    assert 'ignore_args=(-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro")' in scan
+    assert 'trivy_ignore_args=(--ignorefile "$CONTAINER_IGNORE_FILE")' in scan
     assert "--skip-files /usr/local/bin/gosu" in scan
     assert "final filesystem replaces it with the audited setpriv shell wrapper" in scan
     assert "geointel-container-vulnerabilities.json" in scan
@@ -86,11 +128,12 @@ def test_readiness_guards_lock_and_supply_chain_entrypoints() -> None:
         assert f"bash -n {path}" in readiness
 
 
-def test_python_audit_exceptions_are_timeboxed_and_full_evidence_is_kept() -> None:
+def test_python_audit_policy_has_no_active_exceptions_and_keeps_full_evidence() -> None:
     policy = read("security/pip-audit-exceptions.json")
     audit_script = read("scripts/audit_python_dependencies.sh")
 
-    assert '"review_by": "2026-08-31"' in policy
+    assert '"schema_version": 1' in policy
+    assert '"advisories": []' in policy
     assert "pip-audit-full.json" in audit_script
     assert "pip-audit-policy.json" in audit_script
     assert "--ignore-vuln" in audit_script

backend/tests/test_rc_backup_restore_scripts.py

@@ -20,12 +20,35 @@ def test_backup_is_atomic_read_only_and_checksum_bound() -> None:
     assert "--no-owner" in script
     assert "CHECKSUMS.sha256" in script
     assert "database-password" not in script.lower()
-    assert 'git -C "$ROOT" rev-parse HEAD' in script
-    assert 'git -C "$ROOT" status --porcelain=v1' in script
+    assert 'for required in docker python3 sha256sum; do' in script
+    assert 'for required in docker python3 sha256sum git; do' not in script
+    assert "GITEA_COMMIT_SHA" in script
+    assert "GITHUB_SHA" in script
+    assert "GEOINTEL_BUILD_SHA" in script
+    assert 'if command -v git >/dev/null 2>&1' in script
     assert "mv \"$PARTIAL\" \"$FINAL\"" in script
     assert "rm -rf -- \"$PARTIAL\"" in script
     assert "DROP DATABASE" not in script
     assert "pg_restore --clean" not in script
+    assert "/mnt/user/appdata/geointel/backups" in script
+    assert "release_backup_snapshot.py" in script
+    assert "storage-snapshot" not in script  # labels are composed without unsafe path interpolation
+    assert "--link-dest-backup" in script
+    assert "--rollback-image-tag" in script
+    assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in script
+
+
+def test_backup_binds_prepared_source_without_requiring_dot_git() -> None:
+    script = read("backup_release_state.sh")
+
+    controller_resolution = script.index('local gitea_sha="${GITEA_COMMIT_SHA:-}"')
+    optional_git_fallback = script.index('if command -v git >/dev/null 2>&1')
+    docker_access = script.index("docker inspect -f '{{.State.Running}}'")
+    assert controller_resolution < optional_git_fallback < docker_access
+    assert 'SOURCE_REVISION="$explicit_sha"' in script
+    assert '"backup_tool_revision": ${SOURCE_REVISION@Q}' in script
+    assert '"running_image_revision": ${RUNNING_IMAGE_REVISION@Q}' in script
+    assert "Cannot bind backup to a source revision" in script
 
 
 def test_backup_verification_is_read_only() -> None:
@@ -55,9 +78,11 @@ def test_release_safety_scripts_have_valid_bash_syntax() -> None:
         "backup_release_state.sh",
         "verify_release_backup.sh",
         "restore_release_backup_smoke.sh",
+        "../deploy/unraid/restore-predeploy-database.sh",
     ):
+        script_path = f"scripts/{name}" if not name.startswith("../") else name.removeprefix("../")
         result = subprocess.run(
-            ["bash", "-n", f"scripts/{name}"],
+            ["bash", "-n", script_path],
             cwd=ROOT,
             capture_output=True,
             text=True,
@@ -66,6 +91,30 @@ def test_release_safety_scripts_have_valid_bash_syntax() -> None:
         assert result.returncode == 0, f"{name}: {result.stderr}"
 
 
+def test_production_restore_is_explicit_bounded_and_verified() -> None:
+    script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
+        encoding="utf-8"
+    )
+
+    assert "--confirm-production-database-restore" in script
+    assert "/mnt/user/appdata/geointel/backups" in script
+    assert "backup.relative_to(root)" in script
+    assert "sha256sum -c CHECKSUMS.sha256" in script
+    assert '"$RESTORE_PROOF_DB"' in script
+    assert "pg_restore" in script
+    assert "Restored Alembic head" in script
+    assert "Restored count mismatch" in script
+    assert "pg_restore --clean" not in script
+    assert "geointel_restore_proof_" in script
+    assert "Isolated predeploy restore proof passed" in script
+    assert "ALTER DATABASE" in script
+    assert "Pre-restore production database retained" in script
+    drop_start = script.index("dropdb --if-exists --force")
+    drop_command = script[drop_start : script.index("\n  fi", drop_start)]
+    assert '"$RESTORE_PROOF_DB"' in drop_command
+    assert '"$GEOINTEL_POSTGRES_DB"' not in drop_command
+
+
 def test_readiness_gate_checks_release_safety_scripts() -> None:
     readiness = read("run_readiness_check.sh")
 

backend/tests/test_readiness_gate.py

@@ -1,11 +1,12 @@
 from pathlib import Path
 
 
-def test_backend_keeps_starlette_on_the_supported_pre_httpx2_line() -> None:
+def test_backend_uses_patched_starlette_and_explicit_httpx2_test_client() -> None:
     pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml"
     content = pyproject.read_text(encoding="utf-8")
 
-    assert '"starlette>=0.46.0,<1.0.0"' in content
+    assert '"starlette>=1.3.1,<2.0.0"' in content
+    assert '"httpx2>=2.0.0"' in content
 
 
 def test_readiness_gate_treats_deprecation_warnings_as_errors() -> None:

backend/tests/test_release_backup_snapshot.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = ROOT / "scripts" / "release_backup_snapshot.py"
+
+
+def load_snapshot_module():
+    spec = importlib.util.spec_from_file_location("release_backup_snapshot_test", SCRIPT)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_snapshot_is_byte_complete_and_reuses_only_verified_backup_bytes(tmp_path: Path) -> None:
+    snapshot = load_snapshot_module()
+    source = tmp_path / "source"
+    source.mkdir()
+    (source / "same.bin").write_bytes(b"unchanged")
+    (source / "changed.bin").write_bytes(b"before")
+    (source / "empty").mkdir()
+
+    prior = tmp_path / "prior"
+    prior_manifest = tmp_path / "prior.tsv"
+    snapshot.create_snapshot(source, prior, prior_manifest, label="storage")
+    snapshot.verify_snapshot(prior, prior_manifest)
+
+    (source / "changed.bin").write_bytes(b"after")
+    current = tmp_path / "current"
+    current_manifest = tmp_path / "current.tsv"
+    snapshot.create_snapshot(
+        source,
+        current,
+        current_manifest,
+        label="storage",
+        link_dest_snapshot=prior,
+        link_dest_manifest=prior_manifest,
+    )
+    snapshot.verify_snapshot(current, current_manifest)
+
+    assert os.path.samefile(prior / "same.bin", current / "same.bin")
+    assert not os.path.samefile(prior / "changed.bin", current / "changed.bin")
+    assert (current / "changed.bin").read_bytes() == b"after"
+    assert (current / "empty").is_dir()
+
+
+def test_snapshot_rejects_symlinked_content(tmp_path: Path) -> None:
+    snapshot = load_snapshot_module()
+    source = tmp_path / "source"
+    source.mkdir()
+    target = source / "target.bin"
+    target.write_bytes(b"target")
+    try:
+        (source / "link.bin").symlink_to(target)
+    except OSError:
+        pytest.skip("Symlink creation is unavailable on this host")
+
+    with pytest.raises(RuntimeError, match="refuses symlinked content"):
+        snapshot.create_snapshot(source, tmp_path / "snapshot", tmp_path / "manifest.tsv", label="storage")
+
+
+def test_snapshot_verification_rejects_changed_retained_bytes(tmp_path: Path) -> None:
+    snapshot = load_snapshot_module()
+    source = tmp_path / "source"
+    source.mkdir()
+    (source / "artifact.bin").write_bytes(b"retained")
+    retained = tmp_path / "snapshot"
+    manifest = tmp_path / "manifest.tsv"
+    snapshot.create_snapshot(source, retained, manifest, label="storage")
+    (retained / "artifact.bin").chmod(0o644)
+    (retained / "artifact.bin").write_bytes(b"tampered")
+
+    with pytest.raises(RuntimeError, match="checksum differs"):
+        snapshot.verify_snapshot(retained, manifest)

backend/tests/test_sprint31_unraid_template.py

@@ -150,7 +150,7 @@ def test_tower_deploy_uses_single_container_unraid_compose() -> None:
     assert "LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh" in release_script
 
 
-def test_tower_deploy_build_uses_remote_env_ai_setting_by_default() -> None:
+def test_tower_deploy_build_requires_the_production_ai_variant_by_default() -> None:
     powershell = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8")
     bash = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8")
     release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
@@ -161,7 +161,8 @@ def test_tower_deploy_build_uses_remote_env_ai_setting_by_default() -> None:
 
     assert "if [ -f .env ]; then" in release_script
     assert ". ./.env" in release_script
-    assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-false}"' in release_script
+    assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"' in release_script
+    assert "Production release deployment requires the gated AI image" in release_script
     assert "--build-arg GEOINTEL_INSTALL_AI=" in release_script