Jens
bcd023ec36
Managed validation / Managed repository validation (pull_request) Successful in 3m35s
GeoIntel release gates / Compile, test, contracts and builds (pull_request) Successful in 3m42s
GeoIntel release gates / Python and npm vulnerability policy (pull_request) Successful in 45s
GeoIntel release gates / Production AI image, SBOM and container scan (pull_request) Failing after 6m19s
GeoIntel release gates / Deploy exact gated revision to Unraid (pull_request) Skipped
220 lines
11 KiB
Python
220 lines
11 KiB
Python
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def test_build_identity_does_not_invalidate_dependency_layers() -> None:
|
|
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
|
|
|
dependency_install = dockerfile.index("/usr/bin/python3.11 -m venv /opt/geointel/venv")
|
|
source_copy = dockerfile.index("COPY backend/ /app/")
|
|
build_identity = dockerfile.index("ARG GEOINTEL_BUILD_SHA=unknown")
|
|
|
|
assert build_identity > dependency_install
|
|
assert build_identity > source_copy
|
|
assert 'org.opencontainers.image.revision="${GEOINTEL_BUILD_SHA}"' in dockerfile
|
|
assert 'org.opencontainers.image.created="${GEOINTEL_BUILD_TIME}"' in dockerfile
|
|
assert 'io.geointel.ai.enabled="${GEOINTEL_INSTALL_AI}"' in dockerfile
|
|
|
|
|
|
def test_release_deploy_preserves_immutable_and_backup_specific_rollback_images() -> None:
|
|
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
|
|
|
assert 'GEOINTEL_RELEASE_VARIANT="ai"' in script
|
|
assert 'GEOINTEL_RELEASE_VARIANT="gis"' in script
|
|
assert 'GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"' in script
|
|
assert 'GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"' in script
|
|
assert 'docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
|
|
assert '--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"' in script
|
|
assert "GEOINTEL_PREVIOUS_IMAGE" not in script
|
|
assert 'if docker image inspect "$GEOINTEL_RELEASE_IMAGE"' in script
|
|
assert "Immutable release tag has conflicting metadata" in script
|
|
assert "Reusing existing immutable image" in script
|
|
assert "rollback_previous()" in script
|
|
assert "Deployed immutable image" in script
|
|
|
|
|
|
def test_release_creates_verified_backup_before_candidate_migrations() -> None:
|
|
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
|
|
|
backup_index = script.index("create_predeploy_backup\n")
|
|
scan_index = script.index("scan_release_image\n")
|
|
candidate_start_index = script.index('if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"')
|
|
assert scan_index < backup_index
|
|
assert backup_index < candidate_start_index
|
|
assert script.index("docker build") < backup_index
|
|
assert "/mnt/user/appdata/geointel/backups" in script
|
|
assert "--inventory-mode sha256" in script
|
|
assert "scripts/verify_release_backup.sh" in script
|
|
assert "refusing an unbacked migration" in script
|
|
assert "Quiescing the current backend" in script
|
|
assert "restarting the unchanged current release" in script
|
|
assert "preflight_backup_capacity" in script
|
|
assert "select_verified_link_dest" in script
|
|
assert "release_backup_snapshot.py" in script
|
|
assert "run_low_impact()" in script
|
|
assert "ionice -c 2 -n 7" in script
|
|
assert "nice -n 10" in script
|
|
assert "run_low_impact bash scripts/backup_release_state.sh" in script
|
|
assert "run_low_impact bash scripts/verify_release_backup.sh" in script
|
|
|
|
|
|
def test_gitea_deploy_waits_for_the_mandatory_large_snapshot() -> None:
|
|
workflow = (ROOT / ".gitea" / "workflows" / "release-gates.yml").read_text(encoding="utf-8")
|
|
|
|
assert "cancel-in-progress: false" in workflow
|
|
deploy_job = workflow.split("\n deploy:\n", maxsplit=1)[1]
|
|
assert "timeout-minutes: 720" in deploy_job
|
|
assert "docker exec gitea-deploy-control" in deploy_job
|
|
|
|
|
|
def test_release_starts_only_the_locally_attested_ai_image() -> None:
|
|
script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
|
|
|
assert 'GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"' in script
|
|
assert "Production release deployment requires the gated AI image" in script
|
|
assert 'bash scripts/generate_container_sbom.sh "$scanned_image_id"' in script
|
|
assert 'bash scripts/scan_container_image.sh "$scanned_image_id"' in script
|
|
assert 'running_image_id="$(docker inspect --format \'{{.Image}}\' geointel)"' in script
|
|
assert 'if [ "$running_image_id" != "$image" ]' in script
|
|
assert "deployment-attestation.json" in script
|
|
assert "artifacts/release-evidence/deploy" in script
|
|
assert "GITEA_COMMIT_SHA" in script
|
|
assert "GITHUB_SHA" in script
|
|
assert "must contain one full 40-character Git commit SHA" in script
|
|
assert 'marker_path="$ROOT/.gitea-deploy/revision"' in script
|
|
assert "git rev-parse --show-toplevel" in script
|
|
assert "Prepared source revision marker does not match" in script
|
|
assert "neither an exact Git checkout nor bound" in script
|
|
assert 'running_revision" != "$GEOINTEL_BUILD_SHA"' in script
|
|
assert 'running_ai" != "true"' in script
|
|
assert '"revision": revision' in script
|
|
|
|
|
|
def test_release_and_container_replacement_are_serialized() -> None:
|
|
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
|
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
|
rollback_script = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
restore_script = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
assert "GEOINTEL_DEPLOY_LOCK_FILE" in release_script
|
|
assert "flock -n 9" in release_script
|
|
assert "GEOINTEL_DEPLOY_LOCK_FILE" in rollback_script
|
|
assert "flock -n 9" in rollback_script
|
|
assert "GEOINTEL_DEPLOY_LOCK_FILE" in restore_script
|
|
assert "flock -n 9" in restore_script
|
|
assert "GEOINTEL_DEPLOY_LOCK_HELD=true" in release_script
|
|
assert "GEOINTEL_CONTAINER_LOCK_FILE" in run_script
|
|
assert "flock -w 300 8" in run_script
|
|
assert "GeoIntel container removal did not complete within 60 seconds" in run_script
|
|
|
|
|
|
def test_release_waits_for_large_postgis_volume_recovery() -> None:
|
|
release_script = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
|
start_script = (ROOT / "deploy" / "unraid" / "all-in-one-start.sh").read_text(encoding="utf-8")
|
|
|
|
assert "for attempt in $(seq 1 480)" in release_script
|
|
assert "for attempt in $(seq 1 450)" in start_script
|
|
assert "PostGIS did not become ready within 15 minutes." in start_script
|
|
assert 'chown postgres:postgres "$PGDATA"' in start_script
|
|
assert 'chown -R postgres:postgres "$PGDATA"' not in start_script
|
|
|
|
|
|
def test_runtime_configuration_is_validated_before_container_replacement() -> None:
|
|
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
|
|
|
validation_index = run_script.index("validate_runtime_config")
|
|
# The script replaces the container with "docker rm -f"; what matters is
|
|
# that no removal happens before the runtime config has been validated.
|
|
replacement_index = run_script.index("docker rm -f geointel")
|
|
|
|
assert validation_index < replacement_index
|
|
assert "known-default PostGIS password" in run_script
|
|
assert "GEOINTEL_MAX_UPLOAD_MB must be between 1 and 2048" in run_script
|
|
assert 'docker image inspect "$GEOINTEL_IMAGE"' in run_script
|
|
|
|
|
|
def test_fresh_install_smoke_is_isolated_and_cleans_only_its_temp_path() -> None:
|
|
script = (ROOT / "scripts" / "verify_release_fresh_install.sh").read_text(encoding="utf-8")
|
|
|
|
assert "mktemp -d" in script
|
|
assert "geointel-fresh-smoke.*" in script
|
|
assert "-p 127.0.0.1::80" in script
|
|
assert "GEOINTEL_POSTGRES_PASSWORD=" in script
|
|
assert "/health/ready" in script
|
|
assert "/api/v1/system/capabilities" in script
|
|
assert "docker exec" in script
|
|
assert "python -m alembic heads" in script
|
|
|
|
|
|
def test_manual_rollback_reuses_persistent_paths_and_requires_existing_image() -> None:
|
|
rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(encoding="utf-8")
|
|
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
|
|
|
assert "Backup manifest lacks an immutable rollback image ID" in rollback
|
|
assert 'get("image_id", "")' in rollback
|
|
assert 'docker image inspect "$GEOINTEL_ROLLBACK_IMAGE"' in rollback
|
|
assert 'GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE"' in rollback
|
|
assert "restore-predeploy-database.sh" in rollback
|
|
assert "--confirm-production-database-restore" in rollback
|
|
assert "Image-only rollback" in rollback
|
|
assert '-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data"' in run_script
|
|
assert '-v "${GEOINTEL_STORAGE_PATH}:/app/storage"' in run_script
|
|
|
|
|
|
def test_same_revision_redeploy_rolls_back_by_backup_bound_image_id() -> None:
|
|
release = (ROOT / "deploy" / "unraid" / "deploy-release.sh").read_text(encoding="utf-8")
|
|
rollback = (ROOT / "deploy" / "unraid" / "rollback-dockerman-container.sh").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
restore = (ROOT / "deploy" / "unraid" / "restore-predeploy-database.sh").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
backup = (ROOT / "scripts" / "backup_release_state.sh").read_text(encoding="utf-8")
|
|
|
|
assert "current_image_id" in release
|
|
assert "release_image_id" not in release
|
|
assert '"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None' in backup
|
|
assert 'if [ -z "$RESTORE_IMAGE" ]; then\n RESTORE_IMAGE="$BACKUP_IMAGE_ID"' in restore
|
|
assert 'GEOINTEL_ROLLBACK_IMAGE="$(python3 - "$BACKUP_DIR/manifest.json"' in rollback
|
|
assert "geointel-all-in-one:previous" not in release + rollback + restore
|
|
|
|
|
|
def test_readiness_checks_all_release_shell_entrypoints() -> None:
|
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
|
|
|
for path in (
|
|
"scripts/deploy_tower.sh",
|
|
"scripts/verify_release_fresh_install.sh",
|
|
"scripts/verify_release_upgrade_smoke.sh",
|
|
"deploy/unraid/all-in-one-start.sh",
|
|
"deploy/unraid/run-dockerman-container.sh",
|
|
"deploy/unraid/deploy-release.sh",
|
|
"deploy/unraid/rollback-dockerman-container.sh",
|
|
"deploy/unraid/restore-predeploy-database.sh",
|
|
):
|
|
assert f"bash -n {path}" in readiness
|
|
|
|
|
|
def test_upgrade_smoke_restores_and_upgrades_only_an_isolated_database() -> None:
|
|
script = (ROOT / "scripts" / "verify_release_upgrade_smoke.sh").read_text(encoding="utf-8")
|
|
|
|
assert "--confirm-isolated-upgrade" in script
|
|
assert "restore_release_backup_smoke.sh" in script
|
|
assert "--keep-database" in script
|
|
assert "geointel_restore_verify_" in script
|
|
assert "from sqlalchemy import URL" in script
|
|
assert 'username=os.environ["GEOINTEL_POSTGRES_USER"]' in script
|
|
assert 'password=os.environ["GEOINTEL_POSTGRES_PASSWORD"]' in script
|
|
assert 'database=os.environ["TARGET_DB"]' in script
|
|
assert '"$CONTAINER" sh -c' in script
|
|
assert '"$CONTAINER" sh -lc' not in script
|
|
assert "python -m alembic upgrade head" in script
|
|
assert "production_database_untouched" in script
|
|
assert 'dropdb --if-exists -U "$db_user" "$TARGET_DB"' in script
|