diff --git a/.gitea/workflows/release-gates.yml b/.gitea/workflows/release-gates.yml index 3c2351ec..c174e473 100644 --- a/.gitea/workflows/release-gates.yml +++ b/.gitea/workflows/release-gates.yml @@ -11,7 +11,10 @@ permissions: concurrency: group: geointel-release-${{ gitea.ref }} - cancel-in-progress: true + # A cancelled HTTP caller does not terminate the allowlisted controller + # process that already owns the production lock. Queue a newer revision + # instead of orphaning an in-flight backup or deploy. + cancel-in-progress: false jobs: quality: @@ -162,7 +165,10 @@ jobs: needs: [quality, dependency-audit, container] if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }} runs-on: unraid-deploy - timeout-minutes: 180 + # The first byte-complete storage snapshot can exceed 100 GiB. Keep the + # gated caller attached for the full conservative backup/build window; + # the controller and deploy script still serialize every mutation. + timeout-minutes: 720 steps: - name: Deploy only after every release gate is green run: | diff --git a/backend/tests/test_rc5_release_deployment.py b/backend/tests/test_rc5_release_deployment.py index 358feeb6..9b55d33f 100644 --- a/backend/tests/test_rc5_release_deployment.py +++ b/backend/tests/test_rc5_release_deployment.py @@ -53,6 +53,20 @@ def test_release_creates_verified_backup_before_candidate_migrations() -> None: assert "preflight_backup_capacity" in script assert "select_verified_link_dest" in script assert "release_backup_snapshot.py" in script + assert "run_low_impact()" in script + assert "ionice -c 2 -n 7" in script + assert "nice -n 10" in script + assert "run_low_impact bash scripts/backup_release_state.sh" in script + assert "run_low_impact bash scripts/verify_release_backup.sh" in script + + +def test_gitea_deploy_waits_for_the_mandatory_large_snapshot() -> None: + workflow = (ROOT / ".gitea" / "workflows" / "release-gates.yml").read_text(encoding="utf-8") + + assert "cancel-in-progress: false" in workflow + deploy_job = workflow.split("\n deploy:\n", maxsplit=1)[1] + assert "timeout-minutes: 720" in deploy_job + assert "docker exec gitea-deploy-control" in deploy_job def test_release_starts_only_the_locally_attested_ai_image() -> None: diff --git a/deploy/unraid/deploy-release.sh b/deploy/unraid/deploy-release.sh index 2ec08faa..26456dc2 100644 --- a/deploy/unraid/deploy-release.sh +++ b/deploy/unraid/deploy-release.sh @@ -355,14 +355,34 @@ print( PY } +run_low_impact() { + local priority_command=() + + # Backups are mandatory, but their first byte-complete copy and SHA-256 + # verification must not starve the live Unraid services. BusyBox hosts do + # not always provide both tools, so use every available scheduler without + # weakening the backup when one is absent. + if command -v ionice >/dev/null 2>&1; then + priority_command+=(ionice -c 2 -n 7) + fi + if command -v nice >/dev/null 2>&1; then + priority_command+=(nice -n 10) + fi + if [ "${#priority_command[@]}" -eq 0 ]; then + "$@" + return + fi + "${priority_command[@]}" "$@" +} + select_verified_link_dest() { local candidate="" GEOINTEL_BACKUP_LINK_DEST="" while IFS= read -r candidate; do if ( cd "$candidate" \ - && sha256sum -c CHECKSUMS.sha256 >/dev/null \ - && python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate" + && run_low_impact sha256sum -c CHECKSUMS.sha256 >/dev/null \ + && run_low_impact python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate" ); then GEOINTEL_BACKUP_LINK_DEST="$candidate" echo "Using verified prior byte snapshot as link-dest: ${candidate}" @@ -482,7 +502,7 @@ PY GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}" echo "Creating mandatory pre-deploy backup ${release_id}..." - if ! bash scripts/backup_release_state.sh \ + if ! run_low_impact bash scripts/backup_release_state.sh \ --container geointel \ --output-root "$GEOINTEL_BACKUPS_PATH" \ --release-id "$release_id" \ @@ -491,7 +511,7 @@ PY --inventory-mode sha256 \ --rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \ "${backup_link_args[@]}" \ - || ! bash scripts/verify_release_backup.sh \ + || ! run_low_impact bash scripts/verify_release_backup.sh \ --container geointel \ --backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then echo "Pre-deploy backup failed; restarting the unchanged current release." >&2 diff --git a/docs/RELEASE_RUNBOOK.md b/docs/RELEASE_RUNBOOK.md index 0b9253a7..6d7343a9 100644 --- a/docs/RELEASE_RUNBOOK.md +++ b/docs/RELEASE_RUNBOOK.md @@ -80,6 +80,15 @@ Create an immutable byte-complete backup. Initial storage/model copy and verification can be I/O-heavy; subsequent backups deduplicate unchanged bytes against the newest verified prior snapshot: +The gated Unraid job keeps its controller request attached for up to twelve +hours because an initial snapshot can exceed 100 GiB. Release runs for the same +branch queue instead of cancelling an in-flight deploy. Snapshot copying and +checksum verification use the lowest available best-effort CPU and I/O +priority (`nice` and `ionice`) while the deploy lock prevents overlap. If an +operator manually cancels the caller, inspect the controller state and +`/mnt/user/appdata/gitea-deploy-runners/deploy.lock` before retrying; never +assume that cancelling the HTTP client terminated the server-side operation. + ```bash bash scripts/backup_release_state.sh \ --container geointel \