fix(release): keep large snapshots attached and low impact
Managed validation / ${{ inputs.profile || 'full' }} (pull_request) Failing after 1m35s
GeoIntel release gates / Compile, test, contracts and builds (pull_request) Failing after 1m43s
GeoIntel release gates / Python and npm vulnerability policy (pull_request) Failing after 32s
GeoIntel release gates / Production AI image, SBOM and container scan (pull_request) Failing after 31s
GeoIntel release gates / Deploy exact gated revision to Unraid (pull_request) Skipped

This commit is contained in:
Jens
2026-08-30 10:09:26 +02:00
parent f77f6f2d93
commit fd58cb2378
4 changed files with 55 additions and 6 deletions
+8 -2
View File
@@ -11,7 +11,10 @@ permissions:
concurrency: concurrency:
group: geointel-release-${{ gitea.ref }} group: geointel-release-${{ gitea.ref }}
cancel-in-progress: true # A cancelled HTTP caller does not terminate the allowlisted controller
# process that already owns the production lock. Queue a newer revision
# instead of orphaning an in-flight backup or deploy.
cancel-in-progress: false
jobs: jobs:
quality: quality:
@@ -162,7 +165,10 @@ jobs:
needs: [quality, dependency-audit, container] needs: [quality, dependency-audit, container]
if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }} if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }}
runs-on: unraid-deploy runs-on: unraid-deploy
timeout-minutes: 180 # The first byte-complete storage snapshot can exceed 100 GiB. Keep the
# gated caller attached for the full conservative backup/build window;
# the controller and deploy script still serialize every mutation.
timeout-minutes: 720
steps: steps:
- name: Deploy only after every release gate is green - name: Deploy only after every release gate is green
run: | run: |
@@ -53,6 +53,20 @@ def test_release_creates_verified_backup_before_candidate_migrations() -> None:
assert "preflight_backup_capacity" in script assert "preflight_backup_capacity" in script
assert "select_verified_link_dest" in script assert "select_verified_link_dest" in script
assert "release_backup_snapshot.py" in script assert "release_backup_snapshot.py" in script
assert "run_low_impact()" in script
assert "ionice -c 2 -n 7" in script
assert "nice -n 10" in script
assert "run_low_impact bash scripts/backup_release_state.sh" in script
assert "run_low_impact bash scripts/verify_release_backup.sh" in script
def test_gitea_deploy_waits_for_the_mandatory_large_snapshot() -> None:
workflow = (ROOT / ".gitea" / "workflows" / "release-gates.yml").read_text(encoding="utf-8")
assert "cancel-in-progress: false" in workflow
deploy_job = workflow.split("\n deploy:\n", maxsplit=1)[1]
assert "timeout-minutes: 720" in deploy_job
assert "docker exec gitea-deploy-control" in deploy_job
def test_release_starts_only_the_locally_attested_ai_image() -> None: def test_release_starts_only_the_locally_attested_ai_image() -> None:
+24 -4
View File
@@ -355,14 +355,34 @@ print(
PY PY
} }
run_low_impact() {
local priority_command=()
# Backups are mandatory, but their first byte-complete copy and SHA-256
# verification must not starve the live Unraid services. BusyBox hosts do
# not always provide both tools, so use every available scheduler without
# weakening the backup when one is absent.
if command -v ionice >/dev/null 2>&1; then
priority_command+=(ionice -c 2 -n 7)
fi
if command -v nice >/dev/null 2>&1; then
priority_command+=(nice -n 10)
fi
if [ "${#priority_command[@]}" -eq 0 ]; then
"$@"
return
fi
"${priority_command[@]}" "$@"
}
select_verified_link_dest() { select_verified_link_dest() {
local candidate="" local candidate=""
GEOINTEL_BACKUP_LINK_DEST="" GEOINTEL_BACKUP_LINK_DEST=""
while IFS= read -r candidate; do while IFS= read -r candidate; do
if ( if (
cd "$candidate" \ cd "$candidate" \
&& sha256sum -c CHECKSUMS.sha256 >/dev/null \ && run_low_impact sha256sum -c CHECKSUMS.sha256 >/dev/null \
&& python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate" && run_low_impact python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate"
); then ); then
GEOINTEL_BACKUP_LINK_DEST="$candidate" GEOINTEL_BACKUP_LINK_DEST="$candidate"
echo "Using verified prior byte snapshot as link-dest: ${candidate}" echo "Using verified prior byte snapshot as link-dest: ${candidate}"
@@ -482,7 +502,7 @@ PY
GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}" GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}"
echo "Creating mandatory pre-deploy backup ${release_id}..." echo "Creating mandatory pre-deploy backup ${release_id}..."
if ! bash scripts/backup_release_state.sh \ if ! run_low_impact bash scripts/backup_release_state.sh \
--container geointel \ --container geointel \
--output-root "$GEOINTEL_BACKUPS_PATH" \ --output-root "$GEOINTEL_BACKUPS_PATH" \
--release-id "$release_id" \ --release-id "$release_id" \
@@ -491,7 +511,7 @@ PY
--inventory-mode sha256 \ --inventory-mode sha256 \
--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \ --rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \
"${backup_link_args[@]}" \ "${backup_link_args[@]}" \
|| ! bash scripts/verify_release_backup.sh \ || ! run_low_impact bash scripts/verify_release_backup.sh \
--container geointel \ --container geointel \
--backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then --backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then
echo "Pre-deploy backup failed; restarting the unchanged current release." >&2 echo "Pre-deploy backup failed; restarting the unchanged current release." >&2
+9
View File
@@ -80,6 +80,15 @@ Create an immutable byte-complete backup. Initial storage/model copy and
verification can be I/O-heavy; subsequent backups deduplicate unchanged bytes verification can be I/O-heavy; subsequent backups deduplicate unchanged bytes
against the newest verified prior snapshot: against the newest verified prior snapshot:
The gated Unraid job keeps its controller request attached for up to twelve
hours because an initial snapshot can exceed 100 GiB. Release runs for the same
branch queue instead of cancelling an in-flight deploy. Snapshot copying and
checksum verification use the lowest available best-effort CPU and I/O
priority (`nice` and `ionice`) while the deploy lock prevents overlap. If an
operator manually cancels the caller, inspect the controller state and
`/mnt/user/appdata/gitea-deploy-runners/deploy.lock` before retrying; never
assume that cancelling the HTTP client terminated the server-side operation.
```bash ```bash
bash scripts/backup_release_state.sh \ bash scripts/backup_release_state.sh \
--container geointel \ --container geointel \