fix(release): keep large snapshots attached and low impact
Managed validation / ${{ inputs.profile || 'full' }} (pull_request) Failing after 1m35s
GeoIntel release gates / Compile, test, contracts and builds (pull_request) Failing after 1m43s
GeoIntel release gates / Python and npm vulnerability policy (pull_request) Failing after 32s
GeoIntel release gates / Production AI image, SBOM and container scan (pull_request) Failing after 31s
GeoIntel release gates / Deploy exact gated revision to Unraid (pull_request) Skipped
Managed validation / ${{ inputs.profile || 'full' }} (pull_request) Failing after 1m35s
GeoIntel release gates / Compile, test, contracts and builds (pull_request) Failing after 1m43s
GeoIntel release gates / Python and npm vulnerability policy (pull_request) Failing after 32s
GeoIntel release gates / Production AI image, SBOM and container scan (pull_request) Failing after 31s
GeoIntel release gates / Deploy exact gated revision to Unraid (pull_request) Skipped
This commit is contained in:
@@ -11,7 +11,10 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: geointel-release-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
# A cancelled HTTP caller does not terminate the allowlisted controller
|
||||
# process that already owns the production lock. Queue a newer revision
|
||||
# instead of orphaning an in-flight backup or deploy.
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
quality:
|
||||
@@ -162,7 +165,10 @@ jobs:
|
||||
needs: [quality, dependency-audit, container]
|
||||
if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }}
|
||||
runs-on: unraid-deploy
|
||||
timeout-minutes: 180
|
||||
# The first byte-complete storage snapshot can exceed 100 GiB. Keep the
|
||||
# gated caller attached for the full conservative backup/build window;
|
||||
# the controller and deploy script still serialize every mutation.
|
||||
timeout-minutes: 720
|
||||
steps:
|
||||
- name: Deploy only after every release gate is green
|
||||
run: |
|
||||
|
||||
@@ -53,6 +53,20 @@ def test_release_creates_verified_backup_before_candidate_migrations() -> None:
|
||||
assert "preflight_backup_capacity" in script
|
||||
assert "select_verified_link_dest" in script
|
||||
assert "release_backup_snapshot.py" in script
|
||||
assert "run_low_impact()" in script
|
||||
assert "ionice -c 2 -n 7" in script
|
||||
assert "nice -n 10" in script
|
||||
assert "run_low_impact bash scripts/backup_release_state.sh" in script
|
||||
assert "run_low_impact bash scripts/verify_release_backup.sh" in script
|
||||
|
||||
|
||||
def test_gitea_deploy_waits_for_the_mandatory_large_snapshot() -> None:
|
||||
workflow = (ROOT / ".gitea" / "workflows" / "release-gates.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert "cancel-in-progress: false" in workflow
|
||||
deploy_job = workflow.split("\n deploy:\n", maxsplit=1)[1]
|
||||
assert "timeout-minutes: 720" in deploy_job
|
||||
assert "docker exec gitea-deploy-control" in deploy_job
|
||||
|
||||
|
||||
def test_release_starts_only_the_locally_attested_ai_image() -> None:
|
||||
|
||||
@@ -355,14 +355,34 @@ print(
|
||||
PY
|
||||
}
|
||||
|
||||
run_low_impact() {
|
||||
local priority_command=()
|
||||
|
||||
# Backups are mandatory, but their first byte-complete copy and SHA-256
|
||||
# verification must not starve the live Unraid services. BusyBox hosts do
|
||||
# not always provide both tools, so use every available scheduler without
|
||||
# weakening the backup when one is absent.
|
||||
if command -v ionice >/dev/null 2>&1; then
|
||||
priority_command+=(ionice -c 2 -n 7)
|
||||
fi
|
||||
if command -v nice >/dev/null 2>&1; then
|
||||
priority_command+=(nice -n 10)
|
||||
fi
|
||||
if [ "${#priority_command[@]}" -eq 0 ]; then
|
||||
"$@"
|
||||
return
|
||||
fi
|
||||
"${priority_command[@]}" "$@"
|
||||
}
|
||||
|
||||
select_verified_link_dest() {
|
||||
local candidate=""
|
||||
GEOINTEL_BACKUP_LINK_DEST=""
|
||||
while IFS= read -r candidate; do
|
||||
if (
|
||||
cd "$candidate" \
|
||||
&& sha256sum -c CHECKSUMS.sha256 >/dev/null \
|
||||
&& python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate"
|
||||
&& run_low_impact sha256sum -c CHECKSUMS.sha256 >/dev/null \
|
||||
&& run_low_impact python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate"
|
||||
); then
|
||||
GEOINTEL_BACKUP_LINK_DEST="$candidate"
|
||||
echo "Using verified prior byte snapshot as link-dest: ${candidate}"
|
||||
@@ -482,7 +502,7 @@ PY
|
||||
|
||||
GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}"
|
||||
echo "Creating mandatory pre-deploy backup ${release_id}..."
|
||||
if ! bash scripts/backup_release_state.sh \
|
||||
if ! run_low_impact bash scripts/backup_release_state.sh \
|
||||
--container geointel \
|
||||
--output-root "$GEOINTEL_BACKUPS_PATH" \
|
||||
--release-id "$release_id" \
|
||||
@@ -491,7 +511,7 @@ PY
|
||||
--inventory-mode sha256 \
|
||||
--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \
|
||||
"${backup_link_args[@]}" \
|
||||
|| ! bash scripts/verify_release_backup.sh \
|
||||
|| ! run_low_impact bash scripts/verify_release_backup.sh \
|
||||
--container geointel \
|
||||
--backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then
|
||||
echo "Pre-deploy backup failed; restarting the unchanged current release." >&2
|
||||
|
||||
@@ -80,6 +80,15 @@ Create an immutable byte-complete backup. Initial storage/model copy and
|
||||
verification can be I/O-heavy; subsequent backups deduplicate unchanged bytes
|
||||
against the newest verified prior snapshot:
|
||||
|
||||
The gated Unraid job keeps its controller request attached for up to twelve
|
||||
hours because an initial snapshot can exceed 100 GiB. Release runs for the same
|
||||
branch queue instead of cancelling an in-flight deploy. Snapshot copying and
|
||||
checksum verification use the lowest available best-effort CPU and I/O
|
||||
priority (`nice` and `ionice`) while the deploy lock prevents overlap. If an
|
||||
operator manually cancels the caller, inspect the controller state and
|
||||
`/mnt/user/appdata/gitea-deploy-runners/deploy.lock` before retrying; never
|
||||
assume that cancelling the HTTP client terminated the server-side operation.
|
||||
|
||||
```bash
|
||||
bash scripts/backup_release_state.sh \
|
||||
--container geointel \
|
||||
|
||||
Reference in New Issue
Block a user