Full platform remediation #7

Merged
Jens merged 11 commits from codex/full-platform-remediation into main 2026-08-30 09:30:42 +00:00
4 changed files with 55 additions and 6 deletions
Showing only changes of commit fd58cb2378 - Show all commits
+8 -2
View File
@@ -11,7 +11,10 @@ permissions:
concurrency:
group: geointel-release-${{ gitea.ref }}
cancel-in-progress: true
# A cancelled HTTP caller does not terminate the allowlisted controller
# process that already owns the production lock. Queue a newer revision
# instead of orphaning an in-flight backup or deploy.
cancel-in-progress: false
jobs:
quality:
@@ -162,7 +165,10 @@ jobs:
needs: [quality, dependency-audit, container]
if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }}
runs-on: unraid-deploy
timeout-minutes: 180
# The first byte-complete storage snapshot can exceed 100 GiB. Keep the
# gated caller attached for the full conservative backup/build window;
# the controller and deploy script still serialize every mutation.
timeout-minutes: 720
steps:
- name: Deploy only after every release gate is green
run: |
@@ -53,6 +53,20 @@ def test_release_creates_verified_backup_before_candidate_migrations() -> None:
assert "preflight_backup_capacity" in script
assert "select_verified_link_dest" in script
assert "release_backup_snapshot.py" in script
assert "run_low_impact()" in script
assert "ionice -c 2 -n 7" in script
assert "nice -n 10" in script
assert "run_low_impact bash scripts/backup_release_state.sh" in script
assert "run_low_impact bash scripts/verify_release_backup.sh" in script
def test_gitea_deploy_waits_for_the_mandatory_large_snapshot() -> None:
workflow = (ROOT / ".gitea" / "workflows" / "release-gates.yml").read_text(encoding="utf-8")
assert "cancel-in-progress: false" in workflow
deploy_job = workflow.split("\n deploy:\n", maxsplit=1)[1]
assert "timeout-minutes: 720" in deploy_job
assert "docker exec gitea-deploy-control" in deploy_job
def test_release_starts_only_the_locally_attested_ai_image() -> None:
+24 -4
View File
@@ -355,14 +355,34 @@ print(
PY
}
run_low_impact() {
local priority_command=()
# Backups are mandatory, but their first byte-complete copy and SHA-256
# verification must not starve the live Unraid services. BusyBox hosts do
# not always provide both tools, so use every available scheduler without
# weakening the backup when one is absent.
if command -v ionice >/dev/null 2>&1; then
priority_command+=(ionice -c 2 -n 7)
fi
if command -v nice >/dev/null 2>&1; then
priority_command+=(nice -n 10)
fi
if [ "${#priority_command[@]}" -eq 0 ]; then
"$@"
return
fi
"${priority_command[@]}" "$@"
}
select_verified_link_dest() {
local candidate=""
GEOINTEL_BACKUP_LINK_DEST=""
while IFS= read -r candidate; do
if (
cd "$candidate" \
&& sha256sum -c CHECKSUMS.sha256 >/dev/null \
&& python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate"
&& run_low_impact sha256sum -c CHECKSUMS.sha256 >/dev/null \
&& run_low_impact python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate"
); then
GEOINTEL_BACKUP_LINK_DEST="$candidate"
echo "Using verified prior byte snapshot as link-dest: ${candidate}"
@@ -482,7 +502,7 @@ PY
GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}"
echo "Creating mandatory pre-deploy backup ${release_id}..."
if ! bash scripts/backup_release_state.sh \
if ! run_low_impact bash scripts/backup_release_state.sh \
--container geointel \
--output-root "$GEOINTEL_BACKUPS_PATH" \
--release-id "$release_id" \
@@ -491,7 +511,7 @@ PY
--inventory-mode sha256 \
--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \
"${backup_link_args[@]}" \
|| ! bash scripts/verify_release_backup.sh \
|| ! run_low_impact bash scripts/verify_release_backup.sh \
--container geointel \
--backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then
echo "Pre-deploy backup failed; restarting the unchanged current release." >&2
+9
View File
@@ -80,6 +80,15 @@ Create an immutable byte-complete backup. Initial storage/model copy and
verification can be I/O-heavy; subsequent backups deduplicate unchanged bytes
against the newest verified prior snapshot:
The gated Unraid job keeps its controller request attached for up to twelve
hours because an initial snapshot can exceed 100 GiB. Release runs for the same
branch queue instead of cancelling an in-flight deploy. Snapshot copying and
checksum verification use the lowest available best-effort CPU and I/O
priority (`nice` and `ionice`) while the deploy lock prevents overlap. If an
operator manually cancels the caller, inspect the controller state and
`/mnt/user/appdata/gitea-deploy-runners/deploy.lock` before retrying; never
assume that cancelling the HTTP client terminated the server-side operation.
```bash
bash scripts/backup_release_state.sh \
--container geointel \