Initial public ModelForge release

2026-09-01 21:30:16 +02:00
commit 7082ab955a
490 changed files with 104252 additions and 0 deletions
@@ -0,0 +1,67 @@
+#!/usr/bin/env sh
+# Scheduled ModelForge backup.
+#
+# Creates a backup set, verifies it and applies retention, in that order. Deployment
+# infrastructure owns the schedule — cron, a systemd timer, an Unraid User Script or the
+# `backup-scheduler` service in docker-compose.backup.yml — so the platform never depends on an
+# operator being present to protect its authoritative state.
+#
+# The backup id is derived from the interval so a re-run inside the same interval is a no-op rather
+# than a second copy. A non-zero exit means the platform has no *new* verified recovery point, which
+# is what the BACKUP_STALE and BACKUP_FAILED alerts exist to catch.
+set -eu
+
+BASE_URL="${MODELFORGE_BASE_URL:-http://api:8000}"
+TOKEN="${MODELFORGE_OPERATOR_API_KEY:?operator API key is required for scheduled backups}"
+PREFIX="${MODELFORGE_BACKUP_ID_PREFIX:-scheduled}"
+STAMP="$(date -u +%Y%m%d-%H%M)"
+BACKUP_ID="${PREFIX}-${STAMP}"
+REASON="${MODELFORGE_BACKUP_REASON:-Scheduled ModelForge control-plane backup}"
+
+log() { printf '%s modelforge-backup %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; }
+
+# Extract the first occurrence of a top-level JSON string field. Deliberately not greedy: the
+# response embeds nested objects and a greedy match would read the wrong one.
+field() {
+  printf '%s' "$2" | grep -o "\"$1\":\"[^\"]*\"" | head -n 1 | cut -d'"' -f4
+}
+
+api() {
+  method="$1"
+  path="$2"
+  shift 2
+  curl -sS -X "${method}" -H "X-ModelForge-Admin-Token: ${TOKEN}" \
+    -H "Content-Type: application/json" "${BASE_URL}${path}" "$@"
+}
+
+log "creating ${BACKUP_ID}"
+created="$(api POST /api/v1/admin/recovery/backups \
+  --data "{\"backup_id\":\"${BACKUP_ID}\",\"reason\":\"${REASON}\"}")"
+
+backup_uuid="$(field id "${created}")"
+if [ -z "${backup_uuid}" ]; then
+  log "backup creation did not return an id: ${created}"
+  exit 1
+fi
+
+state="$(field state "${created}")"
+log "created ${BACKUP_ID} (${backup_uuid}) state=${state}"
+if [ "${state}" = "FAILED" ]; then
+  log "backup failed; leaving it journalled as evidence"
+  exit 1
+fi
+
+log "verifying ${BACKUP_ID}"
+verified="$(api POST "/api/v1/admin/recovery/backups/${backup_uuid}/verify")"
+verified_state="$(field state "${verified}")"
+log "verification result state=${verified_state}"
+if [ "${verified_state}" != "VERIFIED" ]; then
+  log "backup is not restore eligible; see RUNBOOK_BACKUP_FAILURE.md"
+  exit 1
+fi
+
+# Retention runs only after a new verified backup exists, so the last known-good recovery point can
+# never be expired in favour of one that has not proven itself.
+log "applying retention"
+api POST /api/v1/admin/recovery/retention/run > /dev/null
+log "scheduled backup complete: ${BACKUP_ID} VERIFIED"