#!/usr/bin/env sh # Scheduled ModelForge backup. # # Creates a backup set, verifies it and applies retention, in that order. Deployment # infrastructure owns the schedule — cron, a systemd timer, an Unraid User Script or the # `backup-scheduler` service in docker-compose.backup.yml — so the platform never depends on an # operator being present to protect its authoritative state. # # The backup id is derived from the interval so a re-run inside the same interval is a no-op rather # than a second copy. A non-zero exit means the platform has no *new* verified recovery point, which # is what the BACKUP_STALE and BACKUP_FAILED alerts exist to catch. set -eu BASE_URL="${MODELFORGE_BASE_URL:-http://api:8000}" TOKEN="${MODELFORGE_OPERATOR_API_KEY:?operator API key is required for scheduled backups}" PREFIX="${MODELFORGE_BACKUP_ID_PREFIX:-scheduled}" STAMP="$(date -u +%Y%m%d-%H%M)" BACKUP_ID="${PREFIX}-${STAMP}" REASON="${MODELFORGE_BACKUP_REASON:-Scheduled ModelForge control-plane backup}" log() { printf '%s modelforge-backup %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; } # Extract the first occurrence of a top-level JSON string field. Deliberately not greedy: the # response embeds nested objects and a greedy match would read the wrong one. field() { printf '%s' "$2" | grep -o "\"$1\":\"[^\"]*\"" | head -n 1 | cut -d'"' -f4 } api() { method="$1" path="$2" shift 2 curl -sS -X "${method}" -H "X-ModelForge-Admin-Token: ${TOKEN}" \ -H "Content-Type: application/json" "${BASE_URL}${path}" "$@" } log "creating ${BACKUP_ID}" created="$(api POST /api/v1/admin/recovery/backups \ --data "{\"backup_id\":\"${BACKUP_ID}\",\"reason\":\"${REASON}\"}")" backup_uuid="$(field id "${created}")" if [ -z "${backup_uuid}" ]; then log "backup creation did not return an id: ${created}" exit 1 fi state="$(field state "${created}")" log "created ${BACKUP_ID} (${backup_uuid}) state=${state}" if [ "${state}" = "FAILED" ]; then log "backup failed; leaving it journalled as evidence" exit 1 fi log "verifying ${BACKUP_ID}" verified="$(api POST "/api/v1/admin/recovery/backups/${backup_uuid}/verify")" verified_state="$(field state "${verified}")" log "verification result state=${verified_state}" if [ "${verified_state}" != "VERIFIED" ]; then log "backup is not restore eligible; see RUNBOOK_BACKUP_FAILURE.md" exit 1 fi # Retention runs only after a new verified backup exists, so the last known-good recovery point can # never be expired in favour of one that has not proven itself. log "applying retention" api POST /api/v1/admin/recovery/retention/run > /dev/null log "scheduled backup complete: ${BACKUP_ID} VERIFIED"