68 lines
2.6 KiB
Bash
68 lines
2.6 KiB
Bash
#!/usr/bin/env sh
|
|
# Scheduled ModelForge backup.
|
|
#
|
|
# Creates a backup set, verifies it and applies retention, in that order. Deployment
|
|
# infrastructure owns the schedule — cron, a systemd timer, an Unraid User Script or the
|
|
# `backup-scheduler` service in docker-compose.backup.yml — so the platform never depends on an
|
|
# operator being present to protect its authoritative state.
|
|
#
|
|
# The backup id is derived from the interval so a re-run inside the same interval is a no-op rather
|
|
# than a second copy. A non-zero exit means the platform has no *new* verified recovery point, which
|
|
# is what the BACKUP_STALE and BACKUP_FAILED alerts exist to catch.
|
|
set -eu
|
|
|
|
BASE_URL="${MODELFORGE_BASE_URL:-http://api:8000}"
|
|
TOKEN="${MODELFORGE_OPERATOR_API_KEY:?operator API key is required for scheduled backups}"
|
|
PREFIX="${MODELFORGE_BACKUP_ID_PREFIX:-scheduled}"
|
|
STAMP="$(date -u +%Y%m%d-%H%M)"
|
|
BACKUP_ID="${PREFIX}-${STAMP}"
|
|
REASON="${MODELFORGE_BACKUP_REASON:-Scheduled ModelForge control-plane backup}"
|
|
|
|
log() { printf '%s modelforge-backup %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; }
|
|
|
|
# Extract the first occurrence of a top-level JSON string field. Deliberately not greedy: the
|
|
# response embeds nested objects and a greedy match would read the wrong one.
|
|
field() {
|
|
printf '%s' "$2" | grep -o "\"$1\":\"[^\"]*\"" | head -n 1 | cut -d'"' -f4
|
|
}
|
|
|
|
api() {
|
|
method="$1"
|
|
path="$2"
|
|
shift 2
|
|
curl -sS -X "${method}" -H "X-ModelForge-Admin-Token: ${TOKEN}" \
|
|
-H "Content-Type: application/json" "${BASE_URL}${path}" "$@"
|
|
}
|
|
|
|
log "creating ${BACKUP_ID}"
|
|
created="$(api POST /api/v1/admin/recovery/backups \
|
|
--data "{\"backup_id\":\"${BACKUP_ID}\",\"reason\":\"${REASON}\"}")"
|
|
|
|
backup_uuid="$(field id "${created}")"
|
|
if [ -z "${backup_uuid}" ]; then
|
|
log "backup creation did not return an id: ${created}"
|
|
exit 1
|
|
fi
|
|
|
|
state="$(field state "${created}")"
|
|
log "created ${BACKUP_ID} (${backup_uuid}) state=${state}"
|
|
if [ "${state}" = "FAILED" ]; then
|
|
log "backup failed; leaving it journalled as evidence"
|
|
exit 1
|
|
fi
|
|
|
|
log "verifying ${BACKUP_ID}"
|
|
verified="$(api POST "/api/v1/admin/recovery/backups/${backup_uuid}/verify")"
|
|
verified_state="$(field state "${verified}")"
|
|
log "verification result state=${verified_state}"
|
|
if [ "${verified_state}" != "VERIFIED" ]; then
|
|
log "backup is not restore eligible; see RUNBOOK_BACKUP_FAILURE.md"
|
|
exit 1
|
|
fi
|
|
|
|
# Retention runs only after a new verified backup exists, so the last known-good recovery point can
|
|
# never be expired in favour of one that has not proven itself.
|
|
log "applying retention"
|
|
api POST /api/v1/admin/recovery/retention/run > /dev/null
|
|
log "scheduled backup complete: ${BACKUP_ID} VERIFIED"
|