M48: harden demo operations and offsite recovery
This commit is contained in:
NuklearRabbit
+174
-12
@@ -50,29 +50,191 @@ for image in "$api_image" "$web_image"; do
|
||||
done
|
||||
|
||||
compose="docker compose --env-file $root/.env -p $project -f $release_dir/compose.yaml -f $release_dir/compose.unraid.yaml -f $release_dir/compose.observability.yaml -f $release_dir/compose.release.yaml --profile observability"
|
||||
old_api_id="$(docker inspect --format '{{.Image}}' "$project-api-1" 2>/dev/null || true)"
|
||||
old_web_id="$(docker inspect --format '{{.Image}}' "$project-web-1" 2>/dev/null || true)"
|
||||
export MOBILITYOPS_API_IMAGE="$api_image" MOBILITYOPS_WEB_IMAGE="$web_image"
|
||||
$compose up --no-build -d api web backup prometheus alertmanager grafana
|
||||
gateway_config="$root/.deploy/gateway.conf"
|
||||
candidate_gateway="$(mktemp "$root/.deploy/gateway-candidate.XXXXXX")"
|
||||
previous_gateway="$(mktemp "$root/.deploy/gateway-previous.XXXXXX")"
|
||||
sed -e "s/server web:80/server web-$short_revision:80/" \
|
||||
-e "s/server api:8000/server api-$short_revision:8000/" \
|
||||
"$release_dir/deploy/unraid/gateway.conf" > "$candidate_gateway"
|
||||
if [ -f "$gateway_config" ]; then
|
||||
cp "$gateway_config" "$previous_gateway"
|
||||
else
|
||||
cp "$candidate_gateway" "$gateway_config"
|
||||
fi
|
||||
export MOBILITYOPS_GATEWAY_CONFIG="$gateway_config"
|
||||
|
||||
# Bootstrap infrastructure only when it is absent. Existing stateful/monitoring
|
||||
# containers are deliberately not reconciled against every commit-named source path.
|
||||
missing_infrastructure=""
|
||||
for service in db backup prometheus alertmanager grafana; do
|
||||
[ -n "$($compose ps -q "$service")" ] || missing_infrastructure="$missing_infrastructure $service"
|
||||
done
|
||||
if [ -n "$missing_infrastructure" ]; then
|
||||
# shellcheck disable=SC2086
|
||||
$compose up --no-build -d $missing_infrastructure
|
||||
fi
|
||||
|
||||
# Apply schema changes once before starting the new stateless replicas. Migrations in a
|
||||
# release must remain backwards compatible with the still-serving previous API.
|
||||
$compose run --rm --no-deps --entrypoint alembic api upgrade head
|
||||
|
||||
network="$(docker inspect --format '{{range $name, $network := .NetworkSettings.Networks}}{{$name}}{{end}}' "$project-db-1")"
|
||||
[ -n "$network" ] || { echo "MobilityOps network was not found" >&2; exit 1; }
|
||||
|
||||
old_api_ids="$(docker ps -q --filter label=com.mobilityops.role=api --filter "label=com.mobilityops.project=$project")"
|
||||
old_web_ids="$(docker ps -q --filter label=com.mobilityops.role=web --filter "label=com.mobilityops.project=$project")"
|
||||
if [ -z "$old_api_ids" ]; then
|
||||
old_api_ids="$(docker ps -q --filter "name=^/${project}-api-")"
|
||||
fi
|
||||
if [ -z "$old_web_ids" ]; then
|
||||
old_web_ids="$(docker ps -q --filter "name=^/${project}-web-")"
|
||||
fi
|
||||
|
||||
# Reuse the effective, already-secret-resolved API environment without printing it.
|
||||
api_environment="$(mktemp "$root/.deploy/api-environment.XXXXXX")"
|
||||
chmod 600 "$api_environment"
|
||||
cleanup() {
|
||||
rm -f "$api_environment" "$candidate_gateway" "$previous_gateway"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
if [ -n "$old_api_ids" ]; then
|
||||
# shellcheck disable=SC2086
|
||||
first_old_api="$(printf '%s\n' $old_api_ids | head -n 1)"
|
||||
docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$first_old_api" > "$api_environment"
|
||||
else
|
||||
# A clean bootstrap first lets Compose resolve the complete API environment. This
|
||||
# container is retained as the previous slot until the candidates pass.
|
||||
$compose up --no-build --no-deps -d api web
|
||||
old_api_ids="$($compose ps -q api)"
|
||||
old_web_ids="$($compose ps -q web)"
|
||||
docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$old_api_ids" > "$api_environment"
|
||||
fi
|
||||
|
||||
new_api_ids=""
|
||||
new_web_ids=""
|
||||
old_web_image=""
|
||||
# shellcheck disable=SC2086
|
||||
[ -z "$old_web_ids" ] || old_web_image="$(docker inspect --format '{{.Config.Image}}' "$(printf '%s\n' $old_web_ids | head -n 1)")"
|
||||
for replica in 1 2; do
|
||||
name="$project-api-$short_revision-$replica"
|
||||
id="$(docker run -d --name "$name" --restart unless-stopped \
|
||||
--label com.mobilityops.role=api --label "com.mobilityops.project=$project" \
|
||||
--label "com.mobilityops.revision=$revision" \
|
||||
--network "$network" --network-alias "api-$short_revision" --env-file "$api_environment" \
|
||||
--env RUN_MIGRATIONS=false "$api_image")"
|
||||
new_api_ids="$new_api_ids $id"
|
||||
done
|
||||
|
||||
wait_for_ids() {
|
||||
role="$1"
|
||||
shift
|
||||
attempt=0
|
||||
while true; do
|
||||
healthy=0
|
||||
count=0
|
||||
for id in "$@"; do
|
||||
count=$((count + 1))
|
||||
if [ "$role" = "api" ]; then
|
||||
docker exec "$id" python -c \
|
||||
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/ready')" \
|
||||
> /dev/null 2>&1 && healthy=$((healthy + 1))
|
||||
else
|
||||
docker exec "$id" wget -q -O /dev/null http://127.0.0.1/health \
|
||||
> /dev/null 2>&1 && healthy=$((healthy + 1))
|
||||
fi
|
||||
done
|
||||
[ "$count" -eq 2 ] && [ "$healthy" -eq 2 ] && return 0
|
||||
attempt=$((attempt + 1))
|
||||
[ "$attempt" -lt 60 ] || { echo "$role candidates did not become healthy" >&2; return 1; }
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
if ! wait_for_ids api $new_api_ids; then
|
||||
docker rm -f $new_api_ids > /dev/null 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for replica in 1 2; do
|
||||
name="$project-web-$short_revision-$replica"
|
||||
id="$(docker run -d --name "$name" --restart unless-stopped \
|
||||
--label com.mobilityops.role=web --label "com.mobilityops.project=$project" \
|
||||
--label "com.mobilityops.revision=$revision" \
|
||||
--network "$network" --network-alias "web-$short_revision" "$web_image")"
|
||||
new_web_ids="$new_web_ids $id"
|
||||
done
|
||||
# shellcheck disable=SC2086
|
||||
if ! wait_for_ids web $new_web_ids; then
|
||||
docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Install the stable gateway once. Future releases leave it running while Docker DNS
|
||||
# exposes both old and new web aliases. The first migration from direct port ownership
|
||||
# necessarily has a brief hand-off while port 1236 moves to the gateway.
|
||||
gateway_id="$($compose ps -q gateway)"
|
||||
cp "$candidate_gateway" "$gateway_config"
|
||||
if [ -z "$gateway_id" ]; then
|
||||
if [ -n "$old_web_ids" ]; then
|
||||
# shellcheck disable=SC2086
|
||||
docker rm -f $old_web_ids > /dev/null
|
||||
old_web_ids=""
|
||||
fi
|
||||
$compose up --no-build --no-deps -d gateway
|
||||
else
|
||||
if ! docker exec "$gateway_id" nginx -t; then
|
||||
[ ! -s "$previous_gateway" ] || cp "$previous_gateway" "$gateway_config"
|
||||
# shellcheck disable=SC2086
|
||||
docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
|
||||
echo "Candidate gateway configuration was rejected; previous replicas remain active" >&2
|
||||
exit 1
|
||||
fi
|
||||
docker exec "$gateway_id" nginx -s reload
|
||||
fi
|
||||
|
||||
attempt=0
|
||||
until curl -fsS http://127.0.0.1:1236/health/ready > /dev/null; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 30 ]; then
|
||||
if [ -n "$old_api_id" ] && [ -n "$old_web_id" ]; then
|
||||
rollback_api="mobilityops-api:rollback-$short_revision"
|
||||
rollback_web="mobilityops-web:rollback-$short_revision"
|
||||
docker tag "$old_api_id" "$rollback_api"
|
||||
docker tag "$old_web_id" "$rollback_web"
|
||||
export MOBILITYOPS_API_IMAGE="$rollback_api" MOBILITYOPS_WEB_IMAGE="$rollback_web"
|
||||
$compose up --no-build -d api web || true
|
||||
if [ -n "$gateway_id" ] && [ -s "$previous_gateway" ]; then
|
||||
cp "$previous_gateway" "$gateway_config"
|
||||
docker exec "$gateway_id" nginx -t && docker exec "$gateway_id" nginx -s reload
|
||||
# shellcheck disable=SC2086
|
||||
docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
|
||||
elif [ -n "$old_web_image" ]; then
|
||||
# The first gateway migration keeps the already healthy candidate APIs because
|
||||
# the rendered gateway points at their versioned alias.
|
||||
# shellcheck disable=SC2086
|
||||
docker rm -f $new_web_ids > /dev/null 2>&1 || true
|
||||
for replica in 1 2; do
|
||||
docker run -d --name "$project-web-rollback-$short_revision-$replica" --restart unless-stopped \
|
||||
--label com.mobilityops.role=web --label "com.mobilityops.project=$project" \
|
||||
--label com.mobilityops.revision=rollback \
|
||||
--network "$network" --network-alias "web-$short_revision" "$old_web_image" > /dev/null
|
||||
done
|
||||
else
|
||||
# shellcheck disable=SC2086
|
||||
docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
|
||||
fi
|
||||
echo "Release failed readiness; source revision was not promoted" >&2
|
||||
echo "Release failed readiness; previous API and restored web replicas remain active" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
$compose exec -T api alembic current
|
||||
|
||||
# Promotion is start-first: only now remove the previous containers. A short drain lets
|
||||
# workers using the pre-reload configuration complete their in-flight requests.
|
||||
sleep 3
|
||||
# shellcheck disable=SC2086
|
||||
[ -z "$old_api_ids" ] || docker rm -f $old_api_ids > /dev/null
|
||||
# shellcheck disable=SC2086
|
||||
[ -z "$old_web_ids" ] || docker rm -f $old_web_ids > /dev/null
|
||||
curl -fsS http://127.0.0.1:1236/health/ready > /dev/null
|
||||
# shellcheck disable=SC2086
|
||||
first_new_api="$(printf '%s\n' $new_api_ids | head -n 1)"
|
||||
docker exec "$first_new_api" alembic current
|
||||
for service in backup prometheus alertmanager grafana; do
|
||||
attempt=0
|
||||
until status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$project-$service-1" 2>/dev/null)" \
|
||||
|
||||
Reference in New Issue
Block a user