#!/bin/sh set -eu # Stage a committed source archive, build commit-labelled immutable images and promote # them without overlaying source files or reseeding persisted data. archive="${1:-}" expected_checksum="${2:-}" revision="${3:-}" root="${MOBILITYOPS_DEPLOY_ROOT:-/mnt/user/appdata/mobilityops}" project="${COMPOSE_PROJECT_NAME:-mobilityops}" [ "${#revision}" -eq 40 ] || { echo "Revision must be a full Git SHA" >&2; exit 1; } case "$revision" in *[!0-9a-f]*) echo "Revision must be lowercase hexadecimal" >&2; exit 1;; esac [ -f "$archive" ] || { echo "Archive not found: $archive" >&2; exit 1; } [ -n "$expected_checksum" ] || { echo "Expected SHA-256 is required" >&2; exit 1; } [ -f "$root/.env" ] || { echo "Production .env is missing" >&2; exit 1; } read_env_value() { key="$1" awk -v key="$key" ' index($0, key "=") == 1 { sub("^[^=]*=", "") print exit } ' "$root/.env" } # Stateless replicas normally inherit the complete, already-secret-resolved environment # from the serving API. RAGcore routing is an explicit exception: its private endpoint and # cross-project Docker network are non-secret deployment topology and the server .env is # authoritative so infrastructure changes take effect on the next release. ragcore_base_url="$(read_env_value RAGCORE_BASE_URL)" ragcore_network="$(read_env_value RAGCORE_DOCKER_NETWORK)" case "$ragcore_base_url" in http://*|https://*) ;; *) echo "RAGCORE_BASE_URL must be an absolute HTTP(S) URL" >&2; exit 1 ;; esac case "$ragcore_network" in "") ;; *[!a-zA-Z0-9_.-]*) echo "RAGCORE_DOCKER_NETWORK contains invalid characters" >&2; exit 1 ;; *) docker network inspect "$ragcore_network" > /dev/null ;; esac actual_checksum="$(sha256sum "$archive" | awk '{print $1}')" [ "$actual_checksum" = "$expected_checksum" ] || { echo "Archive checksum mismatch" >&2 exit 1 } release_root="$root/.deploy/releases" release_dir="$release_root/$revision" mkdir -p "$release_root" if [ -e "$release_dir" ]; then echo "Release directory already exists; refusing to overwrite: $release_dir" >&2 exit 1 fi mkdir "$release_dir" tar -xzf "$archive" -C "$release_dir" short_revision="$(printf '%s' "$revision" | cut -c1-12)" build_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)" api_image="mobilityops-api:$short_revision" web_image="mobilityops-web:$short_revision" docker build --target runtime --build-arg "VCS_REF=$revision" --build-arg "BUILD_DATE=$build_date" \ --tag "$api_image" --file "$release_dir/backend/Dockerfile" "$release_dir" docker build --build-arg "VCS_REF=$revision" --build-arg "BUILD_DATE=$build_date" \ --tag "$web_image" "$release_dir/frontend" for image in "$api_image" "$web_image"; do labelled_revision="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image")" [ "$labelled_revision" = "$revision" ] || { echo "Image revision label mismatch for $image" >&2 exit 1 } done compose="docker compose --env-file $root/.env -p $project -f $release_dir/compose.yaml -f $release_dir/compose.unraid.yaml -f $release_dir/compose.observability.yaml -f $release_dir/compose.release.yaml --profile observability" export MOBILITYOPS_API_IMAGE="$api_image" MOBILITYOPS_WEB_IMAGE="$web_image" gateway_config="$root/.deploy/gateway.conf" candidate_gateway="$(mktemp "$root/.deploy/gateway-candidate.XXXXXX")" previous_gateway="$(mktemp "$root/.deploy/gateway-previous.XXXXXX")" sed -e "s/server web:80/server web-$short_revision:80/" \ -e "s/server api:8000/server api-$short_revision:8000/" \ "$release_dir/deploy/unraid/gateway.conf" > "$candidate_gateway" if [ -f "$gateway_config" ]; then cp "$gateway_config" "$previous_gateway" else cp "$candidate_gateway" "$gateway_config" fi export MOBILITYOPS_GATEWAY_CONFIG="$gateway_config" # Bootstrap infrastructure only when it is absent. Existing stateful/monitoring # containers are deliberately not reconciled against every commit-named source path. missing_infrastructure="" for service in db backup prometheus alertmanager grafana; do [ -n "$($compose ps -q "$service")" ] || missing_infrastructure="$missing_infrastructure $service" done if [ -n "$missing_infrastructure" ]; then # shellcheck disable=SC2086 $compose up --no-build -d $missing_infrastructure fi # Apply schema changes once before starting the new stateless replicas. Migrations in a # release must remain backwards compatible with the still-serving previous API. $compose run --rm --no-deps --entrypoint alembic api upgrade head network="$(docker inspect --format '{{range $name, $network := .NetworkSettings.Networks}}{{$name}}{{end}}' "$project-db-1")" [ -n "$network" ] || { echo "MobilityOps network was not found" >&2; exit 1; } old_api_ids="$(docker ps -q --filter label=com.mobilityops.role=api --filter "label=com.mobilityops.project=$project")" old_web_ids="$(docker ps -q --filter label=com.mobilityops.role=web --filter "label=com.mobilityops.project=$project")" if [ -z "$old_api_ids" ]; then old_api_ids="$(docker ps -q --filter "name=^/${project}-api-")" fi if [ -z "$old_web_ids" ]; then old_web_ids="$(docker ps -q --filter "name=^/${project}-web-")" fi # Reuse the effective, already-secret-resolved API environment without printing it. api_environment="$(mktemp "$root/.deploy/api-environment.XXXXXX")" chmod 600 "$api_environment" cleanup() { rm -f "$api_environment" "$candidate_gateway" "$previous_gateway" } trap cleanup EXIT INT TERM if [ -n "$old_api_ids" ]; then # shellcheck disable=SC2086 first_old_api="$(printf '%s\n' $old_api_ids | head -n 1)" docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$first_old_api" > "$api_environment" else # A clean bootstrap first lets Compose resolve the complete API environment. This # container is retained as the previous slot until the candidates pass. $compose up --no-build --no-deps -d api web old_api_ids="$($compose ps -q api)" old_web_ids="$($compose ps -q web)" docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$old_api_ids" > "$api_environment" fi new_api_ids="" new_web_ids="" old_web_image="" # shellcheck disable=SC2086 [ -z "$old_web_ids" ] || old_web_image="$(docker inspect --format '{{.Config.Image}}' "$(printf '%s\n' $old_web_ids | head -n 1)")" for replica in 1 2; do name="$project-api-$short_revision-$replica" id="$(docker run -d --name "$name" --restart unless-stopped \ --label com.mobilityops.role=api --label "com.mobilityops.project=$project" \ --label "com.mobilityops.revision=$revision" \ --network "$network" --network-alias api --network-alias "api-$short_revision" \ --env-file "$api_environment" \ --env RUN_MIGRATIONS=false --env "RAGCORE_BASE_URL=$ragcore_base_url" "$api_image")" if [ -n "$ragcore_network" ] && ! docker network connect "$ragcore_network" "$id"; then # shellcheck disable=SC2086 docker rm -f $new_api_ids "$id" > /dev/null 2>&1 || true echo "Could not attach API candidates to RAGcore network $ragcore_network" >&2 exit 1 fi new_api_ids="$new_api_ids $id" done wait_for_ids() { role="$1" shift attempt=0 while true; do healthy=0 count=0 for id in "$@"; do count=$((count + 1)) if [ "$role" = "api" ]; then docker exec "$id" python -c \ "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/ready')" \ > /dev/null 2>&1 && healthy=$((healthy + 1)) else docker exec "$id" wget -q -O /dev/null http://127.0.0.1/health \ > /dev/null 2>&1 && healthy=$((healthy + 1)) fi done [ "$count" -eq 2 ] && [ "$healthy" -eq 2 ] && return 0 attempt=$((attempt + 1)) [ "$attempt" -lt 60 ] || { echo "$role candidates did not become healthy" >&2; return 1; } sleep 2 done } # shellcheck disable=SC2086 if ! wait_for_ids api $new_api_ids; then docker rm -f $new_api_ids > /dev/null 2>&1 || true exit 1 fi for replica in 1 2; do name="$project-web-$short_revision-$replica" id="$(docker run -d --name "$name" --restart unless-stopped \ --label com.mobilityops.role=web --label "com.mobilityops.project=$project" \ --label "com.mobilityops.revision=$revision" \ --network "$network" --network-alias "web-$short_revision" "$web_image")" new_web_ids="$new_web_ids $id" done # shellcheck disable=SC2086 if ! wait_for_ids web $new_web_ids; then docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true exit 1 fi # Install the stable gateway once. Future releases leave it running while Docker DNS # exposes both old and new web aliases. The first migration from direct port ownership # necessarily has a brief hand-off while port 1236 moves to the gateway. gateway_id="$($compose ps -q gateway)" cp "$candidate_gateway" "$gateway_config" if [ -z "$gateway_id" ]; then if [ -n "$old_web_ids" ]; then # shellcheck disable=SC2086 docker rm -f $old_web_ids > /dev/null old_web_ids="" fi $compose up --no-build --no-deps -d gateway else if ! docker exec "$gateway_id" nginx -t; then [ ! -s "$previous_gateway" ] || cp "$previous_gateway" "$gateway_config" # shellcheck disable=SC2086 docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true echo "Candidate gateway configuration was rejected; previous replicas remain active" >&2 exit 1 fi docker exec "$gateway_id" nginx -s reload fi attempt=0 until curl -fsS http://127.0.0.1:1236/health/ready > /dev/null; do attempt=$((attempt + 1)) if [ "$attempt" -ge 30 ]; then if [ -n "$gateway_id" ] && [ -s "$previous_gateway" ]; then cp "$previous_gateway" "$gateway_config" docker exec "$gateway_id" nginx -t && docker exec "$gateway_id" nginx -s reload # shellcheck disable=SC2086 docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true elif [ -n "$old_web_image" ]; then # The first gateway migration keeps the already healthy candidate APIs because # the rendered gateway points at their versioned alias. # shellcheck disable=SC2086 docker rm -f $new_web_ids > /dev/null 2>&1 || true for replica in 1 2; do docker run -d --name "$project-web-rollback-$short_revision-$replica" --restart unless-stopped \ --label com.mobilityops.role=web --label "com.mobilityops.project=$project" \ --label com.mobilityops.revision=rollback \ --network "$network" --network-alias "web-$short_revision" "$old_web_image" > /dev/null done else # shellcheck disable=SC2086 docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true fi echo "Release failed readiness; previous API and restored web replicas remain active" >&2 exit 1 fi sleep 2 done # Promotion is start-first: only now remove the previous containers. A short drain lets # workers using the pre-reload configuration complete their in-flight requests. sleep 3 # shellcheck disable=SC2086 [ -z "$old_api_ids" ] || docker rm -f $old_api_ids > /dev/null # shellcheck disable=SC2086 [ -z "$old_web_ids" ] || docker rm -f $old_web_ids > /dev/null curl -fsS http://127.0.0.1:1236/health/ready > /dev/null # shellcheck disable=SC2086 first_new_api="$(printf '%s\n' $new_api_ids | head -n 1)" docker exec "$first_new_api" alembic current for service in backup prometheus alertmanager grafana; do attempt=0 until status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$project-$service-1" 2>/dev/null)" \ && { [ "$status" = "healthy" ] || [ "$status" = "running" ]; }; do attempt=$((attempt + 1)) [ "$attempt" -lt 60 ] || { echo "$service did not become healthy" >&2; exit 1; } sleep 5 done done printf '%s\n' "$revision" > "$root/.deploy/source-revision" printf '%s %s %s %s\n' "$revision" "$api_image" "$web_image" "$expected_checksum" \ >> "$root/.deploy/release-history.log" echo "Promoted Fleet Ops release $revision"