285 lines
12 KiB
Bash
Executable File
285 lines
12 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
# Stage a committed source archive, build commit-labelled immutable images and promote
|
|
# them without overlaying source files or reseeding persisted data.
|
|
archive="${1:-}"
|
|
expected_checksum="${2:-}"
|
|
revision="${3:-}"
|
|
root="${MOBILITYOPS_DEPLOY_ROOT:-/mnt/user/appdata/mobilityops}"
|
|
project="${COMPOSE_PROJECT_NAME:-mobilityops}"
|
|
|
|
[ "${#revision}" -eq 40 ] || { echo "Revision must be a full Git SHA" >&2; exit 1; }
|
|
case "$revision" in *[!0-9a-f]*) echo "Revision must be lowercase hexadecimal" >&2; exit 1;; esac
|
|
[ -f "$archive" ] || { echo "Archive not found: $archive" >&2; exit 1; }
|
|
[ -n "$expected_checksum" ] || { echo "Expected SHA-256 is required" >&2; exit 1; }
|
|
[ -f "$root/.env" ] || { echo "Production .env is missing" >&2; exit 1; }
|
|
|
|
read_env_value() {
|
|
key="$1"
|
|
awk -v key="$key" '
|
|
index($0, key "=") == 1 {
|
|
sub("^[^=]*=", "")
|
|
print
|
|
exit
|
|
}
|
|
' "$root/.env"
|
|
}
|
|
|
|
# Stateless replicas normally inherit the complete, already-secret-resolved environment
|
|
# from the serving API. RAGcore routing is an explicit exception: its private endpoint and
|
|
# cross-project Docker network are non-secret deployment topology and the server .env is
|
|
# authoritative so infrastructure changes take effect on the next release.
|
|
ragcore_base_url="$(read_env_value RAGCORE_BASE_URL)"
|
|
ragcore_network="$(read_env_value RAGCORE_DOCKER_NETWORK)"
|
|
case "$ragcore_base_url" in
|
|
http://*|https://*) ;;
|
|
*) echo "RAGCORE_BASE_URL must be an absolute HTTP(S) URL" >&2; exit 1 ;;
|
|
esac
|
|
case "$ragcore_network" in
|
|
"") ;;
|
|
*[!a-zA-Z0-9_.-]*) echo "RAGCORE_DOCKER_NETWORK contains invalid characters" >&2; exit 1 ;;
|
|
*) docker network inspect "$ragcore_network" > /dev/null ;;
|
|
esac
|
|
|
|
actual_checksum="$(sha256sum "$archive" | awk '{print $1}')"
|
|
[ "$actual_checksum" = "$expected_checksum" ] || {
|
|
echo "Archive checksum mismatch" >&2
|
|
exit 1
|
|
}
|
|
|
|
release_root="$root/.deploy/releases"
|
|
release_dir="$release_root/$revision"
|
|
mkdir -p "$release_root"
|
|
if [ -e "$release_dir" ]; then
|
|
echo "Release directory already exists; refusing to overwrite: $release_dir" >&2
|
|
exit 1
|
|
fi
|
|
mkdir "$release_dir"
|
|
tar -xzf "$archive" -C "$release_dir"
|
|
|
|
short_revision="$(printf '%s' "$revision" | cut -c1-12)"
|
|
build_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
api_image="mobilityops-api:$short_revision"
|
|
web_image="mobilityops-web:$short_revision"
|
|
|
|
docker build --target runtime --build-arg "VCS_REF=$revision" --build-arg "BUILD_DATE=$build_date" \
|
|
--tag "$api_image" --file "$release_dir/backend/Dockerfile" "$release_dir"
|
|
docker build --build-arg "VCS_REF=$revision" --build-arg "BUILD_DATE=$build_date" \
|
|
--tag "$web_image" "$release_dir/frontend"
|
|
|
|
for image in "$api_image" "$web_image"; do
|
|
labelled_revision="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image")"
|
|
[ "$labelled_revision" = "$revision" ] || {
|
|
echo "Image revision label mismatch for $image" >&2
|
|
exit 1
|
|
}
|
|
done
|
|
|
|
compose="docker compose --env-file $root/.env -p $project -f $release_dir/compose.yaml -f $release_dir/compose.unraid.yaml -f $release_dir/compose.observability.yaml -f $release_dir/compose.release.yaml --profile observability"
|
|
export MOBILITYOPS_API_IMAGE="$api_image" MOBILITYOPS_WEB_IMAGE="$web_image"
|
|
gateway_config="$root/.deploy/gateway.conf"
|
|
candidate_gateway="$(mktemp "$root/.deploy/gateway-candidate.XXXXXX")"
|
|
previous_gateway="$(mktemp "$root/.deploy/gateway-previous.XXXXXX")"
|
|
sed -e "s/server web:80/server web-$short_revision:80/" \
|
|
-e "s/server api:8000/server api-$short_revision:8000/" \
|
|
"$release_dir/deploy/unraid/gateway.conf" > "$candidate_gateway"
|
|
if [ -f "$gateway_config" ]; then
|
|
cp "$gateway_config" "$previous_gateway"
|
|
else
|
|
cp "$candidate_gateway" "$gateway_config"
|
|
fi
|
|
export MOBILITYOPS_GATEWAY_CONFIG="$gateway_config"
|
|
|
|
# Bootstrap infrastructure only when it is absent. Existing stateful/monitoring
|
|
# containers are deliberately not reconciled against every commit-named source path.
|
|
missing_infrastructure=""
|
|
for service in db backup prometheus alertmanager grafana; do
|
|
[ -n "$($compose ps -q "$service")" ] || missing_infrastructure="$missing_infrastructure $service"
|
|
done
|
|
if [ -n "$missing_infrastructure" ]; then
|
|
# shellcheck disable=SC2086
|
|
$compose up --no-build -d $missing_infrastructure
|
|
fi
|
|
|
|
# Apply schema changes once before starting the new stateless replicas. Migrations in a
|
|
# release must remain backwards compatible with the still-serving previous API.
|
|
$compose run --rm --no-deps --entrypoint alembic api upgrade head
|
|
|
|
network="$(docker inspect --format '{{range $name, $network := .NetworkSettings.Networks}}{{$name}}{{end}}' "$project-db-1")"
|
|
[ -n "$network" ] || { echo "MobilityOps network was not found" >&2; exit 1; }
|
|
|
|
old_api_ids="$(docker ps -q --filter label=com.mobilityops.role=api --filter "label=com.mobilityops.project=$project")"
|
|
old_web_ids="$(docker ps -q --filter label=com.mobilityops.role=web --filter "label=com.mobilityops.project=$project")"
|
|
if [ -z "$old_api_ids" ]; then
|
|
old_api_ids="$(docker ps -q --filter "name=^/${project}-api-")"
|
|
fi
|
|
if [ -z "$old_web_ids" ]; then
|
|
old_web_ids="$(docker ps -q --filter "name=^/${project}-web-")"
|
|
fi
|
|
|
|
# Reuse the effective, already-secret-resolved API environment without printing it.
|
|
api_environment="$(mktemp "$root/.deploy/api-environment.XXXXXX")"
|
|
chmod 600 "$api_environment"
|
|
cleanup() {
|
|
rm -f "$api_environment" "$candidate_gateway" "$previous_gateway"
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
if [ -n "$old_api_ids" ]; then
|
|
# shellcheck disable=SC2086
|
|
first_old_api="$(printf '%s\n' $old_api_ids | head -n 1)"
|
|
docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$first_old_api" > "$api_environment"
|
|
else
|
|
# A clean bootstrap first lets Compose resolve the complete API environment. This
|
|
# container is retained as the previous slot until the candidates pass.
|
|
$compose up --no-build --no-deps -d api web
|
|
old_api_ids="$($compose ps -q api)"
|
|
old_web_ids="$($compose ps -q web)"
|
|
docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$old_api_ids" > "$api_environment"
|
|
fi
|
|
|
|
new_api_ids=""
|
|
new_web_ids=""
|
|
old_web_image=""
|
|
# shellcheck disable=SC2086
|
|
[ -z "$old_web_ids" ] || old_web_image="$(docker inspect --format '{{.Config.Image}}' "$(printf '%s\n' $old_web_ids | head -n 1)")"
|
|
for replica in 1 2; do
|
|
name="$project-api-$short_revision-$replica"
|
|
id="$(docker run -d --name "$name" --restart unless-stopped \
|
|
--label com.mobilityops.role=api --label "com.mobilityops.project=$project" \
|
|
--label "com.mobilityops.revision=$revision" \
|
|
--network "$network" --network-alias api --network-alias "api-$short_revision" \
|
|
--env-file "$api_environment" \
|
|
--env RUN_MIGRATIONS=false --env "RAGCORE_BASE_URL=$ragcore_base_url" "$api_image")"
|
|
if [ -n "$ragcore_network" ] && ! docker network connect "$ragcore_network" "$id"; then
|
|
# shellcheck disable=SC2086
|
|
docker rm -f $new_api_ids "$id" > /dev/null 2>&1 || true
|
|
echo "Could not attach API candidates to RAGcore network $ragcore_network" >&2
|
|
exit 1
|
|
fi
|
|
new_api_ids="$new_api_ids $id"
|
|
done
|
|
|
|
wait_for_ids() {
|
|
role="$1"
|
|
shift
|
|
attempt=0
|
|
while true; do
|
|
healthy=0
|
|
count=0
|
|
for id in "$@"; do
|
|
count=$((count + 1))
|
|
if [ "$role" = "api" ]; then
|
|
docker exec "$id" python -c \
|
|
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/ready')" \
|
|
> /dev/null 2>&1 && healthy=$((healthy + 1))
|
|
else
|
|
docker exec "$id" wget -q -O /dev/null http://127.0.0.1/health \
|
|
> /dev/null 2>&1 && healthy=$((healthy + 1))
|
|
fi
|
|
done
|
|
[ "$count" -eq 2 ] && [ "$healthy" -eq 2 ] && return 0
|
|
attempt=$((attempt + 1))
|
|
[ "$attempt" -lt 60 ] || { echo "$role candidates did not become healthy" >&2; return 1; }
|
|
sleep 2
|
|
done
|
|
}
|
|
|
|
# shellcheck disable=SC2086
|
|
if ! wait_for_ids api $new_api_ids; then
|
|
docker rm -f $new_api_ids > /dev/null 2>&1 || true
|
|
exit 1
|
|
fi
|
|
|
|
for replica in 1 2; do
|
|
name="$project-web-$short_revision-$replica"
|
|
id="$(docker run -d --name "$name" --restart unless-stopped \
|
|
--label com.mobilityops.role=web --label "com.mobilityops.project=$project" \
|
|
--label "com.mobilityops.revision=$revision" \
|
|
--network "$network" --network-alias "web-$short_revision" "$web_image")"
|
|
new_web_ids="$new_web_ids $id"
|
|
done
|
|
# shellcheck disable=SC2086
|
|
if ! wait_for_ids web $new_web_ids; then
|
|
docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
|
|
exit 1
|
|
fi
|
|
|
|
# Install the stable gateway once. Future releases leave it running while Docker DNS
|
|
# exposes both old and new web aliases. The first migration from direct port ownership
|
|
# necessarily has a brief hand-off while port 1236 moves to the gateway.
|
|
gateway_id="$($compose ps -q gateway)"
|
|
cp "$candidate_gateway" "$gateway_config"
|
|
if [ -z "$gateway_id" ]; then
|
|
if [ -n "$old_web_ids" ]; then
|
|
# shellcheck disable=SC2086
|
|
docker rm -f $old_web_ids > /dev/null
|
|
old_web_ids=""
|
|
fi
|
|
$compose up --no-build --no-deps -d gateway
|
|
else
|
|
if ! docker exec "$gateway_id" nginx -t; then
|
|
[ ! -s "$previous_gateway" ] || cp "$previous_gateway" "$gateway_config"
|
|
# shellcheck disable=SC2086
|
|
docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
|
|
echo "Candidate gateway configuration was rejected; previous replicas remain active" >&2
|
|
exit 1
|
|
fi
|
|
docker exec "$gateway_id" nginx -s reload
|
|
fi
|
|
|
|
attempt=0
|
|
until curl -fsS http://127.0.0.1:1236/health/ready > /dev/null; do
|
|
attempt=$((attempt + 1))
|
|
if [ "$attempt" -ge 30 ]; then
|
|
if [ -n "$gateway_id" ] && [ -s "$previous_gateway" ]; then
|
|
cp "$previous_gateway" "$gateway_config"
|
|
docker exec "$gateway_id" nginx -t && docker exec "$gateway_id" nginx -s reload
|
|
# shellcheck disable=SC2086
|
|
docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
|
|
elif [ -n "$old_web_image" ]; then
|
|
# The first gateway migration keeps the already healthy candidate APIs because
|
|
# the rendered gateway points at their versioned alias.
|
|
# shellcheck disable=SC2086
|
|
docker rm -f $new_web_ids > /dev/null 2>&1 || true
|
|
for replica in 1 2; do
|
|
docker run -d --name "$project-web-rollback-$short_revision-$replica" --restart unless-stopped \
|
|
--label com.mobilityops.role=web --label "com.mobilityops.project=$project" \
|
|
--label com.mobilityops.revision=rollback \
|
|
--network "$network" --network-alias "web-$short_revision" "$old_web_image" > /dev/null
|
|
done
|
|
else
|
|
# shellcheck disable=SC2086
|
|
docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
|
|
fi
|
|
echo "Release failed readiness; previous API and restored web replicas remain active" >&2
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
# Promotion is start-first: only now remove the previous containers. A short drain lets
|
|
# workers using the pre-reload configuration complete their in-flight requests.
|
|
sleep 3
|
|
# shellcheck disable=SC2086
|
|
[ -z "$old_api_ids" ] || docker rm -f $old_api_ids > /dev/null
|
|
# shellcheck disable=SC2086
|
|
[ -z "$old_web_ids" ] || docker rm -f $old_web_ids > /dev/null
|
|
curl -fsS http://127.0.0.1:1236/health/ready > /dev/null
|
|
# shellcheck disable=SC2086
|
|
first_new_api="$(printf '%s\n' $new_api_ids | head -n 1)"
|
|
docker exec "$first_new_api" alembic current
|
|
for service in backup prometheus alertmanager grafana; do
|
|
attempt=0
|
|
until status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$project-$service-1" 2>/dev/null)" \
|
|
&& { [ "$status" = "healthy" ] || [ "$status" = "running" ]; }; do
|
|
attempt=$((attempt + 1))
|
|
[ "$attempt" -lt 60 ] || { echo "$service did not become healthy" >&2; exit 1; }
|
|
sleep 5
|
|
done
|
|
done
|
|
printf '%s\n' "$revision" > "$root/.deploy/source-revision"
|
|
printf '%s %s %s %s\n' "$revision" "$api_image" "$web_image" "$expected_checksum" \
|
|
>> "$root/.deploy/release-history.log"
|
|
echo "Promoted Fleet Ops release $revision"
|