Initial deploy setup

2026-07-21 14:00:00 +02:00
commit b8091e59bd
285 changed files with 27854 additions and 0 deletions
@@ -0,0 +1,381 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+release_ts="${VR_RELEASE_TIMESTAMP:-$(date -u +%Y%m%dT%H%M%SZ)}"
+release_root="${VR_RELEASE_ROOT:-./release-artifacts/${release_ts}}"
+stack_base="vr-release-${release_ts}"
+image_tag="vacatureradar:release-${release_ts}"
+smoke_fixture="fixtures/pages/sample_jsonld_job.html"
+smoke_url="https://release-smoke.example.org/jobs/infrastructure-engineer"
+smoke_source="Release Smoke Source"
+admin_username="release-admin"
+admin_password="release-admin-${release_ts}"
+smoke_port="18080"
+restore_port="18081"
+smoke_stack="${stack_base}-smoke"
+restore_stack="${stack_base}-restore"
+smoke_root="${release_root}/smoke"
+restore_root="${release_root}/restore"
+primary_report="${release_root}/artifacts/smoke-primary.json"
+restore_report="${release_root}/artifacts/smoke-restored-validate.json"
+
+require_cmd() {
+  if ! command -v "$1" >/dev/null 2>&1; then
+    echo "release_verify: ontbrekende command '$1'" >&2
+    exit 1
+  fi
+}
+
+compose_exec() {
+  local stack_name="$1"
+  local stack_root="$2"
+  local service="$3"
+  shift 3
+  docker compose -f docker-compose.yml -f "${stack_root}/docker-compose.release.yml" -p "${stack_name}" exec -T "${service}" "$@"
+}
+
+compose_up() {
+  local stack_name="$1"
+  local stack_root="$2"
+  docker compose -f docker-compose.yml -f "${stack_root}/docker-compose.release.yml" -p "${stack_name}" up -d
+}
+
+compose_down() {
+  local stack_name="$1"
+  local stack_root="$2"
+  docker compose -f docker-compose.yml -f "${stack_root}/docker-compose.release.yml" -p "${stack_name}" down --remove-orphans
+}
+
+build_stack_override() {
+  local stack_root="$1"
+  local stack_port="$2"
+  local env_file="${stack_root}/release.env"
+
+  mkdir -p "${stack_root}"/{media,logs,local,postgres,redis,backups}
+
+  cat > "${env_file}" <<EOF
+DJANGO_SECRET_KEY=vr-release-${release_ts}
+DJANGO_DEBUG=0
+DJANGO_ALLOWED_HOSTS=127.0.0.1,localhost
+DJANGO_CSRF_TRUSTED_ORIGINS=http://127.0.0.1:${stack_port}
+DJANGO_TIME_ZONE=Europe/Brussels
+POSTGRES_DB=vacatureradar
+POSTGRES_USER=vacatureradar
+POSTGRES_PASSWORD=vacatureradar_release
+DATABASE_URL=postgresql://vacatureradar:vacatureradar_release@postgres:5432/vacatureradar
+REDIS_URL=redis://redis:6379/0
+CELERY_TASK_ALWAYS_EAGER=1
+VACATURERADAR_ADMIN_USERNAME=${admin_username}
+VACATURERADAR_ADMIN_PASSWORD=${admin_password}
+VACATURERADAR_ADMIN_EMAIL=release-smoke@invalid
+EMAIL_BACKEND=django.core.mail.backends.locmem.EmailBackend
+DIGEST_RECIPIENT=release-smoke@invalid
+IMAP_ENABLED=0
+IMAP_MARK_SEEN=0
+SOURCE_POLICY_MODE=strict
+OLLAMA_ENABLED=0
+SESSION_COOKIE_SECURE=0
+CSRF_COOKIE_SECURE=0
+SECURE_SSL_REDIRECT=0
+EOF
+
+  cat > "${stack_root}/docker-compose.release.yml" <<EOF
+services:
+  web:
+    image: ${image_tag}
+    env_file:
+      - "${env_file}"
+    volumes:
+      - "${stack_root}/media:/app/media"
+      - "${stack_root}/logs:/app/logs"
+      - "${stack_root}/local:/app/local"
+      - "${stack_root}/backups:/app/backups"
+    ports:
+      - "127.0.0.1:${stack_port}:8080"
+  worker:
+    image: ${image_tag}
+    env_file:
+      - "${env_file}"
+    volumes:
+      - "${stack_root}/media:/app/media"
+      - "${stack_root}/logs:/app/logs"
+      - "${stack_root}/local:/app/local"
+      - "${stack_root}/backups:/app/backups"
+  scheduler:
+    image: ${image_tag}
+    env_file:
+      - "${env_file}"
+    volumes:
+      - "${stack_root}/media:/app/media"
+      - "${stack_root}/logs:/app/logs"
+      - "${stack_root}/local:/app/local"
+      - "${stack_root}/backups:/app/backups"
+  postgres:
+    image: postgres:17.5-bookworm
+    env_file:
+      - "${env_file}"
+    volumes:
+      - "${stack_root}/postgres:/var/lib/postgresql/data"
+  redis:
+    image: redis:7.4.2-alpine
+    volumes:
+      - "${stack_root}/redis:/data"
+EOF
+}
+
+require_cmd docker
+require_cmd python
+require_cmd uv
+require_cmd sha256sum
+require_cmd git
+
+mkdir -p "${release_root}/artifacts"
+
+cleanup() {
+  compose_down "${smoke_stack}" "${smoke_root}" || true
+  compose_down "${restore_stack}" "${restore_root}" || true
+}
+trap cleanup EXIT
+
+json_value() {
+  local report="$1"
+  local key="$2"
+  python - "$report" "$key" <<'PY'
+import json
+import sys
+
+path, key = sys.argv[1], sys.argv[2]
+with open(path, "r", encoding="utf-8") as handle:
+    payload = json.load(handle)
+value = payload.get(key)
+if isinstance(value, bool):
+    value = "1" if value else "0"
+elif value is None:
+    value = ""
+else:
+    value = str(value)
+print(value)
+PY
+}
+
+assert_smoke_report() {
+  local report="$1"
+  local label="$2"
+  python - "$report" "$label" <<'PY'
+import json
+import sys
+
+path, label = sys.argv[1], sys.argv[2]
+with open(path, "r", encoding="utf-8") as handle:
+    payload = json.load(handle)
+
+if payload.get("alias_count", 0) != 1:
+    raise SystemExit(f"release-smoke ({label}): alias_count={payload.get('alias_count')}")
+if payload.get("feedback_count", 0) < 1:
+    raise SystemExit(f"release-smoke ({label}): feedback_count={payload.get('feedback_count')}")
+if payload.get("application_count", 0) < 1:
+    raise SystemExit(f"release-smoke ({label}): application_count={payload.get('application_count')}")
+if payload.get("score_run_count", 0) < 1:
+    raise SystemExit(f"release-smoke ({label}): score_run_count={payload.get('score_run_count')}")
+if payload.get("digest_created") is not True:
+    raise SystemExit(f"release-smoke ({label}): digest_created={payload.get('digest_created')}")
+PY
+}
+
+assert_model_counts() {
+  local source_report="$1"
+  local target_report="$2"
+  python - "$source_report" "$target_report" <<'PY'
+import json
+import sys
+
+left_path, right_path = sys.argv[1], sys.argv[2]
+with open(left_path, "r", encoding="utf-8") as handle:
+    left = json.load(handle)
+with open(right_path, "r", encoding="utf-8") as handle:
+    right = json.load(handle)
+
+checks = [
+    "job_total_count",
+    "source_job_count",
+    "feedback_count",
+    "score_run_count",
+    "application_count",
+    "digest_sent",
+]
+for key in checks:
+    if left.get(key) != right.get(key):
+        raise SystemExit(f"restore-mismatch op {key}: source={left.get(key)!r} restore={right.get(key)!r}")
+PY
+}
+
+run_smoke() {
+  local stack_name="$1"
+  local stack_root="$2"
+  local label="$3"
+  local stack_port="$4"
+  local report_path="$5"
+
+  build_stack_override "${stack_root}" "${stack_port}"
+  compose_up "${stack_name}" "${stack_root}"
+  compose_exec "${stack_name}" "${stack_root}" web python scripts/healthcheck.py http://127.0.0.1:8080/health/ready/
+  compose_exec "${stack_name}" "${stack_root}" web python scripts/healthcheck.py http://127.0.0.1:8080/health/live/
+
+  compose_exec "${stack_name}" "${stack_root}" web python scripts/release_smoke.py \
+    --fixture "${smoke_fixture}" \
+    --url "${smoke_url}" \
+    --source-name "${smoke_source}" \
+    --admin-username "${admin_username}" \
+    --admin-password "${admin_password}" \
+    --admin-email "release-smoke@invalid" \
+    --profile-name "Release Smoke Profiel" \
+    --label "${label}" \
+    | tee "${report_path}"
+
+  assert_smoke_report "${report_path}" "${label}"
+}
+
+echo "Starten VR-117 releaseverificatie (${release_ts})..."
+
+./scripts/codex_verify.sh
+python scripts/backlog.py validate
+python scripts/validate_task_ledger.py
+python scripts/validate_repository.py
+
+echo "Bouw clean release-artifact..."
+package_file="${release_root}/artifacts/VacatureRadar-Release-${release_ts}.zip"
+uv run python scripts/package_project.py "${package_file}"
+uv run python scripts/verify_package.py "${package_file}"
+cp CHANGELOG.md "${release_root}/artifacts/CHANGELOG.md"
+cp docker-compose.yml docker-compose.unraid.yml .env.example "${release_root}/artifacts/"
+
+echo "Genereer configuratie-diff..."
+if git rev-parse --verify --quiet HEAD~1 >/dev/null 2>&1; then
+  git diff --unified=1 HEAD~1 -- docker-compose.yml docker-compose.unraid.yml .env.example docs/operations/UNRAID_DEPLOYMENT.md > "${release_root}/artifacts/config-diff.txt"
+else
+  git diff --unified=1 -- docker-compose.yml docker-compose.unraid.yml .env.example docs/operations/UNRAID_DEPLOYMENT.md > "${release_root}/artifacts/config-diff.txt"
+fi
+
+echo "Bouw image ${image_tag} en verzamel metadata..."
+docker build --pull -t "${image_tag}" .
+docker image inspect "${image_tag}" > "${release_root}/artifacts/docker-image-inspect.json"
+
+python - <<PY
+from __future__ import annotations
+
+import json
+import tomllib
+from pathlib import Path
+
+release_root = Path("${release_root}")
+inspect = json.loads((release_root / "artifacts" / "docker-image-inspect.json").read_text(encoding="utf-8"))
+lock_payload = tomllib.loads((Path("uv.lock").read_text(encoding="utf-8")))
+packages = []
+for row in lock_payload.get("package", []):
+    packages.append(
+        {
+            "name": row.get("name"),
+            "version": row.get("version"),
+            "source": row.get("source"),
+            "url": row.get("url"),
+        }
+    )
+(release_root / "artifacts" / "sbom.json").write_text(
+    json.dumps(
+        {
+            "generated_by": "scripts/release_verify.sh",
+            "package": "VacatureRadar",
+            "project_file": "uv.lock",
+            "image_id": inspect[0].get("Id") if inspect else None,
+            "packages": packages,
+        },
+        sort_keys=True,
+        ensure_ascii=False,
+        indent=2,
+    )
+    + "\n",
+    encoding="utf-8",
+)
+PY
+
+cat > "${release_root}/artifacts/upgrade-rollback.md" <<'EOF'
+# Upgrade- en rollbackpad (VR-117)
+
+## Upgrade
+
+1. Maak eerst een PostgreSQL-backup en documenteer de restorebron.
+2. Verifieer image-id met `docker image inspect`.
+3. Schakel alleen de oude versie uit en houd precies één vorige image-tag of digest klaar.
+4. Start de nieuwe versie met exact `docker-compose.yml` en matching `DATABASE_URL`.
+5. Wacht op `/health/live/` en `/health/ready/` vóór het herstarten van worker en scheduler.
+6. Laat één herstelbaar model-smokepad lopen (fixture-import, score, feedback, application, digest).
+7. Zet de vorige image gereed als rollbackoptie totdat één volledige cycle stabiel draait.
+
+## Rollback
+
+1. Stop worker en scheduler, daarna web.
+2. Start met vorige image-tag of -digest.
+3. Start web, daarna worker en scheduler.
+4. Controleer liveness/readiness, fixture-smoke en modelcounts.
+5. Restore uit de vooraf gemaakte back-up wanneer datamodelmigraties incompatibel zijn.
+EOF
+
+cat > "${release_root}/artifacts/release-notes.txt" <<EOF
+Release artifacts (${release_ts})
+Generated UTC: ${release_ts}
+- package: ${package_file}
+- image: ${image_tag}
+- changelog: ${release_root}/artifacts/CHANGELOG.md
+- sbom: ${release_root}/artifacts/sbom.json
+- configdiff: ${release_root}/artifacts/config-diff.txt
+- upgrade/rollback: ${release_root}/artifacts/upgrade-rollback.md
+- smoke report: ${primary_report}
+- restore smoke report: ${restore_report}
+EOF
+
+sha256sum \
+  "${package_file}" \
+  "${release_root}/artifacts/docker-image-inspect.json" \
+  "${release_root}/artifacts/sbom.json" \
+  "${release_root}/artifacts/config-diff.txt" \
+  "${release_root}/artifacts/upgrade-rollback.md" \
+  "${release_root}/artifacts/release-notes.txt" \
+  > "${release_root}/artifacts/checksums.txt"
+
+echo "Start smoke-stack..."
+run_smoke "${smoke_stack}" "${smoke_root}" "smoke-primary" "${smoke_port}" "${primary_report}"
+
+compose_exec "${smoke_stack}" "${smoke_root}" web ./scripts/backup.sh
+backup_file="$(ls -1t "${smoke_root}/backups"/vacatureradar-*.sql.gz "${smoke_root}/backups"/vacatureradar-*.sqlite3 2>/dev/null | head -n 1)"
+if [[ -z "${backup_file}" ]]; then
+  echo "release_verify: backupbestand niet aangetroffen" >&2
+  exit 1
+fi
+
+echo "Start restore-stack..."
+mkdir -p "${restore_root}/backups"
+cp "${backup_file}" "${restore_root}/backups/$(basename "${backup_file}")"
+
+build_stack_override "${restore_root}" "${restore_port}"
+compose_up "${restore_stack}" "${restore_root}"
+compose_exec "${restore_stack}" "${restore_root}" web python scripts/healthcheck.py http://127.0.0.1:8080/health/ready/
+compose_exec "${restore_stack}" "${restore_root}" web python scripts/healthcheck.py http://127.0.0.1:8080/health/live/
+compose_exec "${restore_stack}" "${restore_root}" web ./scripts/restore.sh "/app/backups/$(basename "${backup_file}")"
+run_smoke "${restore_stack}" "${restore_root}" "smoke-restored-validate" "${restore_port}" "${restore_report}"
+assert_model_counts "${primary_report}" "${restore_report}"
+
+alias_count="$(json_value "${primary_report}" "alias_count")"
+restore_alias_count="$(json_value "${restore_report}" "alias_count")"
+if [[ "${alias_count}" != "${restore_alias_count}" ]]; then
+  echo "release_verify: alias_count mismatch (${alias_count} vs ${restore_alias_count})" >&2
+  exit 1
+fi
+
+echo "VR-117 release verification voltooid; artifacts in: ${release_root}"
+echo "- package : ${package_file}"
+echo "- upgrade/rollback: ${release_root}/artifacts/upgrade-rollback.md"
+echo "- checksums: ${release_root}/artifacts/checksums.txt"
+echo "- configdiff: ${release_root}/artifacts/config-diff.txt"
+exit 0