M48: harden demo operations and offsite recovery

.gitea/workflows/ci.yml

@@ -117,6 +117,8 @@ jobs:
         env:
           MOBILITYOPS_PUBLIC_URL: http://localhost:1228
         run: npx playwright test --config=playwright.live.config.ts
+      - name: Run concurrent persisted-read smoke
+        run: python scripts/run-readonly-load-smoke.py --base-url http://localhost:1228
       - name: Upload Playwright report
         if: failure()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4

.gitea/workflows/live-canary.yml

@@ -0,0 +1,47 @@
+name: MobilityOps live canary
+
+on:
+  schedule:
+    - cron: "7 * * * *"
+  workflow_dispatch:
+
+jobs:
+  public-demo:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+      - name: Verify HTTPS readiness and certificate horizon
+        run: |
+          curl --fail --silent --show-error --retry 3 https://fleetops.itworx.tech/health/ready
+          openssl s_client -servername fleetops.itworx.tech -connect fleetops.itworx.tech:443 </dev/null 2>/dev/null \
+            | openssl x509 -checkend 1209600 -noout
+      - name: Install locked Playwright runtime
+        working-directory: frontend
+        run: |
+          npm ci --no-audit --no-fund
+          npx playwright install --with-deps chromium firefox
+      - name: Run non-destructive cross-browser production canary
+        working-directory: frontend
+        env:
+          MOBILITYOPS_PUBLIC_URL: https://fleetops.itworx.tech
+        run: npx playwright test --config=playwright.live.config.ts
+      - name: Report successful external heartbeat
+        env:
+          HEARTBEAT_URL: ${{ secrets.LIVE_CANARY_HEARTBEAT_URL }}
+        run: |
+          if [ -n "$HEARTBEAT_URL" ]; then
+            curl --fail --silent --show-error --retry 3 "$HEARTBEAT_URL"
+          fi
+      - name: Upload failure evidence
+        if: failure()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: live-canary-failure
+          path: |
+            frontend/playwright-live-report
+            frontend/test-results

.gitea/workflows/release.yml

@@ -13,6 +13,16 @@ jobs:
         run: |
           docker build --target runtime --build-arg VCS_REF="$GITHUB_SHA" --tag mobilityops-api-release --file backend/Dockerfile .
           docker build --build-arg VCS_REF="$GITHUB_SHA" --tag mobilityops-web-release frontend
+          docker build --build-arg VCS_REF="$GITHUB_SHA" --tag mobilityops-backup-tools-release --file deploy/unraid/Dockerfile.backup-tools .
+      - name: Scan all release images
+        run: |
+          for image in mobilityops-api-release mobilityops-web-release mobilityops-backup-tools-release; do
+            docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \
+              -v mobilityops-release-trivy:/root/.cache/ \
+              aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969 \
+              image --scanners vuln --severity HIGH,CRITICAL \
+              --ignore-unfixed --exit-code 1 "$image"
+          done
       - name: Generate API CycloneDX SBOM
         uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # v0.30.0
         with:
@@ -27,11 +37,20 @@ jobs:
           image-ref: mobilityops-web-release
           format: cyclonedx
           output: mobilityops-web-sbom.cdx.json
+      - name: Generate backup-tools CycloneDX SBOM
+        uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # v0.30.0
+        with:
+          scan-type: image
+          image-ref: mobilityops-backup-tools-release
+          format: cyclonedx
+          output: mobilityops-backup-tools-sbom.cdx.json
       - name: Record immutable image metadata
         run: |
           docker image inspect mobilityops-api-release > mobilityops-api-image.json
           docker image inspect mobilityops-web-release > mobilityops-web-image.json
-          sha256sum mobilityops-*-sbom.cdx.json mobilityops-*-image.json > SHA256SUMS
+          docker image inspect mobilityops-backup-tools-release > mobilityops-backup-tools-image.json
+          python scripts/generate-release-provenance.py
+          sha256sum mobilityops-*-sbom.cdx.json mobilityops-*-image.json release-provenance.json > SHA256SUMS
       - name: Upload release evidence
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
@@ -39,4 +58,5 @@ jobs:
           path: |
             mobilityops-*-sbom.cdx.json
             mobilityops-*-image.json
+            release-provenance.json
             SHA256SUMS