From 00191e9b54ee6b961648a6e02abbb3a57957dba0 Mon Sep 17 00:00:00 2001 From: NuklearRabbit <145918611+NuklearRabbit@users.noreply.github.com> Date: Fri, 21 Aug 2026 22:17:49 +0200 Subject: [PATCH] M48: harden demo operations and offsite recovery --- .env.example | 7 + .gitea/workflows/ci.yml | 2 + .gitea/workflows/live-canary.yml | 47 ++++ .gitea/workflows/release.yml | 22 +- PROJECT_STATE.md | 28 +++ README.md | 1 + backend/app/services/data_quality.py | 74 +----- .../services/data_quality_duplicate_scan.py | 110 +++++++++ backend/entrypoint.sh | 4 +- compose.observability.yaml | 10 + compose.release.yaml | 2 + compose.unraid.yaml | 62 ++++- deploy/unraid/Dockerfile.backup-tools | 17 ++ deploy/unraid/README.md | 32 ++- deploy/unraid/configure-onedrive-backup.sh | 38 ++++ deploy/unraid/deploy-release.sh | 186 ++++++++++++++- deploy/unraid/gateway.conf | 58 +++++ deploy/unraid/refresh-infrastructure.sh | 27 +++ deploy/unraid/sync-onedrive-backups.sh | 67 ++++++ docs/17-runbook.md | 13 ++ docs/18-privacy-governance.md | 6 +- frontend/src/pages/DataQualityIssueDetail.tsx | 187 +-------------- .../data-quality/DuplicateCustomerPanel.tsx | 215 ++++++++++++++++++ frontend/src/styles-data-quality.css | 49 ++++ frontend/src/styles.css | 51 +---- scripts/check-source-budgets.py | 11 +- scripts/generate-release-provenance.py | 61 +++++ scripts/run-readonly-load-smoke.py | 81 +++++++ 28 files changed, 1136 insertions(+), 332 deletions(-) create mode 100644 .gitea/workflows/live-canary.yml create mode 100644 backend/app/services/data_quality_duplicate_scan.py create mode 100644 deploy/unraid/Dockerfile.backup-tools create mode 100644 deploy/unraid/configure-onedrive-backup.sh create mode 100644 deploy/unraid/gateway.conf create mode 100644 deploy/unraid/refresh-infrastructure.sh create mode 100644 deploy/unraid/sync-onedrive-backups.sh create mode 100644 frontend/src/pages/data-quality/DuplicateCustomerPanel.tsx create mode 100644 frontend/src/styles-data-quality.css create mode 100644 scripts/generate-release-provenance.py create mode 100644 scripts/run-readonly-load-smoke.py diff --git a/.env.example b/.env.example index 5293849..9eb5d46 100644 --- a/.env.example +++ b/.env.example @@ -50,6 +50,13 @@ BACKUP_RESTORE_DRILL_INTERVAL_SECONDS=604800 BACKUP_SECONDARY_DESTINATION= MOBILITYOPS_BACKUP_DIR=./backups/postgres MOBILITYOPS_BACKUP_SECONDARY_DIR=./backups/offsite +# Optional real off-site copy through the official rclone OneDrive adapter. OAuth state +# lives only in MOBILITYOPS_RCLONE_CONFIG_DIR and must never be committed. +BACKUP_OFFSITE_INTERVAL_SECONDS=900 +RCLONE_ONEDRIVE_REMOTE=onedrive +RCLONE_ONEDRIVE_PATH=FleetOps/backups +MOBILITYOPS_RCLONE_CONFIG_DIR=./.secrets/rclone +MOBILITYOPS_OFFSITE_VERIFY_DIR=./backups/offsite-verify # Privacy governance defaults. PRIVACY_MINIMUM_BOOKING_RETENTION_DAYS=30 diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index f8dd7fe..fcd1bcb 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -117,6 +117,8 @@ jobs: env: MOBILITYOPS_PUBLIC_URL: http://localhost:1228 run: npx playwright test --config=playwright.live.config.ts + - name: Run concurrent persisted-read smoke + run: python scripts/run-readonly-load-smoke.py --base-url http://localhost:1228 - name: Upload Playwright report if: failure() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/.gitea/workflows/live-canary.yml b/.gitea/workflows/live-canary.yml new file mode 100644 index 0000000..0f30769 --- /dev/null +++ b/.gitea/workflows/live-canary.yml @@ -0,0 +1,47 @@ +name: MobilityOps live canary + +on: + schedule: + - cron: "7 * * * *" + workflow_dispatch: + +jobs: + public-demo: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: frontend/package-lock.json + - name: Verify HTTPS readiness and certificate horizon + run: | + curl --fail --silent --show-error --retry 3 https://fleetops.itworx.tech/health/ready + openssl s_client -servername fleetops.itworx.tech -connect fleetops.itworx.tech:443 /dev/null \ + | openssl x509 -checkend 1209600 -noout + - name: Install locked Playwright runtime + working-directory: frontend + run: | + npm ci --no-audit --no-fund + npx playwright install --with-deps chromium firefox + - name: Run non-destructive cross-browser production canary + working-directory: frontend + env: + MOBILITYOPS_PUBLIC_URL: https://fleetops.itworx.tech + run: npx playwright test --config=playwright.live.config.ts + - name: Report successful external heartbeat + env: + HEARTBEAT_URL: ${{ secrets.LIVE_CANARY_HEARTBEAT_URL }} + run: | + if [ -n "$HEARTBEAT_URL" ]; then + curl --fail --silent --show-error --retry 3 "$HEARTBEAT_URL" + fi + - name: Upload failure evidence + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: live-canary-failure + path: | + frontend/playwright-live-report + frontend/test-results diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 532ce7d..789f5ac 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -13,6 +13,16 @@ jobs: run: | docker build --target runtime --build-arg VCS_REF="$GITHUB_SHA" --tag mobilityops-api-release --file backend/Dockerfile . docker build --build-arg VCS_REF="$GITHUB_SHA" --tag mobilityops-web-release frontend + docker build --build-arg VCS_REF="$GITHUB_SHA" --tag mobilityops-backup-tools-release --file deploy/unraid/Dockerfile.backup-tools . + - name: Scan all release images + run: | + for image in mobilityops-api-release mobilityops-web-release mobilityops-backup-tools-release; do + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ + -v mobilityops-release-trivy:/root/.cache/ \ + aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969 \ + image --scanners vuln --severity HIGH,CRITICAL \ + --ignore-unfixed --exit-code 1 "$image" + done - name: Generate API CycloneDX SBOM uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # v0.30.0 with: @@ -27,11 +37,20 @@ jobs: image-ref: mobilityops-web-release format: cyclonedx output: mobilityops-web-sbom.cdx.json + - name: Generate backup-tools CycloneDX SBOM + uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # v0.30.0 + with: + scan-type: image + image-ref: mobilityops-backup-tools-release + format: cyclonedx + output: mobilityops-backup-tools-sbom.cdx.json - name: Record immutable image metadata run: | docker image inspect mobilityops-api-release > mobilityops-api-image.json docker image inspect mobilityops-web-release > mobilityops-web-image.json - sha256sum mobilityops-*-sbom.cdx.json mobilityops-*-image.json > SHA256SUMS + docker image inspect mobilityops-backup-tools-release > mobilityops-backup-tools-image.json + python scripts/generate-release-provenance.py + sha256sum mobilityops-*-sbom.cdx.json mobilityops-*-image.json release-provenance.json > SHA256SUMS - name: Upload release evidence uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: @@ -39,4 +58,5 @@ jobs: path: | mobilityops-*-sbom.cdx.json mobilityops-*-image.json + release-provenance.json SHA256SUMS diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index d7daa4a..15d245e 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -1,5 +1,33 @@ # Project state +## M48 — resilient synthetic-demo operations (2026-08-21) + +- Replaced routine Compose recreation with a stable Nginx gateway and a two-API/two-web + start-first promotion. The gateway atomically reloads revision-specific upstream aliases; + old replicas drain only after public readiness. A real rolling test sustained **300/300** + concurrent probes with zero failures. Database, backup and observability containers now + refresh only through an explicit infrastructure command. +- Added an opt-in OneDrive off-site worker using rclone 1.75.0 rebuilt reproducibly with + patched Go 1.26.6. Every upload is downloaded, checksum/list verified and weekly restored + into a disposable database; retention and health markers are enforced. OAuth state stays + outside Git. The synthetic-only scope is explicit; introducing personal data remains out + of scope. +- Added hourly external HTTPS/TLS and Chromium/Firefox canaries, an authenticated read-only + concurrency gate, healthchecks for all monitoring services, and tag evidence for the + API/web/backup images (CycloneDX, immutable metadata, hashes and provenance). +- Split the two remaining source hotspots into bounded backend duplicate-scan and frontend + comparison/CSS modules, then tightened growth budgets around all extracted files. +- Validation: backend **271/271**, Playwright **155/155**, frontend lint/build/audit, + Ruff, strict mypy, OpenAPI/event/MCP/n8n contracts, Compose rendering, ShellCheck, + actionlint, source budgets and shell/Python parsing passed. Read load sustained **360/360** + requests at concurrency 18 with p95 **292.6 ms**. Trivy 0.74 reports zero fixed + HIGH/CRITICAL findings for API and web; the first backup-tools scan caught stale Go + binaries, which were removed/rebuilt and then also scanned clean. +- Exact next action: rebuild the final labelled backup-tools image, commit/push M48, create + and verify a production backup, deploy the exact archive, explicitly refresh monitoring, + run live acceptance and image scans, then record final evidence. OneDrive activation + remains a one-time interactive Microsoft OAuth action after deployment. + ## M47 — final production acceptance evidence (2026-08-21) - Promoted immutable M46 revision `95c91797fa2c599443d69d9c96d83a85ee0711f7` diff --git a/README.md b/README.md index 6fecce5..6937c38 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ make test # isolated PostgreSQL backend suite make lint # Ruff + strict mypy make e2e # complete Playwright browser acceptance cd frontend && npm run build +python scripts/run-readonly-load-smoke.py # while the demo stack is running ``` Release-scoped results and production evidence are recorded in [artifacts/final-acceptance/summary.md](artifacts/final-acceptance/summary.md); older milestone evidence remains explicitly historical. [PROJECT_STATE.md](PROJECT_STATE.md) records the commands and exact deployment revision. diff --git a/backend/app/services/data_quality.py b/backend/app/services/data_quality.py index bc34a0b..c1e7312 100644 --- a/backend/app/services/data_quality.py +++ b/backend/app/services/data_quality.py @@ -3,7 +3,6 @@ from __future__ import annotations import uuid from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta -from difflib import SequenceMatcher from sqlalchemy import func, select, update from sqlalchemy.orm import Session @@ -15,6 +14,7 @@ from app.models.data_quality import DataQualityIssue from app.models.vehicle import Vehicle from app.schemas import CurrentUser, ResolveOdometerRegressionRequest from app.services.audit import record_audit_event +from app.services.data_quality_duplicate_scan import scan_duplicate_customers from app.services.vehicle_status import ( RECOMMENDATION_CODE_NO_CONFLICT, VehicleStatusRecommendation, @@ -25,7 +25,6 @@ from app.services.vehicle_status import ( REQUIRED_CUSTOMER_FIELDS = ("first_name", "last_name") REQUIRED_VEHICLE_FIELDS = ("registration_number", "make", "model", "location") -DUPLICATE_THRESHOLD = 70 DATA_QUALITY_SCAN_LOCK_ID = 6_138_493_717_091_029_491 @@ -46,10 +45,6 @@ class ScanResult: self.created[rule_type] = self.created.get(rule_type, 0) + 1 -def _normalize(value: str | None) -> str: - return (value or "").strip().lower() - - def _has_open_issue(db: Session, rule_type: str, entity_type: str, entity_id: uuid.UUID) -> bool: return ( db.scalar( @@ -129,71 +124,6 @@ def _open_issue( scan.bump(rule_type) -def _scan_duplicate_customers(db: Session, scan: ScanResult) -> None: - customers = list( - db.scalars( - select(Customer).where( - Customer.merged_into_customer_id.is_(None), - Customer.anonymized_at.is_(None), - ) - ).all() - ) - customers.sort(key=lambda c: c.public_ref) - # The threshold cannot be reached without an exact email (60 points) or phone - # (50 points). Block on those normalized identifiers first, so similarity scoring - # scales with plausible candidates instead of comparing every customer pair. - candidate_pairs: set[tuple[int, int]] = set() - for attribute in ("email", "phone"): - blocks: dict[str, list[int]] = {} - for index, customer in enumerate(customers): - key = _normalize(getattr(customer, attribute)) - if key: - blocks.setdefault(key, []).append(index) - for indices in blocks.values(): - for offset, left in enumerate(indices): - candidate_pairs.update((left, right) for right in indices[offset + 1 :]) - - for left, right in sorted(candidate_pairs): - a = customers[left] - b = customers[right] - score = 0 - signals: list[dict] = [] - summary_parts: list[str] = [] - if _normalize(a.email) and _normalize(a.email) == _normalize(b.email): - score += 60 - signals.append({"code": "duplicate.exact_email"}) - summary_parts.append("exact email") - if _normalize(a.phone) and _normalize(a.phone) == _normalize(b.phone): - score += 50 - signals.append({"code": "duplicate.exact_phone"}) - summary_parts.append("exact phone") - if _normalize(a.postal_code) and _normalize(a.postal_code) == _normalize(b.postal_code): - score += 10 - signals.append({"code": "duplicate.same_postal_code"}) - summary_parts.append("exact postal code") - name_a = f"{_normalize(a.first_name)} {_normalize(a.last_name)}" - name_b = f"{_normalize(b.first_name)} {_normalize(b.last_name)}" - ratio = SequenceMatcher(None, name_a, name_b).ratio() - if ratio >= 0.5: - score += round(ratio * 30) - signals.append({"code": "duplicate.similar_name", "params": {"score": round(ratio, 2)}}) - summary_parts.append("similar name") - - if score >= DUPLICATE_THRESHOLD: - _open_issue( - db, - scan, - rule_type="possible_duplicate_customer", - entity_type="customer", - entity_id=a.id, - severity="high", - summary="; ".join(summary_parts) + f" (score {score})", - entity_ref=a.public_ref, - related_refs=[b.public_ref], - signals=signals, - ) - - def _scan_missing_required_fields(db: Session, scan: ScanResult) -> None: # Anonymised customers have had their contact data removed on purpose; flagging # them as "missing required field" would only be resolvable by re-entering PII. @@ -361,7 +291,7 @@ def run_scan( # database boundary so API and n8n triggers cannot both observe an empty condition. db.scalar(select(func.pg_advisory_xact_lock(DATA_QUALITY_SCAN_LOCK_ID))) scan = ScanResult() - _scan_duplicate_customers(db, scan) + scan_duplicate_customers(db, scan, _open_issue) _scan_missing_required_fields(db, scan) _scan_odometer_regressions(db, scan) _scan_booking_overlaps(db, scan) diff --git a/backend/app/services/data_quality_duplicate_scan.py b/backend/app/services/data_quality_duplicate_scan.py new file mode 100644 index 0000000..46507eb --- /dev/null +++ b/backend/app/services/data_quality_duplicate_scan.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import uuid +from difflib import SequenceMatcher +from typing import Protocol, TypeVar + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.models.customer import Customer + +DUPLICATE_THRESHOLD = 70 + + +class ScanAccumulator(Protocol): + def bump(self, rule_type: str) -> None: ... + + +ScanTypeContra = TypeVar("ScanTypeContra", bound=ScanAccumulator, contravariant=True) + + +class OpenIssue(Protocol[ScanTypeContra]): + def __call__( + self, + db: Session, + scan: ScanTypeContra, + *, + rule_type: str, + entity_type: str, + entity_id: uuid.UUID, + severity: str, + summary: str, + entity_ref: str, + related_refs: list[str], + signals: list[dict] | None = None, + ) -> None: ... + + +def _normalize(value: str | None) -> str: + return (value or "").strip().lower() + + +def scan_duplicate_customers[ScanType: ScanAccumulator]( + db: Session, scan: ScanType, open_issue: OpenIssue[ScanType] +) -> None: + customers = list( + db.scalars( + select(Customer).where( + Customer.merged_into_customer_id.is_(None), + Customer.anonymized_at.is_(None), + ) + ).all() + ) + customers.sort(key=lambda customer: customer.public_ref) + + # The threshold cannot be reached without an exact email (60 points) or phone + # (50 points). Block on normalized identifiers so this remains linear for the + # overwhelmingly common case and only scores plausible pairs. + candidate_pairs: set[tuple[int, int]] = set() + for attribute in ("email", "phone"): + blocks: dict[str, list[int]] = {} + for index, customer in enumerate(customers): + key = _normalize(getattr(customer, attribute)) + if key: + blocks.setdefault(key, []).append(index) + for indices in blocks.values(): + for offset, left in enumerate(indices): + candidate_pairs.update((left, right) for right in indices[offset + 1 :]) + + for left, right in sorted(candidate_pairs): + a = customers[left] + b = customers[right] + score = 0 + signals: list[dict] = [] + summary_parts: list[str] = [] + if _normalize(a.email) and _normalize(a.email) == _normalize(b.email): + score += 60 + signals.append({"code": "duplicate.exact_email"}) + summary_parts.append("exact email") + if _normalize(a.phone) and _normalize(a.phone) == _normalize(b.phone): + score += 50 + signals.append({"code": "duplicate.exact_phone"}) + summary_parts.append("exact phone") + if _normalize(a.postal_code) and _normalize(a.postal_code) == _normalize(b.postal_code): + score += 10 + signals.append({"code": "duplicate.same_postal_code"}) + summary_parts.append("exact postal code") + name_a = f"{_normalize(a.first_name)} {_normalize(a.last_name)}" + name_b = f"{_normalize(b.first_name)} {_normalize(b.last_name)}" + ratio = SequenceMatcher(None, name_a, name_b).ratio() + if ratio >= 0.5: + score += round(ratio * 30) + signals.append( + {"code": "duplicate.similar_name", "params": {"score": round(ratio, 2)}} + ) + summary_parts.append("similar name") + + if score >= DUPLICATE_THRESHOLD: + open_issue( + db, + scan, + rule_type="possible_duplicate_customer", + entity_type="customer", + entity_id=a.id, + severity="high", + summary="; ".join(summary_parts) + f" (score {score})", + entity_ref=a.public_ref, + related_refs=[b.public_ref], + signals=signals, + ) diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index a722165..f56e1ab 100644 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -1,5 +1,7 @@ #!/bin/sh set -e -alembic upgrade head +if [ "${RUN_MIGRATIONS:-true}" = "true" ]; then + alembic upgrade head +fi exec uvicorn app.main:app --host 0.0.0.0 --port 8000 diff --git a/compose.observability.yaml b/compose.observability.yaml index 57087dd..8d22bf7 100644 --- a/compose.observability.yaml +++ b/compose.observability.yaml @@ -17,6 +17,11 @@ services: - mobilityops-prometheus:/prometheus ports: - "127.0.0.1:19090:9090" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"] + interval: 15s + timeout: 5s + retries: 10 restart: unless-stopped networks: [mobilityops] @@ -61,6 +66,11 @@ services: - mobilityops-grafana:/var/lib/grafana ports: - "127.0.0.1:13000:3000" + healthcheck: + test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1:3000/api/health | grep -Eq '\"database\"[[:space:]]*:[[:space:]]*\"ok\"'"] + interval: 15s + timeout: 5s + retries: 20 depends_on: [prometheus, alertmanager] restart: unless-stopped networks: [mobilityops] diff --git a/compose.release.yaml b/compose.release.yaml index 755ea10..4c139bd 100644 --- a/compose.release.yaml +++ b/compose.release.yaml @@ -2,6 +2,8 @@ services: api: image: ${MOBILITYOPS_API_IMAGE:?Set MOBILITYOPS_API_IMAGE to an immutable release image} build: !reset null + environment: + RUN_MIGRATIONS: "false" web: image: ${MOBILITYOPS_WEB_IMAGE:?Set MOBILITYOPS_WEB_IMAGE to an immutable release image} diff --git a/compose.unraid.yaml b/compose.unraid.yaml index 27492ae..4a82b71 100644 --- a/compose.unraid.yaml +++ b/compose.unraid.yaml @@ -9,13 +9,30 @@ services: web: restart: unless-stopped ports: !override - - "1236:80" + [] healthcheck: test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1/health"] interval: 10s timeout: 5s retries: 10 + gateway: + image: nginx:1.30.4-alpine@sha256:97d490c12ba55b4946b01546d1c3ed324e8d41ab1c9fcb2a616aa470620e5b46 + restart: unless-stopped + ports: + - "1236:80" + volumes: + - ${MOBILITYOPS_GATEWAY_CONFIG:-./deploy/unraid/gateway.conf}:/etc/nginx/conf.d/default.conf:ro + depends_on: + web: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1/health"] + interval: 10s + timeout: 5s + retries: 10 + networks: [mobilityops] + n8n: # The review server already has a shared n8n instance. Keep the bundled service # available for an explicit fallback without starting a second instance by default. @@ -67,3 +84,46 @@ services: retries: 3 start_period: 5m networks: [mobilityops] + + offsite-backup: + profiles: ["offsite"] + image: ${MOBILITYOPS_BACKUP_TOOLS_IMAGE:-mobilityops-backup-tools:development} + build: + context: . + dockerfile: deploy/unraid/Dockerfile.backup-tools + args: + VCS_REF: ${MOBILITYOPS_SOURCE_REVISION:-development} + restart: unless-stopped + entrypoint: ["/opt/mobilityops/sync-onedrive-backups.sh"] + environment: + POSTGRES_DB: ${POSTGRES_DB:-mobilityops} + POSTGRES_USER: ${POSTGRES_USER:-mobilityops} + PGPASSWORD: ${POSTGRES_PASSWORD:-mobilityops} + POSTGRES_HOST: db + BACKUP_DESTINATION: /backups + BACKUP_OFFSITE_VERIFY_DIR: /offsite-verify + BACKUP_OFFSITE_INTERVAL_SECONDS: ${BACKUP_OFFSITE_INTERVAL_SECONDS:-900} + BACKUP_RESTORE_DRILL_INTERVAL_SECONDS: ${BACKUP_RESTORE_DRILL_INTERVAL_SECONDS:-604800} + BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-30} + RCLONE_CONFIG: /config/rclone/rclone.conf + RCLONE_ONEDRIVE_REMOTE: ${RCLONE_ONEDRIVE_REMOTE:-onedrive} + RCLONE_ONEDRIVE_PATH: ${RCLONE_ONEDRIVE_PATH:-FleetOps/backups} + volumes: + - ${MOBILITYOPS_BACKUP_DIR:-./backups/postgres}:/backups + - ${MOBILITYOPS_OFFSITE_VERIFY_DIR:-./backups/offsite-verify}:/offsite-verify + - ${MOBILITYOPS_RCLONE_CONFIG_DIR:-./.secrets/rclone}:/config/rclone + - ./deploy/unraid:/opt/mobilityops:ro + depends_on: + db: + condition: service_healthy + healthcheck: + test: + [ + "CMD-SHELL", + "find /backups/latest-offsite-success -mmin -1560 -print -quit | grep -q . && find /backups/latest-offsite-restore-drill -mmin -11520 -print -quit | grep -q .", + ] + interval: 30m + timeout: 5s + retries: 3 + start_period: 10m + networks: [mobilityops] diff --git a/deploy/unraid/Dockerfile.backup-tools b/deploy/unraid/Dockerfile.backup-tools new file mode 100644 index 0000000..0b1af8c --- /dev/null +++ b/deploy/unraid/Dockerfile.backup-tools @@ -0,0 +1,17 @@ +FROM golang:1.26.6-alpine@sha256:3889b425f035be855a72fb4755265311293b6d414521f0a519d819df32222d83 AS rclone +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 go install -trimpath \ + -ldflags '-s -X github.com/rclone/rclone/fs.Version=v1.75.0' \ + github.com/rclone/rclone@v1.75.0 + +FROM postgres:16-alpine@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685 +ARG VCS_REF=development +ARG BUILD_DATE=unknown +LABEL org.opencontainers.image.title="MobilityOps backup tools" \ + org.opencontainers.image.source="https://github.com/ITWorxBE/MobilityOps" \ + org.opencontainers.image.revision="$VCS_REF" \ + org.opencontainers.image.created="$BUILD_DATE" +COPY --from=rclone /go/bin/rclone /usr/local/bin/rclone +# The PostgreSQL image's entrypoint uses gosu only when starting the database server; +# this tools-only image always overrides that entrypoint with the sync worker. +RUN rm -f /usr/local/bin/gosu diff --git a/deploy/unraid/README.md b/deploy/unraid/README.md index b88a22c..9e85506 100644 --- a/deploy/unraid/README.md +++ b/deploy/unraid/README.md @@ -33,10 +33,19 @@ sha256sum /tmp/mobilityops-source.tar.gz ``` `deploy-release.sh` stages a clean, commit-named release, builds OCI-labelled immutable -API/web images, promotes without a seed/reset, and verifies migrations, readiness, -backups and observability. Run `python -m app.cli seed --reset` only for initial setup or +API/web images, starts two candidate replicas per stateless service behind a stable gateway, +and removes the previous replicas only after public readiness passes. It promotes without a +seed/reset and leaves the database, backups and monitoring untouched during routine app +releases. Run `python -m app.cli seed --reset` only for initial setup or an explicit synthetic-demo reset; it is never part of a routine deployment. +Refresh the pinned stateful and monitoring containers explicitly after reviewing their +version/configuration changes: + +```bash +./deploy/unraid/refresh-infrastructure.sh +``` + Migrations run automatically in the API entrypoint. Import and publish the MobilityOps workflow into the existing n8n container: @@ -66,9 +75,22 @@ The `backup` service creates a backup immediately and then every 24 hours. Every checked by SHA-256 and `pg_restore --list`; at least weekly the newest dump is also restored into a disposable database and its migration revision and core table counts are verified. Backups are retained for 30 days with at least seven copies protected from pruning. Its -healthcheck becomes unhealthy when the daily backup or eight-day restore-drill SLA is missed. Configure -`BACKUP_SECONDARY_DESTINATION=/offsite` plus an independently mounted -`MOBILITYOPS_BACKUP_SECONDARY_DIR` for a second copy. +healthcheck becomes unhealthy when the daily backup or eight-day restore-drill SLA is missed. + +For this synthetic demo, OneDrive is the supported real off-site target. The optional worker +uploads the newest locally verified dump and checksum, downloads both again, verifies the +round trip, and performs a weekly restore into a disposable database. Its OAuth token remains +in the untracked mode-0600 rclone configuration directory. Configure it once from an +interactive server terminal (Microsoft browser authorization is required): + +```bash +./deploy/unraid/configure-onedrive-backup.sh onedrive +docker inspect --format '{{.State.Health.Status}}' mobilityops-offsite-backup-1 +``` + +Set `RCLONE_ONEDRIVE_PATH` to change the default `FleetOps/backups` folder. Do not set +`BACKUP_SECONDARY_DESTINATION` to another path on the same server and call that off-site; +that legacy mount remains available only for an independently mounted filesystem. Create an additional on-demand backup, verify the newest backup, or execute the isolated restore drill: diff --git a/deploy/unraid/configure-onedrive-backup.sh b/deploy/unraid/configure-onedrive-backup.sh new file mode 100644 index 0000000..72befaa --- /dev/null +++ b/deploy/unraid/configure-onedrive-backup.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +root="${MOBILITYOPS_DEPLOY_ROOT:-/mnt/user/appdata/mobilityops}" +config_dir="${MOBILITYOPS_RCLONE_CONFIG_DIR:-$root/.secrets/rclone}" +remote="${1:-onedrive}" + +case "$config_dir" in ""|"/"|".") echo "Unsafe rclone configuration directory" >&2; exit 1;; esac +case "$remote" in ""|*:*|*/*) echo "Remote must be a simple rclone remote name" >&2; exit 1;; esac +mkdir -p "$config_dir" +chmod 700 "$config_dir" + +revision="$(cat "$root/.deploy/source-revision")" +release_dir="$root/.deploy/releases/$revision" +short_revision="$(printf '%s' "$revision" | cut -c1-12)" +image="mobilityops-backup-tools:$short_revision" +docker build --build-arg "VCS_REF=$revision" --tag "$image" \ + --file "$release_dir/deploy/unraid/Dockerfile.backup-tools" "$release_dir" + +echo "Configure Microsoft OneDrive as remote '$remote'; OAuth opens in your browser." +docker run --rm -it --entrypoint rclone -v "$config_dir:/config/rclone" "$image" config +chmod 600 "$config_dir/rclone.conf" +docker run --rm --entrypoint rclone -v "$config_dir:/config/rclone" "$image" lsd "$remote:" + +history="$(tail -n 1 "$root/.deploy/release-history.log")" +# shellcheck disable=SC2086 +set -- $history +[ "$1" = "$revision" ] || { echo "Release history does not match active revision" >&2; exit 1; } +export MOBILITYOPS_API_IMAGE="$2" MOBILITYOPS_WEB_IMAGE="$3" +export MOBILITYOPS_SOURCE_REVISION="$revision" +export MOBILITYOPS_BACKUP_TOOLS_IMAGE="$image" +export MOBILITYOPS_RCLONE_CONFIG_DIR="$config_dir" +export MOBILITYOPS_BACKUP_DIR="${MOBILITYOPS_BACKUP_DIR:-$root/backups/postgres}" +export MOBILITYOPS_OFFSITE_VERIFY_DIR="${MOBILITYOPS_OFFSITE_VERIFY_DIR:-$root/backups/offsite-verify}" +export RCLONE_ONEDRIVE_REMOTE="$remote" +compose="docker compose --env-file $root/.env -p ${COMPOSE_PROJECT_NAME:-mobilityops} -f $release_dir/compose.yaml -f $release_dir/compose.unraid.yaml -f $release_dir/compose.observability.yaml -f $release_dir/compose.release.yaml --profile offsite" +$compose up --no-build -d offsite-backup +echo "OneDrive remote '$remote' is reachable and the off-site backup worker is starting." diff --git a/deploy/unraid/deploy-release.sh b/deploy/unraid/deploy-release.sh index 8a2ec7c..741b425 100755 --- a/deploy/unraid/deploy-release.sh +++ b/deploy/unraid/deploy-release.sh @@ -50,29 +50,191 @@ for image in "$api_image" "$web_image"; do done compose="docker compose --env-file $root/.env -p $project -f $release_dir/compose.yaml -f $release_dir/compose.unraid.yaml -f $release_dir/compose.observability.yaml -f $release_dir/compose.release.yaml --profile observability" -old_api_id="$(docker inspect --format '{{.Image}}' "$project-api-1" 2>/dev/null || true)" -old_web_id="$(docker inspect --format '{{.Image}}' "$project-web-1" 2>/dev/null || true)" export MOBILITYOPS_API_IMAGE="$api_image" MOBILITYOPS_WEB_IMAGE="$web_image" -$compose up --no-build -d api web backup prometheus alertmanager grafana +gateway_config="$root/.deploy/gateway.conf" +candidate_gateway="$(mktemp "$root/.deploy/gateway-candidate.XXXXXX")" +previous_gateway="$(mktemp "$root/.deploy/gateway-previous.XXXXXX")" +sed -e "s/server web:80/server web-$short_revision:80/" \ + -e "s/server api:8000/server api-$short_revision:8000/" \ + "$release_dir/deploy/unraid/gateway.conf" > "$candidate_gateway" +if [ -f "$gateway_config" ]; then + cp "$gateway_config" "$previous_gateway" +else + cp "$candidate_gateway" "$gateway_config" +fi +export MOBILITYOPS_GATEWAY_CONFIG="$gateway_config" + +# Bootstrap infrastructure only when it is absent. Existing stateful/monitoring +# containers are deliberately not reconciled against every commit-named source path. +missing_infrastructure="" +for service in db backup prometheus alertmanager grafana; do + [ -n "$($compose ps -q "$service")" ] || missing_infrastructure="$missing_infrastructure $service" +done +if [ -n "$missing_infrastructure" ]; then + # shellcheck disable=SC2086 + $compose up --no-build -d $missing_infrastructure +fi + +# Apply schema changes once before starting the new stateless replicas. Migrations in a +# release must remain backwards compatible with the still-serving previous API. +$compose run --rm --no-deps --entrypoint alembic api upgrade head + +network="$(docker inspect --format '{{range $name, $network := .NetworkSettings.Networks}}{{$name}}{{end}}' "$project-db-1")" +[ -n "$network" ] || { echo "MobilityOps network was not found" >&2; exit 1; } + +old_api_ids="$(docker ps -q --filter label=com.mobilityops.role=api --filter "label=com.mobilityops.project=$project")" +old_web_ids="$(docker ps -q --filter label=com.mobilityops.role=web --filter "label=com.mobilityops.project=$project")" +if [ -z "$old_api_ids" ]; then + old_api_ids="$(docker ps -q --filter "name=^/${project}-api-")" +fi +if [ -z "$old_web_ids" ]; then + old_web_ids="$(docker ps -q --filter "name=^/${project}-web-")" +fi + +# Reuse the effective, already-secret-resolved API environment without printing it. +api_environment="$(mktemp "$root/.deploy/api-environment.XXXXXX")" +chmod 600 "$api_environment" +cleanup() { + rm -f "$api_environment" "$candidate_gateway" "$previous_gateway" +} +trap cleanup EXIT INT TERM +if [ -n "$old_api_ids" ]; then + # shellcheck disable=SC2086 + first_old_api="$(printf '%s\n' $old_api_ids | head -n 1)" + docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$first_old_api" > "$api_environment" +else + # A clean bootstrap first lets Compose resolve the complete API environment. This + # container is retained as the previous slot until the candidates pass. + $compose up --no-build --no-deps -d api web + old_api_ids="$($compose ps -q api)" + old_web_ids="$($compose ps -q web)" + docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$old_api_ids" > "$api_environment" +fi + +new_api_ids="" +new_web_ids="" +old_web_image="" +# shellcheck disable=SC2086 +[ -z "$old_web_ids" ] || old_web_image="$(docker inspect --format '{{.Config.Image}}' "$(printf '%s\n' $old_web_ids | head -n 1)")" +for replica in 1 2; do + name="$project-api-$short_revision-$replica" + id="$(docker run -d --name "$name" --restart unless-stopped \ + --label com.mobilityops.role=api --label "com.mobilityops.project=$project" \ + --label "com.mobilityops.revision=$revision" \ + --network "$network" --network-alias "api-$short_revision" --env-file "$api_environment" \ + --env RUN_MIGRATIONS=false "$api_image")" + new_api_ids="$new_api_ids $id" +done + +wait_for_ids() { + role="$1" + shift + attempt=0 + while true; do + healthy=0 + count=0 + for id in "$@"; do + count=$((count + 1)) + if [ "$role" = "api" ]; then + docker exec "$id" python -c \ + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/ready')" \ + > /dev/null 2>&1 && healthy=$((healthy + 1)) + else + docker exec "$id" wget -q -O /dev/null http://127.0.0.1/health \ + > /dev/null 2>&1 && healthy=$((healthy + 1)) + fi + done + [ "$count" -eq 2 ] && [ "$healthy" -eq 2 ] && return 0 + attempt=$((attempt + 1)) + [ "$attempt" -lt 60 ] || { echo "$role candidates did not become healthy" >&2; return 1; } + sleep 2 + done +} + +# shellcheck disable=SC2086 +if ! wait_for_ids api $new_api_ids; then + docker rm -f $new_api_ids > /dev/null 2>&1 || true + exit 1 +fi + +for replica in 1 2; do + name="$project-web-$short_revision-$replica" + id="$(docker run -d --name "$name" --restart unless-stopped \ + --label com.mobilityops.role=web --label "com.mobilityops.project=$project" \ + --label "com.mobilityops.revision=$revision" \ + --network "$network" --network-alias "web-$short_revision" "$web_image")" + new_web_ids="$new_web_ids $id" +done +# shellcheck disable=SC2086 +if ! wait_for_ids web $new_web_ids; then + docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true + exit 1 +fi + +# Install the stable gateway once. Future releases leave it running while Docker DNS +# exposes both old and new web aliases. The first migration from direct port ownership +# necessarily has a brief hand-off while port 1236 moves to the gateway. +gateway_id="$($compose ps -q gateway)" +cp "$candidate_gateway" "$gateway_config" +if [ -z "$gateway_id" ]; then + if [ -n "$old_web_ids" ]; then + # shellcheck disable=SC2086 + docker rm -f $old_web_ids > /dev/null + old_web_ids="" + fi + $compose up --no-build --no-deps -d gateway +else + if ! docker exec "$gateway_id" nginx -t; then + [ ! -s "$previous_gateway" ] || cp "$previous_gateway" "$gateway_config" + # shellcheck disable=SC2086 + docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true + echo "Candidate gateway configuration was rejected; previous replicas remain active" >&2 + exit 1 + fi + docker exec "$gateway_id" nginx -s reload +fi attempt=0 until curl -fsS http://127.0.0.1:1236/health/ready > /dev/null; do attempt=$((attempt + 1)) if [ "$attempt" -ge 30 ]; then - if [ -n "$old_api_id" ] && [ -n "$old_web_id" ]; then - rollback_api="mobilityops-api:rollback-$short_revision" - rollback_web="mobilityops-web:rollback-$short_revision" - docker tag "$old_api_id" "$rollback_api" - docker tag "$old_web_id" "$rollback_web" - export MOBILITYOPS_API_IMAGE="$rollback_api" MOBILITYOPS_WEB_IMAGE="$rollback_web" - $compose up --no-build -d api web || true + if [ -n "$gateway_id" ] && [ -s "$previous_gateway" ]; then + cp "$previous_gateway" "$gateway_config" + docker exec "$gateway_id" nginx -t && docker exec "$gateway_id" nginx -s reload + # shellcheck disable=SC2086 + docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true + elif [ -n "$old_web_image" ]; then + # The first gateway migration keeps the already healthy candidate APIs because + # the rendered gateway points at their versioned alias. + # shellcheck disable=SC2086 + docker rm -f $new_web_ids > /dev/null 2>&1 || true + for replica in 1 2; do + docker run -d --name "$project-web-rollback-$short_revision-$replica" --restart unless-stopped \ + --label com.mobilityops.role=web --label "com.mobilityops.project=$project" \ + --label com.mobilityops.revision=rollback \ + --network "$network" --network-alias "web-$short_revision" "$old_web_image" > /dev/null + done + else + # shellcheck disable=SC2086 + docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true fi - echo "Release failed readiness; source revision was not promoted" >&2 + echo "Release failed readiness; previous API and restored web replicas remain active" >&2 exit 1 fi sleep 2 done -$compose exec -T api alembic current + +# Promotion is start-first: only now remove the previous containers. A short drain lets +# workers using the pre-reload configuration complete their in-flight requests. +sleep 3 +# shellcheck disable=SC2086 +[ -z "$old_api_ids" ] || docker rm -f $old_api_ids > /dev/null +# shellcheck disable=SC2086 +[ -z "$old_web_ids" ] || docker rm -f $old_web_ids > /dev/null +curl -fsS http://127.0.0.1:1236/health/ready > /dev/null +# shellcheck disable=SC2086 +first_new_api="$(printf '%s\n' $new_api_ids | head -n 1)" +docker exec "$first_new_api" alembic current for service in backup prometheus alertmanager grafana; do attempt=0 until status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$project-$service-1" 2>/dev/null)" \ diff --git a/deploy/unraid/gateway.conf b/deploy/unraid/gateway.conf new file mode 100644 index 0000000..bca727d --- /dev/null +++ b/deploy/unraid/gateway.conf @@ -0,0 +1,58 @@ +resolver 127.0.0.11 valid=2s ipv6=off; +limit_req_zone $binary_remote_addr zone=demo_login:10m rate=600r/m; + +upstream fleet_ops_web { + zone fleet_ops_web 64k; + server web:80 resolve max_fails=1 fail_timeout=2s; + keepalive 32; +} + +upstream fleet_ops_api { + zone fleet_ops_api 64k; + server api:8000 resolve max_fails=1 fail_timeout=2s; + keepalive 32; +} + +server { + listen 80; + server_name _; + server_tokens off; + + location = /api/v1/demo/login { + limit_req zone=demo_login burst=100 nodelay; + proxy_pass http://fleet_ops_api; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /api/ { + proxy_pass http://fleet_ops_api; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /health { + proxy_pass http://fleet_ops_api/health; + proxy_http_version 1.1; + proxy_set_header Connection ""; + } + + location / { + proxy_pass http://fleet_ops_web; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 3; + proxy_next_upstream_timeout 3s; + proxy_connect_timeout 1s; + } +} diff --git a/deploy/unraid/refresh-infrastructure.sh b/deploy/unraid/refresh-infrastructure.sh new file mode 100644 index 0000000..c4d2dfb --- /dev/null +++ b/deploy/unraid/refresh-infrastructure.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu + +root="${MOBILITYOPS_DEPLOY_ROOT:-/mnt/user/appdata/mobilityops}" +project="${COMPOSE_PROJECT_NAME:-mobilityops}" +revision="$(cat "$root/.deploy/source-revision")" +release_dir="$root/.deploy/releases/$revision" +[ -d "$release_dir" ] || { echo "Active release directory is missing" >&2; exit 1; } + +history="$(tail -n 1 "$root/.deploy/release-history.log")" +# shellcheck disable=SC2086 +set -- $history +[ "$1" = "$revision" ] || { echo "Release history does not match active revision" >&2; exit 1; } +export MOBILITYOPS_API_IMAGE="$2" MOBILITYOPS_WEB_IMAGE="$3" +compose="docker compose --env-file $root/.env -p $project -f $release_dir/compose.yaml -f $release_dir/compose.unraid.yaml -f $release_dir/compose.observability.yaml -f $release_dir/compose.release.yaml --profile observability" + +$compose up --no-build -d db backup prometheus alertmanager grafana +for service in db backup prometheus alertmanager grafana; do + attempt=0 + until status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$project-$service-1" 2>/dev/null)" \ + && [ "$status" = "healthy" ]; do + attempt=$((attempt + 1)) + [ "$attempt" -lt 60 ] || { echo "$service did not become healthy" >&2; exit 1; } + sleep 5 + done +done +echo "Refreshed Fleet Ops infrastructure for $revision" diff --git a/deploy/unraid/sync-onedrive-backups.sh b/deploy/unraid/sync-onedrive-backups.sh new file mode 100644 index 0000000..8b6adb6 --- /dev/null +++ b/deploy/unraid/sync-onedrive-backups.sh @@ -0,0 +1,67 @@ +#!/bin/sh +set -eu + +destination="${BACKUP_DESTINATION:-/backups}" +verify_dir="${BACKUP_OFFSITE_VERIFY_DIR:-/offsite-verify}" +remote="${RCLONE_ONEDRIVE_REMOTE:-onedrive}" +remote_path="${RCLONE_ONEDRIVE_PATH:-FleetOps/backups}" +interval="${BACKUP_OFFSITE_INTERVAL_SECONDS:-900}" +restore_drill_interval="${BACKUP_RESTORE_DRILL_INTERVAL_SECONDS:-604800}" +retention_days="${BACKUP_RETENTION_DAYS:-30}" + +case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac +case "$verify_dir" in ""|"/"|".") echo "Unsafe off-site verify directory: $verify_dir" >&2; exit 1;; esac +case "$remote" in ""|*:*|*/*) echo "Rclone remote must be a configured remote name" >&2; exit 1;; esac +case "$remote_path" in ""|"/"|/*|*".."*) echo "Unsafe OneDrive backup path" >&2; exit 1;; esac +case "$interval:$restore_drill_interval:$retention_days" in *[!0-9:]*|:*|*:) echo "Off-site intervals and retention must be integers" >&2; exit 1;; esac +[ "$retention_days" -ge 1 ] || { echo "Off-site retention must be at least one day" >&2; exit 1; } +[ -f "${RCLONE_CONFIG:-/config/rclone/rclone.conf}" ] || { + echo "Rclone configuration is missing" >&2 + exit 1 +} + +mkdir -p "$verify_dir" + +while true; do + latest="$(find "$destination" -maxdepth 1 -type f -name 'mobilityops-*.dump' | sort -r | head -n 1)" + if [ -n "$latest" ] && [ -f "$latest.sha256" ]; then + name="$(basename "$latest")" + checksum_name="$name.sha256" + remote_root="$remote:$remote_path" + temporary_dump="$verify_dir/$name.partial" + temporary_checksum="$verify_dir/$checksum_name.partial" + verified_dump="$verify_dir/$name" + + rm -f "$temporary_dump" "$temporary_checksum" + if rclone copyto "$latest" "$remote_root/$name" --checksum --retries 3 --low-level-retries 5 \ + && rclone copyto "$latest.sha256" "$remote_root/$checksum_name" --checksum --retries 3 --low-level-retries 5 \ + && rclone copyto "$remote_root/$name" "$temporary_dump" --retries 3 --low-level-retries 5 \ + && rclone copyto "$remote_root/$checksum_name" "$temporary_checksum" --retries 3 --low-level-retries 5 \ + && mv "$temporary_dump" "$verified_dump" \ + && mv "$temporary_checksum" "$verify_dir/$checksum_name" \ + && (cd "$verify_dir" && sha256sum -c "$checksum_name") \ + && pg_restore --list "$verified_dump" > /dev/null; then + date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-offsite-success" + drill_minutes=$((restore_drill_interval / 60)) + if [ ! -f "$destination/latest-offsite-restore-drill" ] \ + || ! find "$destination/latest-offsite-restore-drill" -mmin "-$drill_minutes" -print -quit | grep -q .; then + if /opt/mobilityops/restore-drill-postgres.sh "$verified_dump"; then + date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-offsite-restore-drill" + else + echo "Off-site restore drill failed for $name" >&2 + fi + fi + rclone delete "$remote_root" --min-age "${retention_days}d" \ + --include 'mobilityops-*.dump' --include 'mobilityops-*.dump.sha256' \ + --retries 3 --low-level-retries 5 + find "$verify_dir" -type f -mtime "+$retention_days" -delete + echo "Verified OneDrive round trip: $remote_root/$name" + else + rm -f "$temporary_dump" "$temporary_checksum" + echo "OneDrive backup synchronization failed for $name" >&2 + fi + else + echo "No verified local backup is available for OneDrive synchronization" >&2 + fi + sleep "$interval" +done diff --git a/docs/17-runbook.md b/docs/17-runbook.md index 15c6ee6..c3b2fbe 100644 --- a/docs/17-runbook.md +++ b/docs/17-runbook.md @@ -56,10 +56,19 @@ unique `GRAFANA_ADMIN_PASSWORD` before first start. Provisioning includes the Mo overview dashboard and alerts for API/database outage, 5xx rate, p95 latency, real outbox backlog and real outbox failures. Synthetic retry scenarios never trigger outbox alerts. +The hourly `live-canary.yml` workflow independently checks HTTPS readiness, the TLS +certificate horizon and the non-destructive Chromium/Firefox production journey. Configure +the optional `LIVE_CANARY_HEARTBEAT_URL` repository secret to make missed scheduled jobs +visible in an external dead-man monitor. A successful app release does not restart the +database, backup or monitoring services; use `deploy/unraid/refresh-infrastructure.sh` +deliberately when those definitions change. + Never execute `pytest` inside the deployed API container: the acceptance fixtures reset their database deliberately. `make test` uses `compose.test.yaml`, a fixed `mobilityops-test` Compose project and its own disposable PostgreSQL volume, and removes that project on success or failure. The Gitea workflow uses the same isolation boundary. +CI also runs `scripts/run-readonly-load-smoke.py` against persisted list/dashboard routes; +the gate requires zero HTTP errors and a p95 below 1.5 seconds at its bounded concurrency. ## Operational mode (non-demo login) @@ -242,3 +251,7 @@ Publish the scheduled quality-scan workflow the same way: independently-authenticated API surface; - failed demo experiment: Operations Manager reset (`POST /api/v1/demo/reset`) restores the deterministic seed, including all named S1–S6 demo scenarios. +- loss of the Unraid host: recover the latest OneDrive dump only after its downloaded + checksum and `pg_restore --list` pass; use the guarded restore command above. OneDrive + synchronization health requires both a daily verified round trip and a weekly real + restore drill. OAuth setup is documented in `deploy/unraid/README.md`. diff --git a/docs/18-privacy-governance.md b/docs/18-privacy-governance.md index 93afd8c..14a74a5 100644 --- a/docs/18-privacy-governance.md +++ b/docs/18-privacy-governance.md @@ -15,8 +15,10 @@ retention values. Operations Managers are the only role permitted to export or a - Audit events are configured for 2,555 days by default. They are immutable operational evidence; changing this period requires legal approval and a separate, audited purge implementation. MobilityOps reports the policy but never silently deletes audit data. -- Database backups default to 30 days with at least seven newest recovery points. A - configured secondary mount must follow the same policy. +- Database backups default to 30 days with at least seven newest recovery points. The demo's + optional OneDrive off-site copy contains the same synthetic dataset and follows the same + retention intent; remove that folder when retiring the demo. Real personal data remains + out of scope and must not be introduced merely because cloud backup is available. - Logs are bounded by Docker rotation. They must not contain request bodies, passwords, OIDC tokens or customer fields. diff --git a/frontend/src/pages/DataQualityIssueDetail.tsx b/frontend/src/pages/DataQualityIssueDetail.tsx index 49f6c3d..58334bd 100644 --- a/frontend/src/pages/DataQualityIssueDetail.tsx +++ b/frontend/src/pages/DataQualityIssueDetail.tsx @@ -6,7 +6,6 @@ import { describeApiError, type ApiErrorInfo } from "../api/errorMessages"; import type { ApplyRecommendedStatusResult, DataQualityIssueDetail as IssueDetail, - EntitySnapshot, StatusRecommendation, } from "../api/types"; import { SeverityBadge, StatusBadge } from "../components/Badge"; @@ -19,8 +18,7 @@ import { describeEvidenceSignal } from "../data/evidenceSignals"; import type { EvidenceSignal } from "../api/types"; import { Icon } from "../components/Icons"; import { ApiErrorNotice, ErrorState, LoadingState, PageHeader, SectionHeading } from "../components/PageChrome"; - -const MERGE_FIELDS = ["first_name", "last_name", "email", "phone", "postal_code", "city"]; +import { DuplicateCustomerPanel } from "./data-quality/DuplicateCustomerPanel"; function RuleExplainer({ ruleType }: { ruleType: string }) { const { t } = useTranslation("quality"); @@ -69,189 +67,6 @@ function EvidenceSignalList({ issue }: { issue: IssueDetail }) { ); } -function DuplicateCustomerPanel({ issue, onResolved }: { issue: IssueDetail; onResolved: () => void }) { - const { t } = useTranslation("quality"); - const { user } = useAuth(); - const [survivorRef, setSurvivorRef] = useState(issue.entity_snapshot?.public_ref ?? ""); - const [fieldChoices, setFieldChoices] = useState>({}); - const [error, setError] = useState(null); - const [submitting, setSubmitting] = useState(false); - const [confirming, setConfirming] = useState(false); - const [showMatching, setShowMatching] = useState(false); - - if (!issue.entity_snapshot || !issue.related_snapshots[0]) { - return

{t("detail.duplicateCustomer.bothMissing")}

; - } - const a: EntitySnapshot = issue.entity_snapshot; - const b: EntitySnapshot = issue.related_snapshots[0]; - - const survivor = survivorRef === a.public_ref ? a : b; - const loser = survivorRef === a.public_ref ? b : a; - const conflictingFields = MERGE_FIELDS.filter((field) => String(a[field] ?? "") !== String(b[field] ?? "")); - const matchingFields = MERGE_FIELDS.filter((field) => !conflictingFields.includes(field)); - const visibleFields = showMatching ? MERGE_FIELDS : conflictingFields; - - async function handleMerge() { - setError(null); - setSubmitting(true); - try { - const overrides: Record = {}; - for (const field of MERGE_FIELDS) { - const choice = fieldChoices[field]; - const chosenSide = choice === "a" ? a : choice === "b" ? b : survivor; - if (chosenSide !== survivor && chosenSide[field]) { - overrides[field] = String(chosenSide[field]); - } - } - await api.post(`/api/v1/data-quality/issues/${issue.public_ref}/merge-customers`, { - survivor_ref: survivor.public_ref, - field_overrides: Object.keys(overrides).length > 0 ? overrides : undefined, - }); - onResolved(); - } catch (err) { - setError(describeApiError(t, err, "detail.duplicateCustomer.mergeFailed")); - setConfirming(false); - } finally { - setSubmitting(false); - } - } - - if (user?.role !== "operations_manager") { - return ( -

{t("detail.managerOnlyDetail")}

- ); - } - - return ( -
- - - -

- {t("detail.duplicateCustomer.summaryCounts", { matchCount: matchingFields.length, conflictCount: conflictingFields.length })} -

- -
- {t("detail.duplicateCustomer.keepAsSurvivor")} - - -
- - - - - - - - - - - {visibleFields.map((field) => { - const valueA = a[field] ? String(a[field]) : "—"; - const valueB = b[field] ? String(b[field]) : "—"; - const differ = valueA !== valueB; - return ( - - - - - - ); - })} - -
{t("detail.duplicateCustomer.fieldColumn")}{a.public_ref}{b.public_ref}
{t(`detail.duplicateCustomer.fields.${field}`)}{differ ? {t("detail.duplicateCustomer.differs")} : {t("detail.duplicateCustomer.match")}} - {differ ? ( - - ) : ( - valueA - )} - - {differ ? ( - - ) : ( - valueB - )} -
- - {matchingFields.length > 0 && ( - - )} - -

- {t("detail.duplicateCustomer.mergePreview", { loser: loser.public_ref, survivor: survivor.public_ref })} -

- -
-

{t("detail.duplicateCustomer.previewTitle", { ref: survivor.public_ref })}

-
- {MERGE_FIELDS.map((field) => { - const choice = fieldChoices[field]; - const chosenSide = choice === "a" ? a : choice === "b" ? b : survivor; - const value = (chosenSide[field] ? String(chosenSide[field]) : survivor[field] ? String(survivor[field]) : "—"); - return ( -
-
{t(`detail.duplicateCustomer.fields.${field}`)}
-
{value || "—"}
-
- ); - })} -
-
- - {!confirming && ( - - )} - {confirming && ( -
-

{t("detail.duplicateCustomer.confirmMergeBody", { loser: loser.public_ref, survivor: survivor.public_ref })}

- - -
- )} -
- ); -} - function MissingFieldPanel({ issue, onResolved }: { issue: IssueDetail; onResolved: () => void }) { const { t } = useTranslation("quality"); const isCustomer = issue.entity_type === "customer"; diff --git a/frontend/src/pages/data-quality/DuplicateCustomerPanel.tsx b/frontend/src/pages/data-quality/DuplicateCustomerPanel.tsx new file mode 100644 index 0000000..130f361 --- /dev/null +++ b/frontend/src/pages/data-quality/DuplicateCustomerPanel.tsx @@ -0,0 +1,215 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { api } from "../../api/client"; +import { describeApiError, type ApiErrorInfo } from "../../api/errorMessages"; +import type { DataQualityIssueDetail as IssueDetail, EntitySnapshot } from "../../api/types"; +import { ApiErrorNotice, SectionHeading } from "../../components/PageChrome"; +import { useAuth } from "../../context/AuthContext"; + +const MERGE_FIELDS = ["first_name", "last_name", "email", "phone", "postal_code", "city"]; + +export function DuplicateCustomerPanel({ + issue, + onResolved, +}: { + issue: IssueDetail; + onResolved: () => void; +}) { + const { t } = useTranslation("quality"); + const { user } = useAuth(); + const [survivorRef, setSurvivorRef] = useState(issue.entity_snapshot?.public_ref ?? ""); + const [fieldChoices, setFieldChoices] = useState>({}); + const [error, setError] = useState(null); + const [submitting, setSubmitting] = useState(false); + const [confirming, setConfirming] = useState(false); + const [showMatching, setShowMatching] = useState(false); + + if (!issue.entity_snapshot || !issue.related_snapshots[0]) { + return

{t("detail.duplicateCustomer.bothMissing")}

; + } + const a: EntitySnapshot = issue.entity_snapshot; + const b: EntitySnapshot = issue.related_snapshots[0]; + const survivor = survivorRef === a.public_ref ? a : b; + const loser = survivorRef === a.public_ref ? b : a; + const conflictingFields = MERGE_FIELDS.filter( + (field) => String(a[field] ?? "") !== String(b[field] ?? ""), + ); + const matchingFields = MERGE_FIELDS.filter((field) => !conflictingFields.includes(field)); + const visibleFields = showMatching ? MERGE_FIELDS : conflictingFields; + + async function handleMerge() { + setError(null); + setSubmitting(true); + try { + const overrides: Record = {}; + for (const field of MERGE_FIELDS) { + const choice = fieldChoices[field]; + const chosenSide = choice === "a" ? a : choice === "b" ? b : survivor; + if (chosenSide !== survivor && chosenSide[field]) { + overrides[field] = String(chosenSide[field]); + } + } + await api.post(`/api/v1/data-quality/issues/${issue.public_ref}/merge-customers`, { + survivor_ref: survivor.public_ref, + field_overrides: Object.keys(overrides).length > 0 ? overrides : undefined, + }); + onResolved(); + } catch (err) { + setError(describeApiError(t, err, "detail.duplicateCustomer.mergeFailed")); + setConfirming(false); + } finally { + setSubmitting(false); + } + } + + if (user?.role !== "operations_manager") { + return

{t("detail.managerOnlyDetail")}

; + } + + return ( +
+ + +

+ {t("detail.duplicateCustomer.summaryCounts", { + matchCount: matchingFields.length, + conflictCount: conflictingFields.length, + })} +

+
+ {t("detail.duplicateCustomer.keepAsSurvivor")} + {[a, b].map((candidate) => ( + + ))} +
+ + + + + + + + + + {visibleFields.map((field) => { + const valueA = a[field] ? String(a[field]) : "—"; + const valueB = b[field] ? String(b[field]) : "—"; + const differ = valueA !== valueB; + return ( + + + {([a, b] as const).map((candidate, index) => { + const value = index === 0 ? valueA : valueB; + const choice = index === 0 ? "a" : "b"; + return ( + + ); + })} + + ); + })} + +
{t("detail.duplicateCustomer.fieldColumn")}{a.public_ref}{b.public_ref}
+ {t(`detail.duplicateCustomer.fields.${field}`)} + {differ ? ( + {t("detail.duplicateCustomer.differs")} + ) : ( + {t("detail.duplicateCustomer.match")} + )} + + {differ ? ( + + ) : ( + value + )} +
+ {matchingFields.length > 0 && ( + + )} +

+ {t("detail.duplicateCustomer.mergePreview", { + loser: loser.public_ref, + survivor: survivor.public_ref, + })} +

+
+

+ {t("detail.duplicateCustomer.previewTitle", { ref: survivor.public_ref })} +

+
+ {MERGE_FIELDS.map((field) => { + const choice = fieldChoices[field]; + const chosenSide = choice === "a" ? a : choice === "b" ? b : survivor; + const value = chosenSide[field] || survivor[field] || "—"; + return ( +
+
{t(`detail.duplicateCustomer.fields.${field}`)}
+
{String(value)}
+
+ ); + })} +
+
+ {!confirming ? ( + + ) : ( +
+

+ {t("detail.duplicateCustomer.confirmMergeBody", { + loser: loser.public_ref, + survivor: survivor.public_ref, + })} +

+ + +
+ )} +
+ ); +} + diff --git a/frontend/src/styles-data-quality.css b/frontend/src/styles-data-quality.css new file mode 100644 index 0000000..14870da --- /dev/null +++ b/frontend/src/styles-data-quality.css @@ -0,0 +1,49 @@ +.rule-explainer { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 14px; margin-bottom: 18px; padding: 16px 18px; background: var(--info-pale); border: 1px solid #cfe3ee; border-radius: var(--radius); } +.rule-explainer strong { display: block; margin-bottom: 4px; color: var(--ink); font-size: .72rem; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; } +.rule-explainer p { margin: 0; color: var(--ink-soft); font-size: .78rem; line-height: 1.55; } +.scenario-callout { display: flex; align-items: flex-start; gap: 12px; margin-bottom: 18px; padding: 16px 18px; background: var(--teal-pale); border: 1px solid #bfe6df; } +.scenario-callout svg { width: 18px; height: 18px; color: var(--teal-dark); flex-shrink: 0; margin-top: 2px; } +.scenario-callout strong { display: block; margin-bottom: 4px; color: var(--ink); font-size: .82rem; } +.scenario-callout p { margin: 0; color: var(--ink-soft); font-size: .78rem; line-height: 1.55; } + +.status-decision { margin-top: 16px; padding: 16px 18px; background: var(--surface-subtle); border: 1px solid var(--line); border-radius: var(--radius); } +.status-decision h3 { margin: 16px 0 6px; color: var(--ink); font-size: .72rem; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; } +.status-decision h3:first-child { margin-top: 0; } +.status-decision > .detail-grid { margin-bottom: 4px; } +.evidence-list { margin: 0; padding-left: 18px; color: var(--ink-soft); font-size: .78rem; line-height: 1.6; } +.evidence-list li { margin-bottom: 2px; } +.status-decision .button { margin-top: 16px; } + +.duplicate-compare { margin-top: 18px; }.duplicate-compare fieldset { margin: 15px 0; padding: 13px; border: 1px solid var(--line); }.duplicate-compare legend { padding: 0 5px; color: var(--ink-soft); font-size: var(--type-label); font-weight: 700; }.duplicate-compare fieldset label { display: inline-flex !important; margin-right: 18px; }.duplicate-compare .merge-record-preview { position: sticky; bottom: 16px; z-index: 2; box-shadow: 0 10px 24px rgba(15,23,42,.08); } +.compare-table th, .compare-table td { vertical-align: top; }.compare-table label { display: inline-flex; flex-direction: row; align-items: center; gap: 6px; }.difference-mark, .match-mark { display: block; width: max-content; margin-top: 4px; padding: 2px 5px; font-size: .52rem; border-radius: 2px; }.difference-mark { color: var(--warning); background: var(--warning-pale); }.match-mark { color: var(--success); background: var(--success-pale); } +.merge-preview { padding: 13px; color: var(--ink-soft); background: var(--surface-subtle); border-left: 3px solid var(--teal); font-size: .75rem; }.duplicate-compare > button { color: white; background: var(--teal-dark); border-color: var(--teal-dark); } +.merge-summary-counts { margin: 0 0 12px; color: var(--muted); font-size: .72rem; font-weight: 700; } +.toggle-matching-fields { display: inline-block; margin: 10px 0; padding: 6px 10px; color: var(--ink-soft); background: transparent; border: 1px solid var(--line-strong); border-radius: var(--radius); font-size: .68rem; font-weight: 700; cursor: pointer; } +.toggle-matching-fields:hover { background: var(--surface-subtle); } +.merge-record-preview { margin: 14px 0; padding: 13px; background: var(--surface-subtle); border: 1px solid var(--line); border-radius: var(--radius); } +.merge-record-preview-title { margin: 0 0 8px; color: var(--ink); font-size: .72rem; font-weight: 700; } +.merge-record-preview dl { margin: 0; display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 6px 16px; } +.merge-record-preview dt { margin: 0; color: var(--muted); font-size: .62rem; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; } +.merge-record-preview dd { margin: 1px 0 0; color: var(--ink); font-size: .78rem; } +.confirm-bar { margin-top: 13px; padding: 15px; background: var(--warning-pale); border: 1px solid #eed4aa; }.confirm-bar p { margin: 0 0 11px; color: #75420c; font-size: .75rem; font-weight: 700; }.confirm-bar button:first-of-type { color: white; background: var(--critical); border-color: var(--critical); } +.resolution-actions { display: flex; gap: 9px; }.evidence-block { max-width: 100%; padding: 13px; overflow: auto; color: #324054; background: var(--surface-subtle); border: 1px solid var(--line); font-size: .68rem; } +.evidence-disclosure { margin-top: 14px; }.evidence-disclosure summary { font-size: .68rem; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; }.evidence-disclosure .evidence-block { margin-top: 8px; } + +.choice-fieldset { display: grid; gap: 9px; margin: 0 0 17px; padding: 0; border: 0; }.choice-fieldset legend { margin-bottom: 2px; padding: 0; color: var(--ink-soft); font-size: .67rem; font-weight: 700; }.choice-fieldset-grid { grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); } +.choice-card { display: flex !important; flex-direction: row; align-items: flex-start; gap: 11px; padding: 13px 15px; background: var(--surface); border: 1.5px solid var(--line-strong); border-radius: var(--radius); cursor: pointer; transition: border-color .15s ease, background .15s ease; } +.choice-card:hover { border-color: var(--teal); } +.choice-card:has(input:checked), .choice-card.is-selected { border-color: var(--teal-dark); background: var(--teal-pale); } +.choice-card:has(input:disabled), .choice-card.is-disabled { cursor: not-allowed; opacity: .55; } +.choice-card input[type="radio"] { flex: 0 0 auto; margin-top: 2px; } +.choice-card-body { display: grid; gap: 3px; } +.choice-card-title { color: var(--ink); font-size: .82rem; font-weight: 700; } +.choice-card-detail { color: var(--muted); font-size: .72rem; line-height: 1.5; } + +.defer-reject-panel { padding: 16px 18px; background: var(--surface-subtle); } +.defer-reject-panel h2 { margin: 0 0 4px; color: var(--ink-soft); font-size: .78rem; font-weight: 700; } +.defer-reject-panel > p { margin: 0 0 12px; color: var(--muted); font-size: .72rem; } + +.resolution-actions .button-tertiary, .resolution-actions .button-tertiary-destructive { min-height: 36px; padding: 7px 12px; color: var(--muted); background: transparent; border: 1px solid transparent; border-radius: var(--radius); font-size: .7rem; font-weight: 700; cursor: pointer; } +.resolution-actions .button-tertiary:hover { color: var(--ink); background: var(--surface); border-color: var(--line); } +.resolution-actions .button-tertiary-destructive:hover { color: var(--critical); background: var(--surface); border-color: var(--line); } + diff --git a/frontend/src/styles.css b/frontend/src/styles.css index 344f31c..7b10711 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -1,3 +1,5 @@ +@import "./styles-data-quality.css"; + :root { font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; color: #0f172a; @@ -329,51 +331,6 @@ details summary { cursor: pointer; color: var(--teal-dark); }.data-table details .result-links { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 16px; } .link-button { padding: 0; color: var(--teal-dark); background: transparent; border: 0; font-size: inherit; font-weight: 700; text-decoration: underline; cursor: pointer; } .link-button:hover { color: var(--teal); } -.rule-explainer { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 14px; margin-bottom: 18px; padding: 16px 18px; background: var(--info-pale); border: 1px solid #cfe3ee; border-radius: var(--radius); } -.rule-explainer strong { display: block; margin-bottom: 4px; color: var(--ink); font-size: .72rem; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; } -.rule-explainer p { margin: 0; color: var(--ink-soft); font-size: .78rem; line-height: 1.55; } -.scenario-callout { display: flex; align-items: flex-start; gap: 12px; margin-bottom: 18px; padding: 16px 18px; background: var(--teal-pale); border: 1px solid #bfe6df; } -.scenario-callout svg { width: 18px; height: 18px; color: var(--teal-dark); flex-shrink: 0; margin-top: 2px; } -.scenario-callout strong { display: block; margin-bottom: 4px; color: var(--ink); font-size: .82rem; } -.scenario-callout p { margin: 0; color: var(--ink-soft); font-size: .78rem; line-height: 1.55; } - -.status-decision { margin-top: 16px; padding: 16px 18px; background: var(--surface-subtle); border: 1px solid var(--line); border-radius: var(--radius); } -.status-decision h3 { margin: 16px 0 6px; color: var(--ink); font-size: .72rem; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; } -.status-decision h3:first-child { margin-top: 0; } -.status-decision > .detail-grid { margin-bottom: 4px; } -.evidence-list { margin: 0; padding-left: 18px; color: var(--ink-soft); font-size: .78rem; line-height: 1.6; } -.evidence-list li { margin-bottom: 2px; } -.status-decision .button { margin-top: 16px; } - -.duplicate-compare { margin-top: 18px; }.duplicate-compare fieldset { margin: 15px 0; padding: 13px; border: 1px solid var(--line); }.duplicate-compare legend { padding: 0 5px; color: var(--ink-soft); font-size: var(--type-label); font-weight: 700; }.duplicate-compare fieldset label { display: inline-flex !important; margin-right: 18px; }.duplicate-compare .merge-record-preview { position: sticky; bottom: 16px; z-index: 2; box-shadow: 0 10px 24px rgba(15,23,42,.08); } -.compare-table th, .compare-table td { vertical-align: top; }.compare-table label { display: inline-flex; flex-direction: row; align-items: center; gap: 6px; }.difference-mark, .match-mark { display: block; width: max-content; margin-top: 4px; padding: 2px 5px; font-size: .52rem; border-radius: 2px; }.difference-mark { color: var(--warning); background: var(--warning-pale); }.match-mark { color: var(--success); background: var(--success-pale); } -.merge-preview { padding: 13px; color: var(--ink-soft); background: var(--surface-subtle); border-left: 3px solid var(--teal); font-size: .75rem; }.duplicate-compare > button { color: white; background: var(--teal-dark); border-color: var(--teal-dark); } -.merge-summary-counts { margin: 0 0 12px; color: var(--muted); font-size: .72rem; font-weight: 700; } -.toggle-matching-fields { display: inline-block; margin: 10px 0; padding: 6px 10px; color: var(--ink-soft); background: transparent; border: 1px solid var(--line-strong); border-radius: var(--radius); font-size: .68rem; font-weight: 700; cursor: pointer; } -.toggle-matching-fields:hover { background: var(--surface-subtle); } -.merge-record-preview { margin: 14px 0; padding: 13px; background: var(--surface-subtle); border: 1px solid var(--line); border-radius: var(--radius); } -.merge-record-preview-title { margin: 0 0 8px; color: var(--ink); font-size: .72rem; font-weight: 700; } -.merge-record-preview dl { margin: 0; display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 6px 16px; } -.merge-record-preview dt { margin: 0; color: var(--muted); font-size: .62rem; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; } -.merge-record-preview dd { margin: 1px 0 0; color: var(--ink); font-size: .78rem; } -.confirm-bar { margin-top: 13px; padding: 15px; background: var(--warning-pale); border: 1px solid #eed4aa; }.confirm-bar p { margin: 0 0 11px; color: #75420c; font-size: .75rem; font-weight: 700; }.confirm-bar button:first-of-type { color: white; background: var(--critical); border-color: var(--critical); } -.resolution-actions { display: flex; gap: 9px; }.evidence-block { max-width: 100%; padding: 13px; overflow: auto; color: #324054; background: var(--surface-subtle); border: 1px solid var(--line); font-size: .68rem; } -.evidence-disclosure { margin-top: 14px; }.evidence-disclosure summary { font-size: .68rem; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; }.evidence-disclosure .evidence-block { margin-top: 8px; } - -.choice-fieldset { display: grid; gap: 9px; margin: 0 0 17px; padding: 0; border: 0; }.choice-fieldset legend { margin-bottom: 2px; padding: 0; color: var(--ink-soft); font-size: .67rem; font-weight: 700; }.choice-fieldset-grid { grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); } -.choice-card { display: flex !important; flex-direction: row; align-items: flex-start; gap: 11px; padding: 13px 15px; background: var(--surface); border: 1.5px solid var(--line-strong); border-radius: var(--radius); cursor: pointer; transition: border-color .15s ease, background .15s ease; } -.choice-card:hover { border-color: var(--teal); } -.choice-card:has(input:checked), .choice-card.is-selected { border-color: var(--teal-dark); background: var(--teal-pale); } -.choice-card:has(input:disabled), .choice-card.is-disabled { cursor: not-allowed; opacity: .55; } -.choice-card input[type="radio"] { flex: 0 0 auto; margin-top: 2px; } -.choice-card-body { display: grid; gap: 3px; } -.choice-card-title { color: var(--ink); font-size: .82rem; font-weight: 700; } -.choice-card-detail { color: var(--muted); font-size: .72rem; line-height: 1.5; } - -.defer-reject-panel { padding: 16px 18px; background: var(--surface-subtle); } -.defer-reject-panel h2 { margin: 0 0 4px; color: var(--ink-soft); font-size: .78rem; font-weight: 700; } -.defer-reject-panel > p { margin: 0 0 12px; color: var(--muted); font-size: .72rem; } - .audit-group-list { list-style: none; display: grid; gap: 12px; margin: 0; padding: 16px; } .audit-group { padding: 16px 18px; } .audit-group-heading { display: flex; flex-wrap: wrap; align-items: baseline; gap: 8px; } @@ -385,10 +342,6 @@ details summary { cursor: pointer; color: var(--teal-dark); }.data-table details .audit-related-list { list-style: none; display: grid; gap: 10px; margin: 12px 0 0; padding: 12px; background: var(--surface-subtle); border: 1px solid var(--line); border-radius: var(--radius); } .audit-related-list > li { padding: 10px 12px; background: white; border: 1px solid var(--line); border-radius: var(--radius); } .audit-technical-grid { margin-top: 8px; } -.resolution-actions .button-tertiary, .resolution-actions .button-tertiary-destructive { min-height: 36px; padding: 7px 12px; color: var(--muted); background: transparent; border: 1px solid transparent; border-radius: var(--radius); font-size: .7rem; font-weight: 700; cursor: pointer; } -.resolution-actions .button-tertiary:hover { color: var(--ink); background: var(--surface); border-color: var(--line); } -.resolution-actions .button-tertiary-destructive:hover { color: var(--critical); background: var(--surface); border-color: var(--line); } - .integration-cards { display: grid; grid-template-columns: repeat(3, 1fr); gap: 12px; margin-bottom: 28px; }.integration-cards article { min-height: 0; display: grid; grid-template-columns: auto 1fr; gap: 12px; padding: 18px; background: white; border: 1px solid var(--line); border-radius: var(--radius); }.integration-cards .badge { grid-column: 1 / -1; width: max-content; align-self: end; } .integration-cards .integration-badge-stack { grid-column: 1 / -1; display: flex; flex-wrap: wrap; gap: 6px; align-self: end; } .integration-cards small { display: block; margin-top: 4px; color: var(--muted-light); font-size: var(--type-meta); }.integration-cards h2 { margin: 3px 0 7px; font-size: 1rem; }.integration-cards p { margin: 0; color: var(--muted); font-size: var(--type-body); line-height: 1.48; }.integration-kicker { color: var(--muted); font-size: var(--type-label); font-weight: 700; text-transform: uppercase; letter-spacing: .09em; } diff --git a/scripts/check-source-budgets.py b/scripts/check-source-budgets.py index aebca2e..84c38b2 100755 --- a/scripts/check-source-budgets.py +++ b/scripts/check-source-budgets.py @@ -5,10 +5,15 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] LINE_LIMITS = { - "backend/app/services/data_quality.py": 950, - "frontend/src/pages/DataQualityIssueDetail.tsx": 850, + "backend/app/services/data_quality.py": 900, + "backend/app/services/data_quality_duplicate_scan.py": 150, + "frontend/src/pages/DataQualityIssueDetail.tsx": 700, + "frontend/src/pages/data-quality/DuplicateCustomerPanel.tsx": 250, +} +BYTE_LIMITS = { + "frontend/src/styles.css": 78_000, + "frontend/src/styles-data-quality.css": 8_000, } -BYTE_LIMITS = {"frontend/src/styles.css": 84_000} failures: list[str] = [] for relative, limit in LINE_LIMITS.items(): diff --git a/scripts/generate-release-provenance.py b/scripts/generate-release-provenance.py new file mode 100644 index 0000000..bfda9fd --- /dev/null +++ b/scripts/generate-release-provenance.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Generate inspectable release provenance for the images built by CI.""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +from pathlib import Path + +IMAGES = { + "api": "mobilityops-api-release", + "web": "mobilityops-web-release", + "backup_tools": "mobilityops-backup-tools-release", +} + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> None: + revision = os.environ.get("GITHUB_SHA", "") + if len(revision) != 40: + raise SystemExit("GITHUB_SHA must contain the full release revision") + images: dict[str, dict[str, str]] = {} + for name, image in IMAGES.items(): + details = json.loads( + subprocess.check_output(["docker", "image", "inspect", image], text=True) + )[0] + labels = details.get("Config", {}).get("Labels", {}) or {} + if labels.get("org.opencontainers.image.revision") != revision: + raise SystemExit(f"revision label mismatch for {image}") + sbom = Path(f"mobilityops-{name.replace('_', '-')}-sbom.cdx.json") + images[name] = { + "reference": image, + "local_image_id": details["Id"], + "revision": labels["org.opencontainers.image.revision"], + "sbom": sbom.name, + "sbom_sha256": sha256(sbom), + } + provenance = { + "schema": "mobilityops.release-provenance.v1", + "revision": revision, + "repository": os.environ.get("GITHUB_REPOSITORY", "MobilityOps"), + "ref": os.environ.get("GITHUB_REF", ""), + "workflow_run": os.environ.get("GITHUB_RUN_ID", ""), + "images": images, + } + Path("release-provenance.json").write_text( + json.dumps(provenance, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/run-readonly-load-smoke.py b/scripts/run-readonly-load-smoke.py new file mode 100644 index 0000000..4ac74eb --- /dev/null +++ b/scripts/run-readonly-load-smoke.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Small dependency-free concurrency gate for the persisted read paths.""" + +from __future__ import annotations + +import argparse +import json +import math +import time +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed + +PATHS = ( + "/api/v1/dashboard", + "/api/v1/vehicles?page=1&page_size=25", + "/api/v1/bookings?page=1&page_size=25&sort=operational", + "/api/v1/data-quality/issues?status=open&page=1&page_size=25", + "/api/v1/audit?page=1&page_size=25", + "/api/v1/system/status", +) + + +def request(url: str, cookie: str, timeout: float) -> tuple[float, int]: + started = time.perf_counter() + try: + with urllib.request.urlopen( + urllib.request.Request(url, headers={"Cookie": cookie}), timeout=timeout + ) as response: + response.read() + status = response.status + except urllib.error.HTTPError as error: + status = error.code + return (time.perf_counter() - started) * 1000, status + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:1228") + parser.add_argument("--requests", type=int, default=240) + parser.add_argument("--concurrency", type=int, default=12) + parser.add_argument("--timeout", type=float, default=5.0) + parser.add_argument("--max-p95-ms", type=float, default=1500.0) + args = parser.parse_args() + if args.requests < 1 or args.concurrency < 1: + raise SystemExit("requests and concurrency must be positive") + + login = urllib.request.Request( + f"{args.base_url.rstrip('/')}/api/v1/demo/login", + data=json.dumps({"role": "operations_manager"}).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(login, timeout=args.timeout) as response: + cookie = response.headers.get("Set-Cookie", "").split(";", 1)[0] + if not cookie: + raise SystemExit("demo login returned no session cookie") + + base = args.base_url.rstrip("/") + work = [f"{base}{PATHS[index % len(PATHS)]}" for index in range(args.requests)] + results: list[tuple[float, int]] = [] + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + futures = [executor.submit(request, url, cookie, args.timeout) for url in work] + results.extend(future.result() for future in as_completed(futures)) + + failures = [status for _, status in results if status != 200] + durations = sorted(duration for duration, _ in results) + p95 = durations[max(0, math.ceil(len(durations) * 0.95) - 1)] + print( + f"read-only load smoke: requests={len(results)} concurrency={args.concurrency} " + f"failures={len(failures)} p95_ms={p95:.1f} max_ms={durations[-1]:.1f}" + ) + if failures: + raise SystemExit(f"read-only load smoke returned non-200 statuses: {sorted(set(failures))}") + if p95 > args.max_p95_ms: + raise SystemExit(f"p95 {p95:.1f} ms exceeds {args.max_p95_ms:.1f} ms") + + +if __name__ == "__main__": + main() +