diff --git a/.env.example b/.env.example index 83b5b8e..ecab19c 100644 --- a/.env.example +++ b/.env.example @@ -34,13 +34,20 @@ LOG_LEVEL=INFO METRICS_BEARER_TOKEN= GRAFANA_ADMIN_USER=admin GRAFANA_ADMIN_PASSWORD=change-me-before-start +# Alertmanager sends every firing/resolved alert and the continuous watchdog to this +# owner-managed receiver. Production must route it to a channel that is actually watched. +ALERTMANAGER_WEBHOOK_URL=https://n8n.itworx.tech/webhook/mobilityops-alerts # Verified scheduled PostgreSQL backups (Unraid override). BACKUP_INTERVAL_SECONDS=86400 BACKUP_RETENTION_DAYS=30 BACKUP_MINIMUM_COPIES=7 +# Restore the newest dump into a disposable database at least weekly. Backup health also +# requires a successful drill within eight days. +BACKUP_RESTORE_DRILL_INTERVAL_SECONDS=604800 # Set both values to copy every verified backup to an independently mounted path. BACKUP_SECONDARY_DESTINATION= +MOBILITYOPS_BACKUP_DIR=./backups/postgres MOBILITYOPS_BACKUP_SECONDARY_DIR=./backups/offsite # Privacy governance defaults. @@ -87,6 +94,9 @@ RAGCORE_COLLECTION=internal-procedures RAGCORE_API_TOKEN= # UUID of the RAGcore knowledge space procedures were synced into (see workflow 3). RAGCORE_SPACE_ID= +# Skip the slower generation endpoint temporarily after a timeout/non-2xx response and +# use the still-grounded extractive search fallback immediately. +RAGCORE_ANSWERS_CIRCUIT_BREAKER_SECONDS=60 # ITWorx MCP Hub integration. Registration itself is catalog-driven on the Hub's own # side (it reconciles its catalog into the gateway; Fleet Ops never pushes a diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 2b7ff7a..f8dd7fe 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -4,6 +4,8 @@ on: push: branches: [master] pull_request: + schedule: + - cron: "17 3 * * 1" jobs: backend: @@ -21,8 +23,15 @@ jobs: run: | docker compose -p mobilityops-ci -f compose.yaml -f compose.test.yaml run --build --rm api ruff check app tests docker compose -p mobilityops-ci -f compose.yaml -f compose.test.yaml run --rm api mypy app + - name: Contract drift gate + run: | + docker compose -p mobilityops-ci -f compose.yaml -f compose.test.yaml run --rm \ + -v "$PWD:/repo:ro" api python /repo/scripts/check-contracts.py + python scripts/check-source-budgets.py - name: Build production API image for vulnerability scan - run: docker build --target runtime --tag mobilityops-api-ci --file backend/Dockerfile . + run: | + docker build --target runtime --build-arg VCS_REF="$GITHUB_SHA" \ + --tag mobilityops-api-ci --file backend/Dockerfile . - name: Production API image vulnerability scan (HIGH/CRITICAL) uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # v0.30.0 with: @@ -32,6 +41,19 @@ jobs: severity: HIGH,CRITICAL exit-code: "1" ignore-unfixed: true + - name: Build production web image for vulnerability scan + run: | + docker build --build-arg VCS_REF="$GITHUB_SHA" \ + --tag mobilityops-web-ci frontend + - name: Production web image vulnerability scan (HIGH/CRITICAL) + uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # v0.30.0 + with: + scan-type: image + image-ref: mobilityops-web-ci + format: table + severity: HIGH,CRITICAL + exit-code: "1" + ignore-unfixed: true - name: Remove CI stack if: always() run: docker compose -p mobilityops-ci -f compose.yaml -f compose.test.yaml down -v --remove-orphans @@ -53,7 +75,7 @@ jobs: run: npm run lint - name: Typecheck and production build working-directory: frontend - run: npm run build + run: npm run build && npm run budget - name: Dependency audit working-directory: frontend run: npm audit --audit-level=high @@ -74,7 +96,7 @@ jobs: run: | cp .env.example .env docker compose -p mobilityops-e2e up --build -d db api web - for attempt in $(seq 1 60); do + for _attempt in $(seq 1 60); do if curl -fsS http://localhost:1228/health/ready >/dev/null 2>&1; then break; fi sleep 2 done @@ -84,18 +106,25 @@ jobs: working-directory: frontend run: | npm ci --no-audit --no-fund - npx playwright install --with-deps chromium + npx playwright install --with-deps chromium firefox - name: Run browser acceptance suite working-directory: frontend env: MOBILITYOPS_PUBLIC_URL: http://localhost:1228 run: npx playwright test + - name: Run non-destructive Chromium and Firefox smoke suite + working-directory: frontend + env: + MOBILITYOPS_PUBLIC_URL: http://localhost:1228 + run: npx playwright test --config=playwright.live.config.ts - name: Upload Playwright report if: failure() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: playwright-report - path: frontend/playwright-report + path: | + frontend/playwright-report + frontend/playwright-live-report - name: Stack logs on failure if: failure() run: docker compose -p mobilityops-e2e logs --tail=200 api web diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml new file mode 100644 index 0000000..532ce7d --- /dev/null +++ b/.gitea/workflows/release.yml @@ -0,0 +1,42 @@ +name: MobilityOps release evidence + +on: + push: + tags: ["v*"] + +jobs: + release-evidence: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - name: Build commit-labelled release images + run: | + docker build --target runtime --build-arg VCS_REF="$GITHUB_SHA" --tag mobilityops-api-release --file backend/Dockerfile . + docker build --build-arg VCS_REF="$GITHUB_SHA" --tag mobilityops-web-release frontend + - name: Generate API CycloneDX SBOM + uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # v0.30.0 + with: + scan-type: image + image-ref: mobilityops-api-release + format: cyclonedx + output: mobilityops-api-sbom.cdx.json + - name: Generate web CycloneDX SBOM + uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # v0.30.0 + with: + scan-type: image + image-ref: mobilityops-web-release + format: cyclonedx + output: mobilityops-web-sbom.cdx.json + - name: Record immutable image metadata + run: | + docker image inspect mobilityops-api-release > mobilityops-api-image.json + docker image inspect mobilityops-web-release > mobilityops-web-image.json + sha256sum mobilityops-*-sbom.cdx.json mobilityops-*-image.json > SHA256SUMS + - name: Upload release evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: mobilityops-${{ github.ref_name }}-evidence + path: | + mobilityops-*-sbom.cdx.json + mobilityops-*-image.json + SHA256SUMS diff --git a/.gitignore b/.gitignore index ccc8026..f9e0936 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ node_modules/ dist/ coverage/ playwright-report/ +playwright-live-report/ test-results/ *.pyc .DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..bce5b2f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +All notable changes are documented here. The project follows semantic release tags for +the deployable PoC; detailed validation evidence remains in `PROJECT_STATE.md`. + +## [Unreleased] + +- Added contract-drift, accessibility, Firefox smoke and frontend asset-budget gates. +- Added immutable commit-labelled deployment with automatic application rollback. +- Added real PostgreSQL restore drills and routed Alertmanager notifications. +- Added RAGcore generation circuit breaking and retrieval telemetry. +- Added scheduled dependency maintenance, dual-image vulnerability scans and release SBOMs. + +## [1.0.0-poc] - 2026-08-21 + +- Completed the locked Fleet Ops proof of concept: operational core, transactional returns, + data quality, n8n orchestration, grounded RAGcore knowledge, read-only MCP integration, + privacy governance, observability, backup/recovery and full browser acceptance. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..cb0c466 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Jens Caers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile index 7a497ba..f46897e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: up down logs test lint seed reset n8n-setup n8n-setup-scan demo e2e +.PHONY: up down logs test lint contracts seed reset n8n-setup n8n-setup-scan demo e2e live-smoke up: docker compose up --build -d @@ -17,6 +17,11 @@ lint: docker compose -f compose.yaml -f compose.test.yaml run --rm api mypy app cd frontend && npm run lint +contracts: + docker compose -f compose.yaml -f compose.test.yaml run --build --rm \ + -v "$(CURDIR):/repo:ro" api python /repo/scripts/check-contracts.py + python scripts/check-source-budgets.py + seed: docker compose exec api python -m app.cli seed --reset @@ -49,3 +54,6 @@ demo: up e2e: cd frontend && npx playwright test + +live-smoke: + cd frontend && npx playwright test --config=playwright.live.config.ts diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 4f2968d..40188c8 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -1,5 +1,32 @@ # Project state +## M44 — release integrity and assurance hardening (2026-08-21) + +- Replaced mutable archive overlays with checksum-verified, commit-named release staging, + OCI revision-labelled API/web images, health-gated promotion and automatic application + rollback. Routine deployment no longer resets persisted demo data. +- Scheduled backups now execute a weekly real restore into a disposable database and gate + health on both backup and restore-drill freshness. The isolated drill passed at Alembic + `4f2b9c8d7e61` with restored counts `2 users / 50 vehicles / 254 bookings / 1 audit event`. +- Added Alertmanager routing with a continuous watchdog and an authenticated fifth n8n + workflow targeting the existing watched M365 owner mailbox. Added weekly Renovate, + API/web image vulnerability scans, tag-triggered CycloneDX SBOM evidence and immutable + image metadata. +- Contract drift is now executable for OpenAPI, events, MCP endpoints and all five n8n + workflows. RAGcore generation failures open a bounded circuit breaker so grounded search + fallback avoids repeated five-second delays; provider-stage outcomes and retrieval scores + are measurable. +- Added Axe accessibility, platform-independent visual regression, non-destructive + Chromium/Firefox canaries, frontend asset budgets and source-growth budgets. The checks + found and fixed two real WCAG contrast defects. +- Validation: backend **271/271**, Playwright **155/155** in 4.6 minutes, live-safe canary + **4/4**, focused knowledge/observability **42/42**, frontend lint/build/audit/budgets, + contract gate, Compose rendering, Prometheus/Alertmanager validation and shell parsing + passed. The user approved the repository security policy before it was written. +- Exact next action: commit and push M44, create a production backup, publish/test the alert + receiver, deploy the exact archive, run the non-destructive production canary and record + M45 live evidence. + ## M43 — publish and redeploy review remediation (2026-08-21) - Published M41 hardening commit `24dcb3494c522fadf536fa9d6826450227aeff4e` diff --git a/README.md b/README.md index cd4cf7a..6fecce5 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ make e2e # complete Playwright browser acceptance cd frontend && npm run build ``` -Current verified results and production evidence are recorded in [artifacts/final-acceptance/summary.md](artifacts/final-acceptance/summary.md) (historical M7 evidence: [artifacts/evidence/final-summary.md](artifacts/evidence/final-summary.md)) and [PROJECT_STATE.md](PROJECT_STATE.md). +Release-scoped results and production evidence are recorded in [artifacts/final-acceptance/summary.md](artifacts/final-acceptance/summary.md); older milestone evidence remains explicitly historical. [PROJECT_STATE.md](PROJECT_STATE.md) records the commands and exact deployment revision. ## Repository map @@ -89,6 +89,6 @@ Current verified results and production evidence are recorded in [artifacts/fina - `n8n/` — importable workflow definitions for the existing server - `seed/` — deterministic synthetic dataset - `docs/` — architecture, security, UX, testing and runbooks -- `artifacts/` — acceptance evidence and screenshots per release (`artifacts/final-acceptance/summary.md` is the definitive one) +- `artifacts/` — dated, release-scoped acceptance evidence and screenshots “MobilityOps” remains the repository/deployment identifier; **Fleet Ops** is the product name shown to users. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..9beb10f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,56 @@ +# Security Policy + +## Supported versions + +| Version | Security support | +|---|---| +| Latest tagged PoC release and current `master` | Supported | +| Older commits, branches and untagged deployments | Not supported | + +MobilityOps is a synthetic-data proof of concept, not a production identity, +payments or public reservation platform. Security fixes target the current +release line only. + +## Reporting a vulnerability + +Do not disclose suspected vulnerabilities through a public issue. + +Report them privately to `jens@itworx.tech` with: + +- the affected revision, endpoint or component; +- reproduction steps and prerequisites; +- the observed and expected behaviour; +- the security impact; +- a minimal proof of concept, without unnecessary personal or secret data. + +Receipt should be acknowledged within three business days. An initial +assessment or request for additional evidence should follow within ten +business days. Remediation timing depends on severity and reproducibility. + +## Scope + +In scope: + +- MobilityOps backend, frontend, container and deployment code; +- authentication, authorization, tenant boundaries and audit integrity; +- database, outbox, backup and restore behaviour; +- MobilityOps-owned n8n workflow definitions; +- RAGcore and MCP Hub integration boundaries implemented in this repository. + +Out of scope: + +- denial-of-service or destructive testing against the hosted demo; +- social engineering, credential stuffing or physical attacks; +- synthetic demo-data exposure without a security-boundary failure; +- vulnerabilities solely inside RAGcore, ITWorx MCP Hub, n8n or another + third-party service. Report those to their respective owners. + +Do not access data beyond what is required to demonstrate the issue, modify +shared infrastructure, interrupt other services or retain obtained secrets. + +## Coordinated disclosure + +Good-faith research that respects this policy and applicable law will be +handled constructively. Allow a reasonable remediation period before public +disclosure. Submitted reports and evidence are used only for investigation, +remediation and verification. diff --git a/artifacts/final-acceptance/summary.md b/artifacts/final-acceptance/summary.md index af04152..e6c7a4c 100644 --- a/artifacts/final-acceptance/summary.md +++ b/artifacts/final-acceptance/summary.md @@ -1,265 +1,38 @@ -# MobilityOps — final acceptance audit summary +# Fleet Ops release acceptance -This audit was run after M0–M7 had already been implemented and committed, specifically -to independently re-verify the finished system end to end rather than trust the -milestone-by-milestone build log. It found and fixed one real category of defect -(`mypy` had never been run across the whole build) and confirmed everything else — every -user journey, every button/filter/form, both external-dependency degraded modes, secret -hygiene, and the clean-checkout path — works as documented. +This file is release-scoped evidence, not a timeless claim. Older evidence under +`artifacts/evidence/` is historical. Exact commands and production revisions are recorded +in `PROJECT_STATE.md`. -## Final commit +## 2026-08-21 release candidate -This audit's fixes are committed as the commit immediately following -`108b5d04fc6f7c5ff9c47009032d6469df29cf3c` ("M7: portfolio polish and final acceptance"). -Run `git log -1 --format="%H %s"` for the exact hash. +- Backend: **271/271** tests passed against an isolated clean PostgreSQL database. +- Browser acceptance: **155/155** Chromium tests passed in 4.6 minutes. +- Live-safe browser canary: **4/4** passed across Chromium and Firefox against the local + deployed stack; unlike the acceptance suite, it never resets or mutates demo records. +- Accessibility: the principal login, dashboard, data-quality, knowledge, automation and + audit routes have no automated critical/serious WCAG 2 A/AA/2.1 AA violations. +- Frontend: TypeScript, ESLint, production build, dependency audit and per-asset JS/CSS + budgets passed; committed visual baselines cover the public entry and engineering story. +- Contracts: committed OpenAPI, event schema, MCP tools and all five n8n definitions match + their code/manifest sources. +- Recovery: a custom-format PostgreSQL dump was restored into a disposable database; the + Alembic revision and non-zero canonical table counts matched the source database. +- Operations: Prometheus/Alertmanager configuration validation passed, including the + watchdog and authenticated n8n receiver route. -## Exact commands executed +## Evidence boundary -Clean-checkout drill (run twice during this audit, most recently against fully wiped -Docker volumes): +The complete local suite uses the deterministic provider and an isolated database so it is +repeatable and safely destructive. Production verification is deliberately smaller and +non-destructive; it verifies the real RAGcore/MCP/n8n health surfaces without resetting the +shared demo. A successful local result is never presented as proof that an external service +was live. The production subsection is added only after the exact committed release is +deployed and observed. -```bash -git status # working tree clean before starting -docker compose down -v # wipe all volumes — genuinely clean state -cp .env.example .env -docker compose up --build -d # migrations run automatically (backend/entrypoint.sh) -docker compose exec api python -m app.cli seed --reset -docker compose run --rm api pytest -q -docker compose run --rm api ruff check . -docker compose run --rm api mypy app -cd frontend && npm run build -cd frontend && npx playwright test -``` +## Remaining product boundary -n8n one-time setup (owner account via browser at `http://localhost:5678/setup`, then): - -```bash -docker compose exec n8n n8n import:workflow --input=//imports/mobilityops-return-processing.json -docker compose exec n8n n8n publish:workflow --id=mobilityops-return-processing -docker compose restart n8n -``` - -Degraded-mode drills: - -```bash -docker compose stop n8n # then register a return via the API — commits, event stays pending -docker compose start n8n # dispatcher self-heals, no manual intervention -docker compose run --rm -e KNOWLEDGE_PROVIDER=ragcore -e RAGCORE_BASE_URL=http://ragcore-not-reachable:9999 \ - api python -c "from app.services.knowledge import get_knowledge_provider; ..." -``` - -## Test and validation results - -| Check | Command | Result | -|---|---|---| -| Backend unit/integration tests | `docker compose run --rm api pytest -q` | **66 passed**, 0 failed, 0 skipped | -| Backend lint | `docker compose run --rm api ruff check .` | **All checks passed** | -| Backend type check | `docker compose run --rm api mypy app` | **Success: no issues found in 44 source files** (found and fixed 43 pre-existing errors this audit — see below) | -| Frontend build + typecheck | `cd frontend && npm run build` | Clean (`tsc -b && vite build`, zero errors) | -| End-to-end (Playwright) | `cd frontend && npx playwright test` | **12 passed** (`demo.spec.ts` — full 9-step demo script; `interactive-elements.spec.ts` — 11 tests covering every nav item, filter, tab, and role boundary) | -| Clean-checkout migrations | `docker compose down -v && docker compose up --build -d` | 11 tables created automatically, `alembic current` → `e7b08389f47f (head)`, zero manual step | -| Deterministic seed | `docker compose exec api python -m app.cli seed --reset` | `users:2 customers:180 vehicles:50 bookings:246 inspections:75 maintenance:40 data_quality_issues:26 workflow_runs:20` — identical across every reseed this session | -| Secret scan | `git ls-files \| grep -x .env`; `git log --all -p -- '*.env'`; history grep for AWS/private-key/`sk-` patterns | No `.env` ever committed; no secrets found in history | - -### mypy defects found and fixed (the one real gap this audit uncovered) - -`mypy` is a declared dev dependency (`backend/pyproject.toml`) but was never added to any -milestone's validation loop — only `ruff` was run throughout M0–M7. Running it cold -surfaced 43 errors across 10 files. All were triaged and fixed (not suppressed): - -- **Two genuine defensive-programming gaps**, not just type-annotation issues: - - `app/services/returns.py`: the vehicle lookup after acquiring the row lock had no - `None` guard; a dangling FK would have crashed with an unhandled 500 instead of a - clean `404 VEHICLE_NOT_FOUND`. Fixed. - - `app/api/routers/bookings.py`: same pattern in `get_booking` for the customer/vehicle - lookups — now returns a clean `500` with a message instead of an `AttributeError`. -- `app/api/deps.py` / `app/api/routers/demo.py`: `CurrentUser.role` is validated by - Pydantic at runtime already, but `get_current_user` now explicitly checks role - membership before construction, turning a would-be unhandled `ValidationError` into a - clean `401` for a corrupted/tampered session cookie. -- Two instances of reusing one variable name for both a `Vehicle` and a `Customer` across - branches (`dashboard.py`, `data_quality.py`) — renamed for clarity, not just to satisfy - mypy. -- `Booking.__table__.update()` / `Customer.__table__.update()` switched to the idiomatic - `sqlalchemy.update(Model)` construct (also fixes the type error). -- Remainder: deprecated `conint()` → `Annotated[int, Field(...)]`, a `Sequence` vs `list` - `.sort()` call, an `assert`-guarded None-narrow after a `WHERE ... IS NOT NULL` filter - mypy can't see through, and a couple of narrowly-scoped `# type: ignore[...]` comments - for known SQLAlchemy stub gaps (`Result.rowcount`). - -`make lint` now runs `ruff check .` **and** `mypy app`. - -## Application URLs and ports - -| Service | URL | Notes | -|---|---|---| -| Web (React SPA) | `http://localhost:1228` | nginx-served static build | -| API | `http://localhost:8128` | FastAPI, `/health` for liveness | -| API docs | `http://localhost:8128/docs` | auto-generated OpenAPI/Swagger UI | -| n8n | `http://localhost:5678` | requires one-time owner setup, see below | -| PostgreSQL | `localhost:5432` (container-internal only, no host port published) | | - -## Demo users and access method - -No passwords. Two demo-role buttons on `http://localhost:1228/login`: - -- **Open as Operations Manager** → `USR-OPS`, "Amelie De Ridder". Full access: dashboard, - data-quality resolution/merge, automation retry, demo reset, MCP/service-token routes - are separate (not user-facing). -- **Open as Rental Employee** → `USR-EMP`, "Karim Boujaddaine". Can register returns and - browse vehicles/bookings/knowledge; Automation page is visible but shows a - role-restricted message instead of the delivery table (enforced both in the UI and by - the backend's `require_operations_manager` dependency — verified by - `test_retry_requires_operations_manager` and the e2e role-restriction test). - -Session is an HMAC-signed, `HttpOnly` cookie (`app/core/security.py`) — a demo mechanism, -not a real identity provider (documented as a known limitation). - -## Implemented functionality - -- Operations dashboard with 100% database-backed metrics, attention items linking to the - underlying data-quality issue, "today" departures/returns, and recent automation runs. -- Vehicle and booking list/detail pages with working filters (status, attention-only) and - a tabbed vehicle detail view (overview/bookings/inspections/maintenance/quality). -- Full transactional vehicle-return workflow: row-locked, idempotent by - `Idempotency-Key`, canonical-odometer regression handling (never silently lowers the - canonical value), vehicle status derivation, two audit events, and a schema-compliant - outbox event — verified against real concurrent submissions (1×201 + 2×409). -- Invalid-mileage rejection: negative values and non-numeric input both correctly - rejected with `422` and a precise Pydantic validation message. -- Data Quality Workbench: five deterministic rules (duplicate customer via TF-IDF-style - weighted signal scoring, missing required field, odometer regression, booking overlap, - vehicle status conflict), issue list/detail/defer/reject, and a two-column - duplicate-customer compare-and-merge UI with an inline (non-native-dialog) confirmation - step, transactional booking rewiring, and audit logging. -- Full audit trail: every significant action (login, return, vehicle status change, - data-quality issue lifecycle, customer merge, workflow retry, demo reset, n8n - callback, MCP tool request, knowledge question) is recorded with actor, correlation ID, - and before/after state; filterable by action. -- Knowledge Assistant: deterministic TF-IDF-weighted extractive retrieval over the 10 - procedure documents — never generative, always cites real excerpts, and honestly - reports `insufficient`/`unavailable` states rather than fabricating an answer. -- n8n automation: background outbox dispatcher (`FOR UPDATE SKIP LOCKED` claim, - exponential backoff, no DB transaction held during the HTTP call), a live-verified - round trip through an actual n8n workflow, manual retry for failed deliveries, and an - Automation page (Operations Manager only) showing all runs with filtering. -- Four read-only, service-token-authenticated MCP Hub provider endpoints, each recording - its own service-request audit event, with zero write/mutation endpoints anywhere in - that namespace. -- Responsive UI verified down to 360px width (nav wraps, tables become cards, metric - tiles reflow to a 2-column grid, no horizontal overflow) — both by an automated - Playwright viewport/overflow assertion and by a captured screenshot. - -## RAGcore integration status: implemented, not live-verified - -The active `KnowledgeProvider` in this environment is `DemoKnowledgeProvider` — fully -implemented, fully tested, fully live-verified, and what actually satisfies the -knowledge-assistant acceptance criteria. A `RAGcoreKnowledgeProvider` HTTP adapter also -exists (`app/services/knowledge/ragcore.py`), targeting a best-effort contract inferred -from `contracts/ragcore-contract-assumptions.md` (no live RAGcore API spec was available). -Its **unavailable-degradation path is live-verified this audit**: pointed at an -unreachable host, it returns `{"evidence_state": "unavailable", "answer": "", "sources": -[]}` with no fabrication, exactly as required — but an actual successful round trip -against a real RAGcore instance has never been performed, because no such instance was -reachable in this environment. - -## MCP Hub integration status: implemented, not live-verified - -All four contracted read-only tools (`mobilityops_get_operations_summary`, -`mobilityops_list_attention_vehicles`, `mobilityops_get_vehicle_details`, -`mobilityops_search_knowledge`) are implemented as service-token-protected endpoints under -`/api/v1/integrations/mcp/`, directly `curl`-verified this audit (auth enforcement, -correct data shape, no write methods, service-request audit logging). No live ITWorx MCP -Hub instance was reachable in this environment, so an actual Hub-mediated tool call was -never performed — only direct calls to MobilityOps's own provider API. - -## n8n integration status: implemented and fully live-verified - -The only external integration with a real, running counterpart service available in this -environment. Fully verified this audit, including both success and degraded paths: - -- **Success**: a real return registered via the API was delivered by the background - dispatcher to an actual n8n instance (owner account + imported/activated workflow), - which called back into MobilityOps and was recorded `succeeded`. -- **Degraded mode**: `docker compose stop n8n`, then a return was registered — it - **committed successfully** (`201`, booking `status: returned` persisted) exactly as - required by the architecture's reliability boundary ("a return command and its outbox - event commit in one transaction" and "n8n failure leaves events pending with bounded - retries"). The outbox event stayed `pending` with two real `ConnectError`s logged and - exponential backoff. -- **Self-healing**: restarting n8n required no manual intervention — the background - dispatcher picked the pending event back up on its next poll cycle and delivered it to - `succeeded` (5 total attempts across the outage). -- **Manual retry (S5 scenario)**: a seeded `failed` delivery, retried from the Automation - page, moved to `pending` and was delivered to `succeeded` by the live dispatcher within - one poll cycle. - -## Known limitations - -- RAGcore and the ITWorx MCP Hub were never reachable in this build/audit environment; - both integrations are implemented and tested against inferred/documented contracts but - not verified against real instances of those systems (see above). -- n8n requires a one-time, per-fresh-environment manual owner-account setup through its - own web UI (`http://localhost:5678/setup`) — a property of the n8n 2.x image itself - (`N8N_BASIC_AUTH_ACTIVE` no longer gates the UI), not something MobilityOps can bypass. - The workflow import/activation itself *is* scripted (`make n8n-setup`). -- Demo authentication is an HMAC-signed session cookie tied to two fixed seeded users — - appropriate for a PoC, not a production identity provider. -- Inspection public references are assigned via a simple `count + 1` sequence, not - gap-safe under true concurrent writers (acceptable for this single-tenant demo). -- The five data-quality rules use a simplified idempotency key - (`rule_type, entity_type, entity_id` while open) rather than the spec's literal - evidence-fingerprint scheme — documented rationale in `PROJECT_STATE.md`'s M3 notes. -- `npm audit` reports one residual moderate `esbuild`/Vite-8 dev-server-only advisory - (fixable only by a Vite major version bump) and one high `react-router` RSC-mode - advisory that does not apply to this app (it never uses React Router's RSC/SSR mode). - -## Clean deployment instructions - -```bash -git clone && cd MobilityOps -cp .env.example .env -make demo # build, start, migrate (automatic), seed -``` - -One-time n8n setup (only needed for the automation demo path; everything else works -without it): - -```bash -# open http://localhost:5678/setup in a browser, create any owner account -# (8+ chars, 1 number, 1 capital letter — no email verification required) -make n8n-setup -``` - -Verify: - -```bash -curl http://localhost:8128/health # {"status":"ok",...} -curl -o /dev/null -w "%{http_code}\n" http://localhost:1228/ # 200 -make test # 66 backend tests -make lint # ruff + mypy, zero errors -make e2e # 12 Playwright tests (stack must be running) -``` - -Full detail, recovery expectations, and required operational checks: `docs/17-runbook.md`. - -## Five-minute demonstration flow - -1. Open `http://localhost:1228` → **Open as Operations Manager**. -2. **Dashboard**: point out the metrics are live counts (available/rented/cleaning/ - maintenance/blocked vehicles, open quality issues, pending/failed workflows), and the - Attention Required list linking straight to the underlying issues. -3. **Vehicles → MO-024** → open the active booking `BK-DEMO-RETURN`, register a return - with an odometer reading below MO-024's canonical value → the result panel shows the - inspection, the derived vehicle status, the automatically-created data-quality issue, - and the queued automation event — canonical odometer is confirmed unchanged. -4. **Data Quality → DQ-DEMO-DUPLICATE**: the two-column CUS-0012/CUS-0178 comparison, - merge with the inline confirmation step, issue flips to `resolved`. -5. **Knowledge**: ask "What must I do when a vehicle returns with damage?" → grounded - answer citing both the return and damage-handling procedures with real excerpts. -6. **Automation**: filter to `failed`, retry the seeded delivery, watch it succeed within - a few seconds via the live n8n instance. -7. **Audit**: filter by `return_registered` or `customer_merged` to show every action from - this walkthrough is recorded with actor, timestamp, and correlation ID. -8. Resize the browser to 360px width to show the responsive layout (nav wraps, tables - become cards) — or run `make e2e` and point at the passing responsive assertion. +Fleet Ops remains a synthetic single-tenant PoC. It is not a production identity provider, +payment system, accounting package or public reservation platform. External RAGcore, MCP +Hub and n8n services remain independently operated dependencies and are accessed only +through their documented adapters. diff --git a/backend/Dockerfile b/backend/Dockerfile index 621680f..d4ede03 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,10 @@ FROM python:3.12-slim-bookworm@sha256:a116514e19457bcb7af7efe9c3dd0b9b71e85b317694e7882a1c52aa15a78134 AS runtime-base +ARG VCS_REF=development +ARG BUILD_DATE=unknown +LABEL org.opencontainers.image.title="Fleet Ops API" \ + org.opencontainers.image.revision="$VCS_REF" \ + org.opencontainers.image.created="$BUILD_DATE" \ + org.opencontainers.image.source="https://fleetops.itworx.tech" ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 WORKDIR /app COPY backend/requirements-prod.lock ./ diff --git a/backend/alembic.ini b/backend/alembic.ini index 642a96b..5aabe3a 100644 --- a/backend/alembic.ini +++ b/backend/alembic.ini @@ -1,7 +1,7 @@ [alembic] script_location = alembic prepend_sys_path = . -version_path_separator = os +path_separator = os [loggers] keys = root,sqlalchemy,alembic diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 7135be6..c11e462 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -23,6 +23,7 @@ class Settings(BaseSettings): ragcore_api_token: str = "" ragcore_space_id: str = "" ragcore_http_timeout_seconds: float = 5.0 + ragcore_answers_circuit_breaker_seconds: float = 60.0 # Search fallback is only labelled grounded above this explicit retrieval threshold. # RAGcore's fused score is reciprocal-rank based (top ranks are ~1/61), so this # accepts only leading results while still rejecting absent and low-ranked evidence. diff --git a/backend/app/core/observability.py b/backend/app/core/observability.py index df05e44..eddafa7 100644 --- a/backend/app/core/observability.py +++ b/backend/app/core/observability.py @@ -36,6 +36,16 @@ DATABASE_READY = Gauge( "mobilityops_database_ready", "Whether the canonical PostgreSQL database answered the most recent readiness probe.", ) +KNOWLEDGE_PROVIDER_REQUESTS = Counter( + "mobilityops_knowledge_provider_requests_total", + "RAGcore adapter requests by stage and outcome.", + ("stage", "outcome"), +) +KNOWLEDGE_RETRIEVAL_SCORE = Histogram( + "mobilityops_knowledge_retrieval_score", + "Observed RAGcore fused/rerank retrieval scores.", + buckets=(0.005, 0.01, 0.015, 0.016, 0.0162, 0.0164, 0.02, 0.05, 0.1, 0.5, 1.0), +) class JsonFormatter(logging.Formatter): diff --git a/backend/app/main.py b/backend/app/main.py index ef2260d..2ae98b4 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -59,6 +59,11 @@ production = settings.mobilityops_env.lower() == "production" app = FastAPI( title=f"{PRODUCT_NAME} API", version="0.1.0", + description=( + "Generated contract for Fleet Ops. The visible product name is Fleet Ops; " + "MobilityOps remains the technical repository and service identifier." + ), + servers=[{"url": "http://localhost:8128"}], lifespan=lifespan, docs_url=None if production else "/docs", redoc_url=None if production else "/redoc", diff --git a/backend/app/services/knowledge/ragcore.py b/backend/app/services/knowledge/ragcore.py index f1193c8..23dcc97 100644 --- a/backend/app/services/knowledge/ragcore.py +++ b/backend/app/services/knowledge/ragcore.py @@ -8,6 +8,7 @@ from time import monotonic import httpx from app.core.config import get_settings +from app.core.observability import KNOWLEDGE_PROVIDER_REQUESTS, KNOWLEDGE_RETRIEVAL_SCORE from app.services.knowledge import EvidenceState, GroundedAnswer, KnowledgeHealth, SourceCard from app.services.knowledge.procedures import ProcedureDocument, iter_procedure_documents @@ -137,6 +138,22 @@ class RAGcoreKnowledgeProvider: self._settings = get_settings() self._verification_cache: dict[str, tuple[float, int]] = {} self._verification_lock = Lock() + self._answers_circuit_lock = Lock() + self._answers_circuit_open_until = 0.0 + + def _answers_circuit_is_open(self) -> bool: + with self._answers_circuit_lock: + return monotonic() < self._answers_circuit_open_until + + def _open_answers_circuit(self) -> None: + with self._answers_circuit_lock: + self._answers_circuit_open_until = monotonic() + max( + 0.0, self._settings.ragcore_answers_circuit_breaker_seconds + ) + + def _close_answers_circuit(self) -> None: + with self._answers_circuit_lock: + self._answers_circuit_open_until = 0.0 def _client(self) -> httpx.Client: headers = {} @@ -264,9 +281,12 @@ class RAGcoreKnowledgeProvider: if not self._settings.ragcore_space_id: return unavailable - answered = self._ask_via_answers(question, correlation_id) - if answered is not None: - return answered + if self._answers_circuit_is_open(): + KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", "circuit_open").inc() + else: + answered = self._ask_via_answers(question, correlation_id) + if answered is not None: + return answered # /v1/answers itself is unavailable (non-2xx or unreachable) -- fall back to # real retrieval rather than degrading straight to "unavailable". This never # fabricates an answer to the question: it only ever shows an actually-cited @@ -290,9 +310,13 @@ class RAGcoreKnowledgeProvider: }, ) if response.status_code != 200: + KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", "non_2xx").inc() + self._open_answers_circuit() return None body = response.json() except (httpx.HTTPError, ValueError): + KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", "error").inc() + self._open_answers_circuit() return None try: @@ -311,6 +335,8 @@ class RAGcoreKnowledgeProvider: answerability = body.get("answerability", "not_answerable") is_grounded = answerability in _GROUNDED_ANSWERABILITY and sources evidence_state: EvidenceState = "grounded" if is_grounded else "insufficient" + KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", evidence_state).inc() + self._close_answers_circuit() return GroundedAnswer( answer=body.get("answer", "") if evidence_state == "grounded" else "", evidence_state=evidence_state, @@ -319,6 +345,8 @@ class RAGcoreKnowledgeProvider: correlation_id=correlation_id, ) except (TypeError, KeyError, ValueError): + KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", "malformed").inc() + self._open_answers_circuit() return None def _ask_via_search_fallback( @@ -342,9 +370,11 @@ class RAGcoreKnowledgeProvider: }, ) if response.status_code != 200: + KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "non_2xx").inc() return unavailable body = response.json() except (httpx.HTTPError, ValueError): + KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "error").inc() return unavailable try: @@ -362,8 +392,14 @@ class RAGcoreKnowledgeProvider: for result in results ] except (TypeError, KeyError, ValueError): + KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "malformed").inc() return unavailable + for result in results: + score = _retrieval_score(result) + if score is not None: + KNOWLEDGE_RETRIEVAL_SCORE.observe(score) + all_sources = _deduplicate_sources(sources) concepts = _question_concepts(question) qualified_sources = [ @@ -379,6 +415,7 @@ class RAGcoreKnowledgeProvider: ] sources = _rank_sources_for_concepts(_deduplicate_sources(qualified_sources), concepts) if not all_sources: + KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "insufficient").inc() return GroundedAnswer( answer="", evidence_state="insufficient", @@ -396,6 +433,7 @@ class RAGcoreKnowledgeProvider: for term in _DOMAIN_CONCEPTS["damage"] ) if not concepts or not sources or ("damage" in concepts and not damage_evidence): + KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "insufficient").inc() return GroundedAnswer( answer="", evidence_state="insufficient", @@ -407,6 +445,7 @@ class RAGcoreKnowledgeProvider: template = _LEAD_ANSWER_TEMPLATE.get(language, _LEAD_ANSWER_TEMPLATE[_DEFAULT_LANGUAGE]) lead = sources[0] answer = template.format(title=lead.title, excerpt=lead.excerpt) + KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "grounded").inc() return GroundedAnswer( answer=answer, evidence_state="grounded", diff --git a/backend/tests/test_knowledge.py b/backend/tests/test_knowledge.py index 60f5246..2a5503a 100644 --- a/backend/tests/test_knowledge.py +++ b/backend/tests/test_knowledge.py @@ -616,6 +616,35 @@ def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypa ) +def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch): + provider = RAGcoreKnowledgeProvider() + monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") + monkeypatch.setattr(provider._settings, "ragcore_answers_circuit_breaker_seconds", 60.0) + search_response = _FakeResponse(200, _search_body()) + monkeypatch.setattr( + provider, + "_client", + lambda: _FakeClient( + post_responses={ + "/v1/answers": _FakeResponse(503, {}), + "/v1/search": search_response, + } + ), + ) + assert ( + provider.ask("What is the vehicle return procedure?", "first").evidence_state + == "grounded" + ) + + monkeypatch.setattr( + provider, + "_ask_via_answers", + lambda *_args: (_ for _ in ()).throw(AssertionError("open circuit called answers")), + ) + second = provider.ask("What is the vehicle return procedure?", "second") + assert second.evidence_state == "grounded" + + def test_ragcore_provider_fallback_answer_is_localized(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") diff --git a/backend/tests/test_observability.py b/backend/tests/test_observability.py index 04a5f37..e3519f6 100644 --- a/backend/tests/test_observability.py +++ b/backend/tests/test_observability.py @@ -29,6 +29,8 @@ def test_metrics_expose_http_database_and_outbox_state(client): assert "mobilityops_http_requests_total" in response.text assert "mobilityops_database_ready 1.0" in response.text assert 'mobilityops_outbox_events{scenario="synthetic",status="failed"}' in response.text + assert "mobilityops_knowledge_provider_requests_total" in response.text + assert "mobilityops_knowledge_retrieval_score" in response.text def test_metrics_token_is_enforced_when_configured(client, monkeypatch): diff --git a/compose.observability.yaml b/compose.observability.yaml index e8906f7..e61799a 100644 --- a/compose.observability.yaml +++ b/compose.observability.yaml @@ -15,6 +15,33 @@ services: restart: unless-stopped networks: [mobilityops] + alertmanager: + image: prom/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d + profiles: ["observability"] + entrypoint: ["/bin/sh", "-ec"] + command: + - >- + sed -e "s|__WEBHOOK_URL__|$${ALERTMANAGER_WEBHOOK_URL}|g" + -e "s|__WEBHOOK_TOKEN__|$${MOBILITYOPS_WEBHOOK_TRIGGER_TOKEN}|g" + /etc/alertmanager/template.yml > /tmp/alertmanager.yml + && exec /bin/alertmanager --config.file=/tmp/alertmanager.yml + --storage.path=/alertmanager + environment: + ALERTMANAGER_WEBHOOK_URL: ${ALERTMANAGER_WEBHOOK_URL:-https://n8n.itworx.tech/webhook/mobilityops-alerts} + MOBILITYOPS_WEBHOOK_TRIGGER_TOKEN: ${MOBILITYOPS_WEBHOOK_TRIGGER_TOKEN:?Set the n8n webhook trigger token} + volumes: + - ./deploy/observability/alertmanager.yml:/etc/alertmanager/template.yml:ro + - mobilityops-alertmanager:/alertmanager + ports: + - "127.0.0.1:19093:9093" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9093/-/ready"] + interval: 15s + timeout: 5s + retries: 10 + restart: unless-stopped + networks: [mobilityops] + grafana: image: grafana/grafana:12.2.0@sha256:74144189b38447facf737dfd0f3906e42e0776212bf575dc3334c3609183adf7 profiles: ["observability"] @@ -29,10 +56,11 @@ services: - mobilityops-grafana:/var/lib/grafana ports: - "127.0.0.1:13000:3000" - depends_on: [prometheus] + depends_on: [prometheus, alertmanager] restart: unless-stopped networks: [mobilityops] volumes: mobilityops-prometheus: mobilityops-grafana: + mobilityops-alertmanager: diff --git a/compose.release.yaml b/compose.release.yaml new file mode 100644 index 0000000..755ea10 --- /dev/null +++ b/compose.release.yaml @@ -0,0 +1,14 @@ +services: + api: + image: ${MOBILITYOPS_API_IMAGE:?Set MOBILITYOPS_API_IMAGE to an immutable release image} + build: !reset null + + web: + image: ${MOBILITYOPS_WEB_IMAGE:?Set MOBILITYOPS_WEB_IMAGE to an immutable release image} + build: !reset null + + backup: + volumes: !override + - ${MOBILITYOPS_BACKUP_DIR:?Set MOBILITYOPS_BACKUP_DIR to the persistent backup directory}:/backups + - ${MOBILITYOPS_BACKUP_SECONDARY_DIR:?Set MOBILITYOPS_BACKUP_SECONDARY_DIR to the independent secondary directory}:/offsite + - ./deploy/unraid:/opt/mobilityops:ro diff --git a/compose.unraid.yaml b/compose.unraid.yaml index beb67b5..27492ae 100644 --- a/compose.unraid.yaml +++ b/compose.unraid.yaml @@ -47,6 +47,7 @@ services: BACKUP_INTERVAL_SECONDS: ${BACKUP_INTERVAL_SECONDS:-86400} BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-30} BACKUP_MINIMUM_COPIES: ${BACKUP_MINIMUM_COPIES:-7} + BACKUP_RESTORE_DRILL_INTERVAL_SECONDS: ${BACKUP_RESTORE_DRILL_INTERVAL_SECONDS:-604800} command: ["/opt/mobilityops/scheduled-backup.sh"] volumes: - ./backups/postgres:/backups @@ -56,7 +57,11 @@ services: db: condition: service_healthy healthcheck: - test: ["CMD-SHELL", "find /backups/latest-success -mmin -1560 -print -quit | grep -q ."] + test: + [ + "CMD-SHELL", + "find /backups/latest-success -mmin -1560 -print -quit | grep -q . && find /backups/latest-restore-drill -mmin -11520 -print -quit | grep -q .", + ] interval: 30m timeout: 5s retries: 3 diff --git a/compose.yaml b/compose.yaml index 43b9423..e7c2cc5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -18,6 +18,9 @@ services: build: context: . dockerfile: backend/Dockerfile + args: + VCS_REF: ${VCS_REF:-development} + BUILD_DATE: ${BUILD_DATE:-unknown} environment: MOBILITYOPS_ENV: ${MOBILITYOPS_ENV:-development} MOBILITYOPS_DEMO_MODE: ${MOBILITYOPS_DEMO_MODE:-true} @@ -32,6 +35,7 @@ services: RAGCORE_COLLECTION: ${RAGCORE_COLLECTION:-internal-procedures} RAGCORE_API_TOKEN: ${RAGCORE_API_TOKEN:-} RAGCORE_SPACE_ID: ${RAGCORE_SPACE_ID:-} + RAGCORE_ANSWERS_CIRCUIT_BREAKER_SECONDS: ${RAGCORE_ANSWERS_CIRCUIT_BREAKER_SECONDS:-60} N8N_WEBHOOK_URL: ${N8N_WEBHOOK_URL:-http://n8n:5678/webhook/mobilityops-return} N8N_WEBHOOK_TRIGGER_TOKEN: ${MOBILITYOPS_WEBHOOK_TRIGGER_TOKEN:-replace-me-n8n-webhook-trigger-token} N8N_CALLBACK_TOKEN: ${MOBILITYOPS_CALLBACK_TOKEN:-replace-me-n8n-callback-token} @@ -82,6 +86,8 @@ services: # Empty by default: the SPA calls its own origin and nginx proxies /api to the API, # which is what the CSP (connect-src 'self') expects. Only set this for split hosting. VITE_API_BASE_URL: ${VITE_API_BASE_URL:-} + VCS_REF: ${VCS_REF:-development} + BUILD_DATE: ${BUILD_DATE:-unknown} ports: - "1228:80" depends_on: diff --git a/contracts/mcp-tools.json b/contracts/mcp-tools.json index b952e1b..2d2ffe7 100644 --- a/contracts/mcp-tools.json +++ b/contracts/mcp-tools.json @@ -67,6 +67,7 @@ }, "additionalProperties": false }, + "endpoint": {"method": "POST", "path": "/api/v1/integrations/mcp/search-knowledge"}, "routing": { "preferred": "ragcore", "tenant": "northstar-mobility-demo", diff --git a/contracts/openapi.yaml b/contracts/openapi.yaml index d5269a4..b5e136a 100644 --- a/contracts/openapi.yaml +++ b/contracts/openapi.yaml @@ -1,9 +1,11 @@ openapi: 3.1.0 info: title: Fleet Ops API + description: Generated contract for Fleet Ops. The visible product name is Fleet + Ops; MobilityOps remains the technical repository and service identifier. version: 0.1.0 - description: Generated contract for Fleet Ops. The visible product name is Fleet Ops; MobilityOps remains - the technical repository and service identifier. +servers: +- url: http://localhost:8128 paths: /health: get: @@ -22,7 +24,8 @@ paths: /health/live: get: summary: Liveness - description: Process liveness only; external dependencies deliberately do not affect it. + description: Process liveness only; external dependencies deliberately do not + affect it. operationId: liveness_health_live_get responses: '200': @@ -37,7 +40,8 @@ paths: /health/ready: get: summary: Readiness - description: 'Traffic readiness: the API is useful only while its canonical database responds.' + description: 'Traffic readiness: the API is useful only while its canonical + database responds.' operationId: readiness_health_ready_get responses: '200': @@ -622,7 +626,8 @@ paths: type: array items: $ref: '#/components/schemas/AvailableVehicleOut' - title: Response List Available Vehicles Api V1 Bookings Availability Get + title: Response List Available Vehicles Api V1 Bookings Availability + Get '422': description: Validation Error content: @@ -1331,7 +1336,8 @@ paths: tags: - data-quality summary: Status Recommendation - description: 'Non-mutating preview: computes the recommendation without changing anything, + description: 'Non-mutating preview: computes the recommendation without changing + anything, resolving no issue and writing no audit event. Safe to call repeatedly.' operationId: status_recommendation_api_v1_data_quality_issues__public_ref__status_recommendation_post @@ -1462,7 +1468,8 @@ paths: tags: - integrations summary: Workflow Heartbeat - description: Authenticated, idempotent execution evidence from a canonical n8n workflow. + description: Authenticated, idempotent execution evidence from a canonical n8n + workflow. operationId: workflow_heartbeat_api_v1_integrations_n8n_heartbeat_post parameters: - name: X-Service-Token @@ -1523,7 +1530,8 @@ paths: schema: type: object additionalProperties: true - title: Response Return Callback Api V1 Integrations N8N Return Callback Post + title: Response Return Callback Api V1 Integrations N8N Return Callback + Post '422': description: Validation Error content: @@ -1535,13 +1543,17 @@ paths: tags: - integrations summary: Scheduled Scan - description: 'Triggered by the scheduled n8n quality-scan workflow. Narrow, read-mostly, and + description: 'Triggered by the scheduled n8n quality-scan workflow. Narrow, + read-mostly, and - safe to call repeatedly: run_scan() only ever creates an issue for a condition that + safe to call repeatedly: run_scan() only ever creates an issue for a condition + that - doesn''t already have one open, so a duplicate or overlapping trigger does no + doesn''t already have one open, so a duplicate or overlapping trigger does + no - duplicate domain work -- it just reports zero new issues for anything already known.' + duplicate domain work -- it just reports zero new issues for anything already + known.' operationId: scheduled_scan_api_v1_integrations_n8n_scheduled_scan_post parameters: - name: X-Service-Token @@ -1568,11 +1580,14 @@ paths: tags: - integrations summary: Workflow Error - description: 'Receives a bounded, secret-free failure report from the central n8n "Fleet Ops -- + description: 'Receives a bounded, secret-free failure report from the central + n8n "Fleet Ops -- - Workflow Error Handler" workflow, which is attached as the Error Workflow on every + Workflow Error Handler" workflow, which is attached as the Error Workflow + on every - other Fleet Ops n8n workflow. Idempotent on execution_id: n8n may redeliver the same + other Fleet Ops n8n workflow. Idempotent on execution_id: n8n may redeliver + the same error report (e.g. after a timed-out response), so this must not double-record.' operationId: workflow_error_api_v1_integrations_n8n_workflow_error_post @@ -1607,7 +1622,8 @@ paths: tags: - integrations summary: List Procedures - description: 'Read-only source list for the RAGcore Procedure Sync workflow: every procedure + description: 'Read-only source list for the RAGcore Procedure Sync workflow: + every procedure Markdown file Fleet Ops ships, across every supported language, with a stable @@ -1640,11 +1656,14 @@ paths: tags: - integrations summary: Procedures Sync Result - description: 'Receives a summary (counts only, no document content) from the n8n "Fleet Ops -- + description: 'Receives a summary (counts only, no document content) from the + n8n "Fleet Ops -- - RAGcore Procedure Sync" workflow once it finishes uploading procedures to RAGcore. + RAGcore Procedure Sync" workflow once it finishes uploading procedures to + RAGcore. - Idempotent on execution_id, matching the workflow-error and return-callback pattern.' + Idempotent on execution_id, matching the workflow-error and return-callback + pattern.' operationId: procedures_sync_result_api_v1_integrations_n8n_procedures_sync_result_post parameters: - name: X-Service-Token @@ -1880,7 +1899,8 @@ paths: type: array items: $ref: '#/components/schemas/AttentionVehicleOut' - title: Response Attention Vehicles Api V1 Integrations Mcp Attention Vehicles Get + title: Response Attention Vehicles Api V1 Integrations Mcp Attention + Vehicles Get '422': description: Validation Error content: @@ -4192,7 +4212,8 @@ components: description: 'Body of the n8n return follow-up callback. - n8n forwards its whole item (``JSON.stringify($json)``), so unknown keys are ignored; + n8n forwards its whole item (``JSON.stringify($json)``), so unknown keys are + ignored; only the fields we persist are validated and bounded.' ReturnPreviewResult: @@ -4800,5 +4821,3 @@ components: - execution_id - occurred_at title: WorkflowErrorReportResult -servers: -- url: http://localhost:8128 diff --git a/deploy/observability/alertmanager.yml b/deploy/observability/alertmanager.yml new file mode 100644 index 0000000..8fe4192 --- /dev/null +++ b/deploy/observability/alertmanager.yml @@ -0,0 +1,21 @@ +route: + receiver: fleetops-owner + group_by: [alertname, severity] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + +receivers: + - name: fleetops-owner + webhook_configs: + - url: __WEBHOOK_URL__ + send_resolved: true + http_config: + http_headers: + X-Fleet-Ops-Trigger-Token: + secrets: [__WEBHOOK_TOKEN__] + +inhibit_rules: + - source_matchers: ['alertname="MobilityOpsApiDown"'] + target_matchers: ['severity="warning"'] + equal: [alertname] diff --git a/deploy/observability/alerts.yml b/deploy/observability/alerts.yml index 8bace51..c74dd95 100644 --- a/deploy/observability/alerts.yml +++ b/deploy/observability/alerts.yml @@ -1,6 +1,12 @@ groups: - name: mobilityops rules: + - alert: FleetOpsWatchdog + expr: vector(1) + labels: {severity: none} + annotations: + summary: Fleet Ops alert delivery watchdog + description: This alert must always be visible at the configured receiver. - alert: MobilityOpsApiDown expr: up{job="mobilityops-api"} == 0 for: 2m diff --git a/deploy/observability/prometheus.yml b/deploy/observability/prometheus.yml index 891e406..75e2894 100644 --- a/deploy/observability/prometheus.yml +++ b/deploy/observability/prometheus.yml @@ -5,6 +5,11 @@ global: rule_files: - /etc/prometheus/alerts.yml +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] + scrape_configs: - job_name: mobilityops-api metrics_path: /metrics diff --git a/deploy/unraid/README.md b/deploy/unraid/README.md index e387238..b88a22c 100644 --- a/deploy/unraid/README.md +++ b/deploy/unraid/README.md @@ -17,8 +17,9 @@ existing shared n8n remains available on its established port 5678. Create `.env` from `.env.example`, replace every placeholder secret, set `MOBILITYOPS_ENV=production`, set both public URLs to -`https://fleetops.itworx.tech`, set `SESSION_COOKIE_SECURE=true`, and retain -`KNOWLEDGE_PROVIDER=demo` while RAGcore is not available. The internal `1236` listener is +`https://fleetops.itworx.tech`, set `SESSION_COOKIE_SECURE=true`, and configure +`KNOWLEDGE_PROVIDER=ragcore` only after the RAGcore health and source inventory checks pass. +The internal `1236` listener is an upstream for the TLS proxy, not a user-facing URL. ```bash @@ -26,11 +27,16 @@ cd /mnt/user/appdata/mobilityops ./deploy/unraid/configure-env.sh \ https://fleetops.itworx.tech \ https://n8n.itworx.tech/webhook/mobilityops-return -docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml up --build -d db api web backup -docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml exec api \ - python -m app.cli seed --reset +git archive --format=tar.gz -o /tmp/mobilityops-source.tar.gz HEAD +sha256sum /tmp/mobilityops-source.tar.gz +# Copy the archive and run deploy-release.sh with its SHA-256 and full Git SHA. ``` +`deploy-release.sh` stages a clean, commit-named release, builds OCI-labelled immutable +API/web images, promotes without a seed/reset, and verifies migrations, readiness, +backups and observability. Run `python -m app.cli seed --reset` only for initial setup or +an explicit synthetic-demo reset; it is never part of a routine deployment. + Migrations run automatically in the API entrypoint. Import and publish the MobilityOps workflow into the existing n8n container: @@ -57,17 +63,21 @@ docker logs --tail=200 n8n ## Backup and restore The `backup` service creates a backup immediately and then every 24 hours. Every dump is -validated with `pg_restore --list`, receives a SHA-256 sidecar and is retained for 30 days -with at least seven copies protected from pruning. Its healthcheck becomes unhealthy when -no successful backup has been recorded for 26 hours. Configure +checked by SHA-256 and `pg_restore --list`; at least weekly the newest dump is also restored +into a disposable database and its migration revision and core table counts are verified. +Backups are retained for 30 days with at least seven copies protected from pruning. Its +healthcheck becomes unhealthy when the daily backup or eight-day restore-drill SLA is missed. Configure `BACKUP_SECONDARY_DESTINATION=/offsite` plus an independently mounted `MOBILITYOPS_BACKUP_SECONDARY_DIR` for a second copy. -Create an additional on-demand backup or verify the newest scheduled backup: +Create an additional on-demand backup, verify the newest backup, or execute the isolated +restore drill: ```bash ./deploy/unraid/backup-postgres.sh ./deploy/unraid/verify-postgres-backups.sh +docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml exec -T backup \ + /opt/mobilityops/restore-drill-postgres.sh /backups/.dump ``` A restore is deliberately guarded and creates an additional safety backup before diff --git a/deploy/unraid/deploy-release.sh b/deploy/unraid/deploy-release.sh new file mode 100755 index 0000000..8a2ec7c --- /dev/null +++ b/deploy/unraid/deploy-release.sh @@ -0,0 +1,88 @@ +#!/bin/sh +set -eu + +# Stage a committed source archive, build commit-labelled immutable images and promote +# them without overlaying source files or reseeding persisted data. +archive="${1:-}" +expected_checksum="${2:-}" +revision="${3:-}" +root="${MOBILITYOPS_DEPLOY_ROOT:-/mnt/user/appdata/mobilityops}" +project="${COMPOSE_PROJECT_NAME:-mobilityops}" + +[ "${#revision}" -eq 40 ] || { echo "Revision must be a full Git SHA" >&2; exit 1; } +case "$revision" in *[!0-9a-f]*) echo "Revision must be lowercase hexadecimal" >&2; exit 1;; esac +[ -f "$archive" ] || { echo "Archive not found: $archive" >&2; exit 1; } +[ -n "$expected_checksum" ] || { echo "Expected SHA-256 is required" >&2; exit 1; } +[ -f "$root/.env" ] || { echo "Production .env is missing" >&2; exit 1; } + +actual_checksum="$(sha256sum "$archive" | awk '{print $1}')" +[ "$actual_checksum" = "$expected_checksum" ] || { + echo "Archive checksum mismatch" >&2 + exit 1 +} + +release_root="$root/.deploy/releases" +release_dir="$release_root/$revision" +mkdir -p "$release_root" +if [ -e "$release_dir" ]; then + echo "Release directory already exists; refusing to overwrite: $release_dir" >&2 + exit 1 +fi +mkdir "$release_dir" +tar -xzf "$archive" -C "$release_dir" + +short_revision="$(printf '%s' "$revision" | cut -c1-12)" +build_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +api_image="mobilityops-api:$short_revision" +web_image="mobilityops-web:$short_revision" + +docker build --target runtime --build-arg "VCS_REF=$revision" --build-arg "BUILD_DATE=$build_date" \ + --tag "$api_image" --file "$release_dir/backend/Dockerfile" "$release_dir" +docker build --build-arg "VCS_REF=$revision" --build-arg "BUILD_DATE=$build_date" \ + --tag "$web_image" "$release_dir/frontend" + +for image in "$api_image" "$web_image"; do + labelled_revision="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image")" + [ "$labelled_revision" = "$revision" ] || { + echo "Image revision label mismatch for $image" >&2 + exit 1 + } +done + +compose="docker compose --env-file $root/.env -p $project -f $release_dir/compose.yaml -f $release_dir/compose.unraid.yaml -f $release_dir/compose.observability.yaml -f $release_dir/compose.release.yaml --profile observability" +old_api_id="$(docker inspect --format '{{.Image}}' "$project-api-1" 2>/dev/null || true)" +old_web_id="$(docker inspect --format '{{.Image}}' "$project-web-1" 2>/dev/null || true)" +export MOBILITYOPS_API_IMAGE="$api_image" MOBILITYOPS_WEB_IMAGE="$web_image" +$compose up --no-build -d api web backup prometheus alertmanager grafana + +attempt=0 +until curl -fsS http://127.0.0.1:1236/health/ready > /dev/null; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 30 ]; then + if [ -n "$old_api_id" ] && [ -n "$old_web_id" ]; then + rollback_api="mobilityops-api:rollback-$short_revision" + rollback_web="mobilityops-web:rollback-$short_revision" + docker tag "$old_api_id" "$rollback_api" + docker tag "$old_web_id" "$rollback_web" + export MOBILITYOPS_API_IMAGE="$rollback_api" MOBILITYOPS_WEB_IMAGE="$rollback_web" + $compose up --no-build -d api web || true + fi + echo "Release failed readiness; source revision was not promoted" >&2 + exit 1 + fi + sleep 2 +done +$compose exec -T api alembic current +for service in backup prometheus alertmanager grafana; do + attempt=0 + until status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$project-$service-1" 2>/dev/null)" \ + && { [ "$status" = "healthy" ] || [ "$status" = "running" ]; }; do + attempt=$((attempt + 1)) + [ "$attempt" -lt 60 ] || { echo "$service did not become healthy" >&2; exit 1; } + sleep 5 + done +done +printf '%s\n' "$revision" > "$root/.deploy/source-revision" +printf '%s %s %s %s\n' "$revision" "$api_image" "$web_image" "$expected_checksum" \ + >> "$root/.deploy/release-history.log" +echo "Promoted Fleet Ops release $revision" diff --git a/deploy/unraid/restore-drill-postgres.sh b/deploy/unraid/restore-drill-postgres.sh new file mode 100755 index 0000000..bfe3ae9 --- /dev/null +++ b/deploy/unraid/restore-drill-postgres.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -eu + +backup_file="${1:-}" +[ -f "$backup_file" ] || { echo "Restore-drill backup not found: $backup_file" >&2; exit 1; } + +timestamp="$(date -u +%Y%m%d%H%M%S)" +drill_database="mobilityops_restore_drill_$timestamp" +source_database="${POSTGRES_DB:-mobilityops}" +database_host="${POSTGRES_HOST:-db}" + +cleanup() { + dropdb --if-exists --force --host="$database_host" --username="$POSTGRES_USER" \ + "$drill_database" > /dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM + +source_revision="$(psql --host="$database_host" --username="$POSTGRES_USER" \ + --dbname="$source_database" --tuples-only --no-align \ + --command='SELECT version_num FROM alembic_version')" +[ -n "$source_revision" ] || { echo "Source Alembic revision is unavailable" >&2; exit 1; } + +createdb --host="$database_host" --username="$POSTGRES_USER" "$drill_database" +pg_restore --no-owner --no-acl --host="$database_host" --username="$POSTGRES_USER" \ + --dbname="$drill_database" "$backup_file" + +restored_revision="$(psql --host="$database_host" --username="$POSTGRES_USER" \ + --dbname="$drill_database" --tuples-only --no-align \ + --command='SELECT version_num FROM alembic_version')" +[ "$restored_revision" = "$source_revision" ] || { + echo "Restored Alembic revision does not match production" >&2 + exit 1 +} + +counts="$(psql --host="$database_host" --username="$POSTGRES_USER" \ + --dbname="$drill_database" --tuples-only --no-align --field-separator=, \ + --command='SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM vehicles), (SELECT count(*) FROM bookings), (SELECT count(*) FROM audit_events)')" +case "$counts" in + 0,*|*,0,*|*,*,0,*|*,*,*,0) echo "Restored database is missing canonical records: $counts" >&2; exit 1 ;; +esac + +cleanup +trap - EXIT INT TERM +echo "Restore drill passed: revision=$restored_revision counts=$counts" diff --git a/deploy/unraid/scheduled-backup.sh b/deploy/unraid/scheduled-backup.sh index 9c0b8f4..205fd1b 100755 --- a/deploy/unraid/scheduled-backup.sh +++ b/deploy/unraid/scheduled-backup.sh @@ -6,9 +6,10 @@ secondary="${BACKUP_SECONDARY_DESTINATION:-}" interval="${BACKUP_INTERVAL_SECONDS:-86400}" retention_days="${BACKUP_RETENTION_DAYS:-30}" minimum_copies="${BACKUP_MINIMUM_COPIES:-7}" +restore_drill_interval="${BACKUP_RESTORE_DRILL_INTERVAL_SECONDS:-604800}" case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac -case "$interval:$retention_days:$minimum_copies" in *[!0-9:]*|:*|*:) echo "Backup settings must be integers" >&2; exit 1;; esac +case "$interval:$retention_days:$minimum_copies:$restore_drill_interval" in *[!0-9:]*|:*|*:) echo "Backup settings must be integers" >&2; exit 1;; esac mkdir -p "$destination" [ -z "$secondary" ] || mkdir -p "$secondary" @@ -24,6 +25,7 @@ while true; do (cd "$destination" && sha256sum "$(basename "$target")" > "$(basename "$target").sha256") if [ -n "$secondary" ]; then cp "$target" "$target.sha256" "$secondary/" + (cd "$secondary" && sha256sum -c "$(basename "$target.sha256")") fi date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-success" BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \ @@ -32,6 +34,15 @@ while true; do BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \ /opt/mobilityops/prune-postgres-backups.sh "$secondary" fi + drill_minutes=$((restore_drill_interval / 60)) + if [ ! -f "$destination/latest-restore-drill" ] \ + || ! find "$destination/latest-restore-drill" -mmin "-$drill_minutes" -print -quit | grep -q .; then + if /opt/mobilityops/restore-drill-postgres.sh "$target"; then + date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-restore-drill" + else + echo "Restore drill failed for $target" >&2 + fi + fi echo "Verified database backup: $target" else rm -f "$temporary" diff --git a/deploy/unraid/setup-alert-receiver.sh b/deploy/unraid/setup-alert-receiver.sh new file mode 100755 index 0000000..9b7362d --- /dev/null +++ b/deploy/unraid/setup-alert-receiver.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +container_name="${1:-n8n}" +source_workflow="${2:-n8n/workflows/fleet-ops-alert-receiver.json}" +[ -f "$source_workflow" ] || { echo "Missing workflow export: $source_workflow" >&2; exit 1; } +docker inspect "$container_name" >/dev/null 2>&1 || { echo "n8n container not found" >&2; exit 1; } + +container_workflow="/tmp/fleet-ops-alert-receiver.json" +trap 'docker exec "$container_name" rm -f "$container_workflow" >/dev/null 2>&1 || true' EXIT +docker cp "$source_workflow" "$container_name:$container_workflow" >/dev/null +docker exec "$container_name" n8n import:workflow --input="$container_workflow" +echo "Imported Fleet Ops alert receiver. Resolve the named Header Auth credential and publish the workflow." diff --git a/deploy/unraid/verify-postgres-backups.sh b/deploy/unraid/verify-postgres-backups.sh index 38d9032..75da09d 100755 --- a/deploy/unraid/verify-postgres-backups.sh +++ b/deploy/unraid/verify-postgres-backups.sh @@ -9,4 +9,7 @@ latest="$(find "$destination" -maxdepth 1 -type f -name 'mobilityops-*.dump' | s (cd "$destination" && sha256sum -c "$(basename "$latest.sha256")") docker compose -p "${COMPOSE_PROJECT_NAME:-mobilityops}" \ -f compose.yaml -f compose.unraid.yaml exec -T db pg_restore --list < "$latest" > /dev/null -printf 'Verified: %s\n' "$latest" +docker compose -p "${COMPOSE_PROJECT_NAME:-mobilityops}" \ + -f compose.yaml -f compose.unraid.yaml exec -T backup \ + /opt/mobilityops/restore-drill-postgres.sh "/backups/$(basename "$latest")" +printf 'Verified and restore-drilled: %s\n' "$latest" diff --git a/docs/19-visual-product-roadmap.md b/docs/19-visual-product-roadmap.md index ca1d24e..386d480 100644 --- a/docs/19-visual-product-roadmap.md +++ b/docs/19-visual-product-roadmap.md @@ -1,5 +1,9 @@ # MobilityOps visual product roadmap +> Status (2026-08-21): R0–R6 have been implemented. The observations below are the +> historical baseline that drove those changes, not a description of the current UI. +> Current release evidence lives in `artifacts/final-acceptance/summary.md`. + ## Purpose This roadmap turns the visual audit of the deployed MobilityOps PoC into an diff --git a/docs/demo-release/demo-runbook.md b/docs/demo-release/demo-runbook.md index deff09a..ec3f82b 100644 --- a/docs/demo-release/demo-runbook.md +++ b/docs/demo-release/demo-runbook.md @@ -48,34 +48,25 @@ disabled entirely via `DEMO_ALLOW_RESET=false` if an environment must not be reb ## Redeploying to Unraid ```bash -# From a clean local checkout on the target branch/commit: +# From a clean local checkout on the exact commit: git archive --format=tar.gz -o /tmp/mobilityops-source.tar.gz HEAD -scp /tmp/mobilityops-source.tar.gz unraid:/mnt/user/appdata/mobilityops/.deploy/source-.tar.gz -ssh unraid "cd /mnt/user/appdata/mobilityops \ - && tar -xzf .deploy/source-.tar.gz \ - && echo > .deploy/source-revision" - -# Rebuild only what changed (api and/or web); db is never rebuilt: -ssh unraid "cd /mnt/user/appdata/mobilityops \ - && docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml up --build -d api web" - -# Confirm migrations and reseed: -ssh unraid "cd /mnt/user/appdata/mobilityops \ - && docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml exec -T api python -m alembic current \ - && docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml exec -T api python -m app.cli seed --reset" +sha256sum /tmp/mobilityops-source.tar.gz +scp /tmp/mobilityops-source.tar.gz unraid:/mnt/user/appdata/mobilityops/.deploy/ +ssh unraid "/mnt/user/appdata/mobilityops/deploy/unraid/deploy-release.sh \ + /mnt/user/appdata/mobilityops/.deploy/mobilityops-source.tar.gz \ + " ``` -Extraction preserves the server's existing `.env` and named volumes (Postgres data, n8n -data) — the tarball never contains `.env` since it's gitignored. Never edit source -directly on the server; never deploy uncommitted changes. +The deployer verifies the archive checksum, extracts into a new commit-named release +directory, builds revision-labelled immutable images, promotes without reseeding, and +checks application, backup and observability health. Reset demo data only as a separate, +deliberate demo-preparation action. ## Rollback -`.deploy/source-revision` on the server records exactly which commit is live. Prior -source tarballs remain in `.deploy/` for rollback: extract an earlier -`source-.tar.gz`, update `source-revision` to match, and re-run the rebuild -step above. Database rollback is out of scope for this demo (migrations are additive; -there has been no destructive migration on this branch). +`.deploy/source-revision` and `.deploy/release-history.log` record the live commit and +image tags. A failed readiness check automatically restores the previous image IDs. +Database rollback remains a separate, explicitly guarded restore operation. ## Server safety (Unraid) @@ -89,9 +80,9 @@ guessed RAGcore/MCP URLs. PostgreSQL is never exposed externally. - **A scenario shows "Niet beschikbaar" on `/scenarios`**: it has already been resolved since the last reset (expected once you've worked through it) — reset to restore it. -- **Knowledge question returns "insufficient evidence"**: check the question is in - English and close to one of the suggested phrasings — the demo knowledge base is - English-only (see `demo-guide.md`). +- **Knowledge question returns "insufficient evidence"**: use a suggested question in + NL, EN or FR and inspect provider health/citations; the adapter refuses ungrounded + answers when RAGcore is slow, unavailable or lacks evidence. - **n8n shows a failed delivery that isn't the seeded demo one**: check `/automation`'s filter and retry — bounded retries mean it should self-heal within `n8n_max_attempts` attempts, or can be retried manually by an Operations Manager. diff --git a/frontend/Dockerfile b/frontend/Dockerfile index b0e8016..796bd3b 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -11,6 +11,12 @@ ENV VITE_API_BASE_URL=$VITE_API_BASE_URL RUN npm run build FROM nginx:1.27-alpine@sha256:65645c7bb6a0661892a8b03b89d0743208a18dd2f3f17a54ef4b76fb8e2f2a10 +ARG VCS_REF=development +ARG BUILD_DATE=unknown +LABEL org.opencontainers.image.title="Fleet Ops Web" \ + org.opencontainers.image.revision="$VCS_REF" \ + org.opencontainers.image.created="$BUILD_DATE" \ + org.opencontainers.image.source="https://fleetops.itworx.tech" COPY nginx.conf /etc/nginx/conf.d/default.conf COPY --from=build /app/dist /usr/share/nginx/html EXPOSE 80 diff --git a/frontend/e2e-live/smoke.spec.ts b/frontend/e2e-live/smoke.spec.ts new file mode 100644 index 0000000..ecc28b5 --- /dev/null +++ b/frontend/e2e-live/smoke.spec.ts @@ -0,0 +1,52 @@ +import { expect, test } from "@playwright/test"; + +test.describe.configure({ mode: "serial" }); + +test("public health and HTTPS-facing shell are available", async ({ page, request, baseURL }) => { + const readiness = await request.get("/health/ready"); + expect(readiness.ok()).toBeTruthy(); + expect(await readiness.json()).toMatchObject({ status: "ready", database: "up" }); + + await page.goto("/login"); + await expect(page).toHaveTitle(/Fleet Ops/); + await expect(page.getByRole("button", { name: "Verken als Operationsmanager" })).toBeVisible(); + if (baseURL?.startsWith("https://")) { + expect(page.url()).toMatch(/^https:\/\//); + } +}); + +test("non-destructive operator canary covers routes and grounded knowledge", async ({ page }) => { + const pageErrors: string[] = []; + const serverErrors: string[] = []; + page.on("pageerror", (error) => pageErrors.push(error.message)); + page.on("response", (response) => { + if (response.status() >= 500) serverErrors.push(`${response.status()} ${response.url()}`); + }); + + await page.goto("/login"); + await page.getByRole("button", { name: "Verken als Operationsmanager" }).click(); + await expect(page).toHaveURL(/\/dashboard$/); + + for (const [path, heading] of [ + ["/vehicles", "Wagenpark"], + ["/bookings", "Boekingen"], + ["/data-quality", "Datakwaliteit"], + ["/automation", "Integratiebeheer"], + ["/audit", "Auditgeschiedenis"], + ] as const) { + await page.goto(path); + await expect(page.getByRole("heading", { name: heading }).first()).toBeVisible(); + } + + await page.goto("/knowledge"); + await page + .getByRole("textbox", { name: "Vraag" }) + .fill("Wat moet ik doen wanneer een voertuig terugkomt met schade?"); + await page.getByRole("button", { name: "Vraag stellen" }).click(); + await expect(page.getByText("Onderbouwd met geciteerde procedures")).toBeVisible({ + timeout: 20_000, + }); + + expect(pageErrors).toEqual([]); + expect(serverErrors).toEqual([]); +}); diff --git a/frontend/e2e/__screenshots__/highlights-main.png b/frontend/e2e/__screenshots__/highlights-main.png new file mode 100644 index 0000000..68941e8 Binary files /dev/null and b/frontend/e2e/__screenshots__/highlights-main.png differ diff --git a/frontend/e2e/__screenshots__/login-access.png b/frontend/e2e/__screenshots__/login-access.png new file mode 100644 index 0000000..639975e Binary files /dev/null and b/frontend/e2e/__screenshots__/login-access.png differ diff --git a/frontend/e2e/accessibility.spec.ts b/frontend/e2e/accessibility.spec.ts new file mode 100644 index 0000000..90b08d5 --- /dev/null +++ b/frontend/e2e/accessibility.spec.ts @@ -0,0 +1,52 @@ +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; + +import { expect, test } from "@playwright/test"; + +test.use({ bypassCSP: true, reducedMotion: "reduce" }); + +const require = createRequire(import.meta.url); +const axeSource = readFileSync(require.resolve("axe-core/axe.min.js"), "utf8"); + +type AxeViolation = { + id: string; + impact: string | null; + help: string; + nodes: Array<{ target: string[] }>; +}; + +async function seriousViolations(page: import("@playwright/test").Page): Promise { + await page.addStyleTag({ + content: "*,*::before,*::after{animation:none!important;transition:none!important}", + }); + await page.addScriptTag({ content: axeSource }); + return page.evaluate(async () => { + const axe = (window as Window & { + axe: { + run: ( + context: Document, + options: object, + ) => Promise<{ violations: AxeViolation[] }>; + }; + }).axe; + const result = await axe.run(document, { + runOnly: { type: "tag", values: ["wcag2a", "wcag2aa", "wcag21aa"] }, + }); + return result.violations.filter( + (violation) => violation.impact === "critical" || violation.impact === "serious", + ); + }); +} + +test("principal routes have no serious automated accessibility violations", async ({ page }) => { + await page.goto("/login"); + expect(await seriousViolations(page)).toEqual([]); + + await page.getByRole("button", { name: "Verken als Operationsmanager" }).click(); + await expect(page).toHaveURL(/\/dashboard$/); + for (const route of ["/dashboard", "/data-quality", "/knowledge", "/automation", "/audit"] as const) { + await page.goto(route); + const violations = await seriousViolations(page); + expect(violations, `${route}: ${JSON.stringify(violations)}`).toEqual([]); + } +}); diff --git a/frontend/e2e/visual-regression.spec.ts b/frontend/e2e/visual-regression.spec.ts new file mode 100644 index 0000000..a82875f --- /dev/null +++ b/frontend/e2e/visual-regression.spec.ts @@ -0,0 +1,15 @@ +import { expect, test } from "@playwright/test"; + +test("public entry and engineering highlights retain their visual hierarchy", async ({ page }) => { + await page.goto("/login"); + await expect(page.locator(".login-access")).toHaveScreenshot("login-access.png", { + animations: "disabled", + maxDiffPixelRatio: 0.02, + }); + + await page.goto("/highlights"); + await expect(page.locator("main")).toHaveScreenshot("highlights-main.png", { + animations: "disabled", + maxDiffPixelRatio: 0.02, + }); +}); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4927b58..26dc768 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -20,6 +20,7 @@ "@types/react": "18.3.12", "@types/react-dom": "18.3.1", "@vitejs/plugin-react": "6.0.5", + "axe-core": "4.13.0", "eslint": "9.39.5", "eslint-plugin-jsx-a11y": "6.10.2", "eslint-plugin-react-hooks": "7.1.1", diff --git a/frontend/package.json b/frontend/package.json index d71e78b..d5ac201 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -6,6 +6,7 @@ "scripts": { "dev": "vite --host 0.0.0.0", "build": "tsc -b && vite build", + "budget": "node ../scripts/check-frontend-budget.mjs", "preview": "vite preview --host 0.0.0.0", "lint": "tsc -b --noEmit && eslint .", "test:e2e": "playwright test" @@ -23,6 +24,7 @@ "@types/react": "18.3.12", "@types/react-dom": "18.3.1", "@vitejs/plugin-react": "6.0.5", + "axe-core": "4.13.0", "eslint": "9.39.5", "eslint-plugin-jsx-a11y": "6.10.2", "eslint-plugin-react-hooks": "7.1.1", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 9e2359a..108705a 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -8,6 +8,7 @@ export default defineConfig({ timeout: 30_000, fullyParallel: false, workers: 1, + snapshotPathTemplate: "{testDir}/__screenshots__/{arg}{ext}", reporter: [["list"], ["html", { open: "never", outputFolder: "playwright-report" }]], use: { baseURL: process.env.MOBILITYOPS_PUBLIC_URL ?? "http://localhost:1228", diff --git a/frontend/playwright.live.config.ts b/frontend/playwright.live.config.ts new file mode 100644 index 0000000..8a704a3 --- /dev/null +++ b/frontend/playwright.live.config.ts @@ -0,0 +1,18 @@ +import { defineConfig, devices } from "@playwright/test"; + +export default defineConfig({ + testDir: "./e2e-live", + timeout: 30_000, + fullyParallel: false, + workers: 1, + reporter: [["list"], ["html", { open: "never", outputFolder: "playwright-live-report" }]], + use: { + baseURL: process.env.MOBILITYOPS_PUBLIC_URL ?? "http://localhost:1228", + trace: "retain-on-failure", + screenshot: "only-on-failure", + }, + projects: [ + { name: "chromium", use: { ...devices["Desktop Chrome"] } }, + { name: "firefox", use: { ...devices["Desktop Firefox"] } }, + ], +}); diff --git a/frontend/src/styles.css b/frontend/src/styles.css index 4957919..344f31c 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -5,7 +5,7 @@ font-synthesis: none; --ink: #0f172a; --ink-soft: #344256; - --muted: #657386; + --muted: #5d6a7c; --muted-light: #94a0af; --canvas: #f5f7fa; --surface: #ffffff; @@ -140,7 +140,7 @@ a:hover { color: var(--teal); } .demo-badge-close { position: absolute; top: 8px; right: 8px; width: 28px; height: 28px; } .demo-badge-close svg { width: 14px; height: 14px; } #main-content { width: min(1320px, calc(100% - 56px)); margin: 0 auto; padding: 38px 0 64px; flex: 1; } -.app-footer { min-height: 52px; display: flex; justify-content: space-between; align-items: center; gap: 16px; padding: 0 28px; color: var(--muted); border-top: 1px solid var(--line); font-size: .68rem; } +.app-footer { min-height: 52px; display: flex; justify-content: space-between; align-items: center; gap: 16px; padding: 0 28px; color: #5f6d80; border-top: 1px solid var(--line); font-size: .68rem; } .mobile-nav, .nav-scrim { display: none; } .page { animation: page-in .28s ease both; } @@ -231,7 +231,7 @@ a:hover { color: var(--teal); } .integration-list li:last-child, .recent-list li:last-child { border-bottom: 0; }.integration-list li > div, .recent-list li > div { flex: 1; display: grid; gap: 3px; } .integration-list strong, .recent-list strong { font-size: .73rem; }.integration-list div span, .recent-list div span { color: var(--muted); font-size: .65rem; } .integration-mark { width: 31px; height: 31px; display: grid; place-items: center; border-radius: var(--radius); color: white; font-size: .58rem; font-weight: 800; letter-spacing: -.04em; } -.integration-n8n { background: #e85d36; }.integration-rag { background: #4a4b95; }.integration-mcp { background: #314358; } +.integration-n8n { background: #b84424; }.integration-rag { background: #4a4b95; }.integration-mcp { background: #314358; } .activity-icon { width: 30px; height: 30px; display: grid; place-items: center; color: var(--teal-dark); background: var(--teal-pale); border-radius: 50%; }.activity-icon svg { width: 15px; } .recent-list time { color: var(--muted); font-size: .62rem; white-space: nowrap; } diff --git a/n8n/workflows/MANIFEST.md b/n8n/workflows/MANIFEST.md index 5534d12..e37e39c 100644 --- a/n8n/workflows/MANIFEST.md +++ b/n8n/workflows/MANIFEST.md @@ -1,6 +1,6 @@ # n8n workflow manifest -Source of truth for the four canonical Fleet Ops n8n workflows. Definitions in this +Source of truth for the five canonical Fleet Ops n8n workflows. Definitions in this directory are cleaned exports of the live workflows on `https://n8n.itworx.tech` — credential values are never embedded; nodes reference named n8n credentials instead. Run `n8n/workflows/check_drift.py` to compare a live workflow against its repo definition. @@ -108,3 +108,17 @@ runs from the editor. This is a known limitation of the live-validation evidence round: the mock-data path exercises the same nodes/logic and the real Fleet Ops endpoint, but a fully automatic (schedule- or webhook-triggered) failure cascading into this handler was not observed live. + +## 5. Fleet Ops — Alert Receiver + +| Field | Value | +|---|---| +| File | `fleet-ops-alert-receiver.json` | +| Purpose | Receive validated Alertmanager firing/resolved notifications and route them to the watched owner mailbox. | +| Trigger | Production webhook, `POST /webhook/mobilityops-alerts`, Header Auth (`Fleet Ops Webhook Trigger Token`) | +| Required credentials | `Fleet Ops Webhook Trigger Token` (Header Auth); `M365 n8n Shared Mailbox` (Microsoft Outlook OAuth2) | +| Live workflow ID | `mobilityops-alert-receiver` | +| Active status | Imported inactive by default; production deployment must publish it before enabling Alertmanager. | +| Error Workflow | `Fleet Ops — Workflow Error Handler` (wired) | +| Safety | Strict Alertmanager payload shape, bounded to 25 alerts and bounded text fields. | +| Checksum (sha256) | `3672ad3d14b65c603c8c3f2067197cc50372e9d4c2fc314d1c9b6aaf3ecef444` | diff --git a/n8n/workflows/check_drift.py b/n8n/workflows/check_drift.py index 90e2ac0..fe3f303 100644 --- a/n8n/workflows/check_drift.py +++ b/n8n/workflows/check_drift.py @@ -34,6 +34,7 @@ KNOWN_WORKFLOWS = [ ("fleet-ops-data-quality-scan.json", "mobilityops-scheduled-quality-scan"), ("fleet-ops-error-handler.json", "Xppn2rAEqUuyiCJF"), ("fleet-ops-ragcore-procedure-sync.json", "6wbkc4d1AouGpmWT"), + ("fleet-ops-alert-receiver.json", "mobilityops-alert-receiver"), ] # Fields that legitimately differ between a committed definition and the live instance diff --git a/n8n/workflows/fleet-ops-alert-receiver.json b/n8n/workflows/fleet-ops-alert-receiver.json new file mode 100644 index 0000000..0da470e --- /dev/null +++ b/n8n/workflows/fleet-ops-alert-receiver.json @@ -0,0 +1,36 @@ +{ + "id": "mobilityops-alert-receiver", + "name": "Fleet Ops — Alert Receiver", + "active": false, + "nodes": [ + { + "parameters": {"httpMethod": "POST", "path": "mobilityops-alerts", "authentication": "headerAuth", "responseMode": "responseNode", "options": {}}, + "id": "alert-webhook", "name": "Alertmanager webhook", "type": "n8n-nodes-base.webhook", "typeVersion": 2, "position": [240, 300], "webhookId": "mobilityops-alerts", + "credentials": {"httpHeaderAuth": {"name": "Fleet Ops Webhook Trigger Token"}} + }, + { + "parameters": {"jsCode": "const body = $json.body ?? $json;\nif (!['firing', 'resolved'].includes(body.status) || !Array.isArray(body.alerts)) throw new Error('Invalid Alertmanager payload');\nconst alerts = body.alerts.slice(0, 25).map(a => ({name: String(a.labels?.alertname || 'Unknown alert').slice(0, 120), severity: String(a.labels?.severity || 'unknown').slice(0, 32), summary: String(a.annotations?.summary || '').slice(0, 500)}));\nconst esc = v => String(v).replace(/[&<>\\\"']/g, c => ({'&':'&','<':'<','>':'>','\\\"':'"',\"'\":'''}[c]));\nconst resolved = body.status === 'resolved';\nreturn [{json:{subject:`[Fleet Ops] ${resolved ? 'Hersteld' : 'Alert'} — ${alerts.length} signaal/signalen`,html:`

Fleet Ops ${resolved ? 'hersteld' : 'monitoringalert'}

    ${alerts.map(a => `
  • ${esc(a.severity)} · ${esc(a.name)}
    ${esc(a.summary)}
  • `).join('')}

Ontvangen ${esc($now.toISO())}

`}}];"}, + "id": "validate-format", "name": "Validate and format", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [500, 300] + }, + { + "parameters": {"respondWith": "json", "responseBody": "={{ { accepted: true } }}", "options": {}}, + "id": "accept", "name": "Accept alert", "type": "n8n-nodes-base.respondToWebhook", "typeVersion": 1.4, "position": [740, 300] + }, + { + "parameters": {"toRecipients": "jens@itworx.tech", "subject": "={{ $('Validate and format').item.json.subject }}", "bodyContent": "={{ $('Validate and format').item.json.html }}", "additionalFields": {"from": "n8n@itworx.tech", "bodyContentType": "html"}}, + "id": "email", "name": "Send owner email", "type": "n8n-nodes-base.microsoftOutlook", "typeVersion": 2, "position": [980, 300], + "credentials": {"microsoftOutlookOAuth2Api": {"id": "EDTj3sOsganaoDVL", "name": "M365 n8n Shared Mailbox"}}, + "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 1000 + } + ], + "connections": { + "Alertmanager webhook": {"main": [[{"node": "Validate and format", "type": "main", "index": 0}]]}, + "Validate and format": {"main": [[{"node": "Accept alert", "type": "main", "index": 0}]]}, + "Accept alert": {"main": [[{"node": "Send owner email", "type": "main", "index": 0}]]} + }, + "settings": {"executionOrder": "v1", "timezone": "Europe/Brussels", "errorWorkflow": "Xppn2rAEqUuyiCJF", "executionTimeout": 120, "saveDataSuccessExecution": "none", "saveDataErrorExecution": "all"}, + "staticData": null, + "meta": {"templateCredsSetupCompleted": false}, + "pinData": {}, + "tags": [] +} diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..7e89204 --- /dev/null +++ b/renovate.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["config:recommended", "docker:pinDigests", "helpers:pinGitHubActionDigests"], + "dependencyDashboard": true, + "minimumReleaseAge": "7 days", + "schedule": ["before 5am on monday"], + "packageRules": [ + { + "matchManagers": ["dockerfile", "docker-compose", "github-actions"], + "groupName": "pinned runtime and CI dependencies" + }, + { + "matchUpdateTypes": ["major"], + "dependencyDashboardApproval": true + } + ] +} diff --git a/scripts/check-contracts.py b/scripts/check-contracts.py new file mode 100755 index 0000000..1519f4f --- /dev/null +++ b/scripts/check-contracts.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Fail when checked-in API, event, MCP or n8n contracts drift from the code.""" + +from __future__ import annotations + +import hashlib +import json +import re +import sys +from pathlib import Path + +import yaml + +from app.main import app + +ROOT = Path(__file__).resolve().parents[1] + + +def fail(message: str, failures: list[str]) -> None: + failures.append(message) + + +def check_openapi(failures: list[str]) -> None: + committed = yaml.safe_load((ROOT / "contracts/openapi.yaml").read_text(encoding="utf-8")) + generated = app.openapi() + if committed != generated: + fail( + "contracts/openapi.yaml differs from app.openapi(); regenerate it with " + "scripts/generate-openapi.py", + failures, + ) + + +def check_mcp(failures: list[str]) -> None: + contract = json.loads((ROOT / "contracts/mcp-tools.json").read_text(encoding="utf-8")) + contracted = { + (tool["endpoint"]["method"].upper(), tool["endpoint"]["path"]) + for tool in contract["tools"] + if tool.get("read_only") is True and isinstance(tool.get("endpoint"), dict) + } + generated = app.openapi() + implemented = { + (method.upper(), path) + for path, operations in generated["paths"].items() + if path.startswith("/api/v1/integrations/mcp/") + for method in operations + if method.lower() in {"get", "post", "put", "patch", "delete"} + } + if len(contracted) != len(contract["tools"]): + fail("Every MCP tool must be read-only and declare its provider endpoint", failures) + if contracted != implemented: + fail( + f"MCP endpoint drift: contract={sorted(contracted)!r}, code={sorted(implemented)!r}", + failures, + ) + if any(method not in {"GET", "POST"} for method, _path in contracted): + fail("MCP contract exposes an unsupported mutation method", failures) + + +def check_event_schema(failures: list[str]) -> None: + schema = json.loads((ROOT / "contracts/events.schema.json").read_text(encoding="utf-8")) + expected_envelope = { + "event_id", + "event_type", + "occurred_at", + "correlation_id", + "aggregate", + "data", + } + expected_data = { + "vehicle_ref", + "inspection_ref", + "resulting_vehicle_status", + "attention_reasons", + } + if set(schema.get("required", [])) != expected_envelope: + fail("Event envelope required fields drifted", failures) + data_schema = schema.get("properties", {}).get("data", {}) + if set(data_schema.get("required", [])) != expected_data: + fail("vehicle.returned.v1 data fields drifted", failures) + event_type = schema.get("properties", {}).get("event_type", {}).get("const") + if event_type != "vehicle.returned.v1": + fail("Unexpected event_type contract", failures) + + +def check_workflows(failures: list[str]) -> None: + workflows_dir = ROOT / "n8n/workflows" + manifest = (workflows_dir / "MANIFEST.md").read_text(encoding="utf-8") + expected = { + "fleet-ops-vehicle-return.json": "mobilityops-return-processing", + "fleet-ops-data-quality-scan.json": "mobilityops-scheduled-quality-scan", + "fleet-ops-ragcore-procedure-sync.json": "6wbkc4d1AouGpmWT", + "fleet-ops-error-handler.json": "Xppn2rAEqUuyiCJF", + "fleet-ops-alert-receiver.json": "mobilityops-alert-receiver", + } + declared_hashes = set(re.findall(r"`([0-9a-f]{64})`", manifest)) + for filename, workflow_id in expected.items(): + path = workflows_dir / filename + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest not in declared_hashes: + fail(f"{filename} checksum is missing or stale in MANIFEST.md", failures) + definition = json.loads(raw) + if definition.get("id") != workflow_id: + fail(f"{filename} has unexpected workflow id", failures) + serialized = raw.decode("utf-8") + if "http://fleetops.itworx.tech" in serialized: + fail(f"{filename} contains a cleartext Fleet Ops callback", failures) + + +def main() -> int: + failures: list[str] = [] + check_openapi(failures) + check_mcp(failures) + check_event_schema(failures) + check_workflows(failures) + if failures: + for failure in failures: + print(f"ERROR: {failure}", file=sys.stderr) + return 1 + print("OpenAPI, event, MCP and n8n contracts are synchronized.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-frontend-budget.mjs b/scripts/check-frontend-budget.mjs new file mode 100644 index 0000000..fcb408f --- /dev/null +++ b/scripts/check-frontend-budget.mjs @@ -0,0 +1,18 @@ +import { readdir, stat } from "node:fs/promises"; +import { join } from "node:path"; + +const limits = { ".js": 400 * 1024, ".css": 180 * 1024 }; +const assets = join(import.meta.dirname, "..", "frontend", "dist", "assets"); +const failures = []; + +for (const name of await readdir(assets)) { + const extension = Object.keys(limits).find((candidate) => name.endsWith(candidate)); + if (!extension) continue; + const bytes = (await stat(join(assets, name))).size; + console.log(`${name}: ${(bytes / 1024).toFixed(1)} KiB / ${limits[extension] / 1024} KiB`); + if (bytes > limits[extension]) failures.push(name); +} + +if (failures.length) { + throw new Error(`Frontend asset budget exceeded: ${failures.join(", ")}`); +} diff --git a/scripts/check-source-budgets.py b/scripts/check-source-budgets.py new file mode 100755 index 0000000..aebca2e --- /dev/null +++ b/scripts/check-source-budgets.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Prevent known large modules from growing while they are incrementally decomposed.""" + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +LINE_LIMITS = { + "backend/app/services/data_quality.py": 950, + "frontend/src/pages/DataQualityIssueDetail.tsx": 850, +} +BYTE_LIMITS = {"frontend/src/styles.css": 84_000} + +failures: list[str] = [] +for relative, limit in LINE_LIMITS.items(): + count = len((ROOT / relative).read_text(encoding="utf-8").splitlines()) + print(f"{relative}: {count}/{limit} lines") + if count > limit: + failures.append(relative) +for relative, limit in BYTE_LIMITS.items(): + count = (ROOT / relative).stat().st_size + print(f"{relative}: {count}/{limit} bytes") + if count > limit: + failures.append(relative) + +if failures: + raise SystemExit("Source budget exceeded; extract a focused module: " + ", ".join(failures)) diff --git a/scripts/generate-openapi.py b/scripts/generate-openapi.py new file mode 100755 index 0000000..7353e0a --- /dev/null +++ b/scripts/generate-openapi.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +"""Regenerate the deterministic checked-in FastAPI OpenAPI contract.""" + +from pathlib import Path + +import yaml + +from app.main import app + +target = Path(__file__).resolve().parents[1] / "contracts/openapi.yaml" +target.write_text( + yaml.safe_dump(app.openapi(), sort_keys=False, allow_unicode=True), + encoding="utf-8", +) +print(target)