diff --git a/.env.example b/.env.example index 59e6fb8..83b5b8e 100644 --- a/.env.example +++ b/.env.example @@ -13,9 +13,8 @@ POSTGRES_PASSWORD=mobilityops # this (or MOBILITYOPS_CALLBACK_TOKEN) still holds its placeholder value. APP_SECRET=replace-in-production TZ=Europe/Brussels -# Session cookie Secure flag. Keep false for LAN/plain-HTTP deployments (including the -# current Unraid review environment); set true only once MobilityOps is served over HTTPS, -# otherwise browsers will silently drop the cookie and no one can log in. +# Session cookie Secure flag. Development on localhost may use false; production startup +# requires both an HTTPS public URL and this value set to true. SESSION_COOKIE_SECURE=false # Optional OpenID Connect login. Public demo role buttons remain available when enabled. diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index eb2f598..2b7ff7a 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -9,9 +9,9 @@ jobs: backend: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Secret scan - uses: trufflesecurity/trufflehog@v3.79.0 + uses: trufflesecurity/trufflehog@b9dd330365132cd2d01dd5dc8a857a056a2544e1 # v3.79.0 with: path: ./ extra_args: --only-verified @@ -19,13 +19,15 @@ jobs: run: sh scripts/run-isolated-tests.sh - name: Backend static checks run: | - docker compose -p mobilityops-ci -f compose.yaml -f compose.test.yaml run --build --rm api ruff check app tests scripts + docker compose -p mobilityops-ci -f compose.yaml -f compose.test.yaml run --build --rm api ruff check app tests docker compose -p mobilityops-ci -f compose.yaml -f compose.test.yaml run --rm api mypy app - - name: Backend dependency vulnerability scan (HIGH/CRITICAL) - uses: aquasecurity/trivy-action@v0.30.0 + - name: Build production API image for vulnerability scan + run: docker build --target runtime --tag mobilityops-api-ci --file backend/Dockerfile . + - name: Production API image vulnerability scan (HIGH/CRITICAL) + uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # v0.30.0 with: - scan-type: fs - scan-ref: backend + scan-type: image + image-ref: mobilityops-api-ci format: table severity: HIGH,CRITICAL exit-code: "1" @@ -37,8 +39,8 @@ jobs: frontend: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 22 cache: npm @@ -62,8 +64,8 @@ jobs: runs-on: ubuntu-latest needs: [backend, frontend] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 22 cache: npm @@ -90,7 +92,7 @@ jobs: run: npx playwright test - name: Upload Playwright report if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: playwright-report path: frontend/playwright-report diff --git a/Makefile b/Makefile index 91c0600..7a497ba 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,8 @@ test: sh scripts/run-isolated-tests.sh lint: - docker compose run --rm api ruff check . - docker compose run --rm api mypy app + docker compose -f compose.yaml -f compose.test.yaml run --build --rm api ruff check app tests + docker compose -f compose.yaml -f compose.test.yaml run --rm api mypy app cd frontend && npm run lint seed: diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 3e34623..e9e0411 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -1,5 +1,39 @@ # Project state +## M41 — full review remediation and hardening (2026-08-21) + +- Closed all findings from the repository-wide review without expanding the locked PoC: + production now refuses placeholder MCP credentials, cleartext public URLs and insecure + session cookies; OIDC requires an explicit verified-email claim; nginx overwrites the + forwarded client address and the API uses the proxy-appended hop for rate limiting. +- Added bounded per-IP/per-session knowledge requests and an explicit minimum RAGcore + retrieval score. Weak or concept-mismatched search fallback evidence is returned as + `insufficient`, never `grounded`. MCP audit attribution now authenticates the fixed Hub + service identity and stores the Hub-reported caller only as non-authoritative metadata. +- Serialised data-quality scans with a PostgreSQL transaction advisory lock, added a + partial unique index for one open issue per condition, and locked issue rows for every + mutating resolution. Concurrent scan and concurrent-resolution regression tests pass. +- Split the backend production/test image stages and locks: the runtime contains no test + suite, pytest, Ruff or mypy. All container bases and CI actions are digest/SHA pinned. + CI now builds and scans the real runtime image. The initial Debian 13 base exposed 36 + fixable HIGH findings; switching to the pinned Python 3.12 Bookworm image reduced the + final Trivy result to **0 HIGH/CRITICAL** across OS and Python packages. +- Moved every central n8n callback/source URL to the existing HTTPS endpoint + `fleetops.itworx.tech`, refreshed workflow checksums, corrected stale workflow status/ + node-count documentation, fixed the return-odometer documentation and made the Unraid + bootstrap enforce HTTPS plus Secure cookies. Makefile lint now always builds the test + target and cannot silently inspect a stale runtime image. +- Validation evidence: focused security/integration/data-quality suite **116 passed**; + final isolated PostgreSQL backend suite **270 passed**; Ruff and mypy clean; frontend + lint/build clean; npm audit **0 vulnerabilities**; production runtime contains no dev + dependencies/tests; Trivy runtime scan **0 HIGH/CRITICAL**; full Playwright acceptance + **153/153 passed in 5.1 minutes**; Compose test/Unraid configs and `git diff --check` + clean. Existing public TLS returns 200 with HSTS and HTTP redirects to HTTPS. +- Exact next action: commit and push M41, take a verified production backup, update the + deployment's public URL/Secure-cookie settings, deploy the committed archive and + migration `4f2b9c8d7e61`, safely republish the four HTTPS n8n definitions, then repeat + live health, migration, security-header and browser acceptance checks. + ## M40 — publish and redeploy M39 (2026-08-17) - Published three validated commits to Gitea `master`: the backend dependency and secret diff --git a/backend/Dockerfile b/backend/Dockerfile index 76d6faa..621680f 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,19 +1,26 @@ -FROM python:3.12-slim +FROM python:3.12-slim-bookworm@sha256:a116514e19457bcb7af7efe9c3dd0b9b71e85b317694e7882a1c52aa15a78134 AS runtime-base ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 WORKDIR /app -COPY backend/requirements.lock ./ -RUN pip install --no-cache-dir -r requirements.lock +COPY backend/requirements-prod.lock ./ +RUN pip install --no-cache-dir -r requirements-prod.lock COPY backend/pyproject.toml ./ COPY backend/app ./app COPY backend/alembic ./alembic COPY backend/alembic.ini ./ -COPY backend/tests ./tests COPY seed ./seed COPY knowledge ./knowledge COPY backend/entrypoint.sh ./entrypoint.sh RUN pip install --no-cache-dir --no-deps -e . && chmod +x ./entrypoint.sh \ && addgroup --system app && adduser --system --ingroup app --home /app app \ && chown -R app:app /app + +FROM runtime-base AS test +COPY backend/requirements.lock ./requirements.lock +RUN pip install --no-cache-dir -r requirements.lock +COPY backend/tests ./tests +USER app + +FROM runtime-base AS runtime # Run migrations and the API as an unprivileged user; nothing here needs root. USER app EXPOSE 8000 diff --git a/backend/alembic/versions/4f2b9c8d7e61_unique_open_quality_condition.py b/backend/alembic/versions/4f2b9c8d7e61_unique_open_quality_condition.py new file mode 100644 index 0000000..2aea5b3 --- /dev/null +++ b/backend/alembic/versions/4f2b9c8d7e61_unique_open_quality_condition.py @@ -0,0 +1,27 @@ +"""enforce one open issue per detected condition + +Revision ID: 4f2b9c8d7e61 +Revises: 0a4c1d2e3f5b +""" + +from alembic import op +import sqlalchemy as sa + +revision = "4f2b9c8d7e61" +down_revision = "0a4c1d2e3f5b" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index( + "uq_data_quality_one_open_condition", + "data_quality_issues", + ["rule_type", "entity_type", "entity_id"], + unique=True, + postgresql_where=sa.text("status = 'open'"), + ) + + +def downgrade() -> None: + op.drop_index("uq_data_quality_one_open_condition", table_name="data_quality_issues") diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index 7cf7656..ba6d513 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -67,7 +67,7 @@ def require_operations_manager( @dataclass(frozen=True) class McpClientContext: - client_id: str + reported_client_id: str tenant: str @@ -91,4 +91,4 @@ def require_mcp_service_token( ) if x_tenant_id is not None and x_tenant_id != settings.ragcore_tenant: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Tenant mismatch") - return McpClientContext(client_id=x_client_id, tenant=settings.ragcore_tenant) + return McpClientContext(reported_client_id=x_client_id, tenant=settings.ragcore_tenant) diff --git a/backend/app/api/routers/auth.py b/backend/app/api/routers/auth.py index a05781f..1b0ee81 100644 --- a/backend/app/api/routers/auth.py +++ b/backend/app/api/routers/auth.py @@ -38,10 +38,11 @@ _login_limiter = ( def _client_key(request: Request) -> str: # The API sits behind the web container's reverse proxy in every documented - # deployment; honour the first hop of X-Forwarded-For when present. + # deployment. The proxy appends/overwrites the socket peer as the final hop, so an + # attacker-controlled leading value must never select a fresh limiter bucket. forwarded = request.headers.get("x-forwarded-for", "") if forwarded: - return forwarded.split(",")[0].strip() + return forwarded.split(",")[-1].strip() return request.client.host if request.client else "unknown" @@ -143,7 +144,7 @@ def _allowed_oidc_email(email: str) -> bool: def _resolve_oidc_user(db: Session, claims: dict[str, object]) -> User: subject = str(claims.get("sub") or "").strip() email = str(claims.get("email") or "").strip().lower() - if not subject or not email or claims.get("email_verified") is False: + if not subject or not email or claims.get("email_verified") is not True: raise HTTPException(status_code=401, detail="Verified OIDC email and subject are required") if not _allowed_oidc_email(email): raise HTTPException(status_code=403, detail="Email domain is not allowed") diff --git a/backend/app/api/routers/knowledge.py b/backend/app/api/routers/knowledge.py index 662a3ce..7dd211c 100644 --- a/backend/app/api/routers/knowledge.py +++ b/backend/app/api/routers/knowledge.py @@ -1,20 +1,32 @@ from __future__ import annotations +import hashlib import uuid from typing import Literal -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Request, status from pydantic import BaseModel, Field from sqlalchemy import select from sqlalchemy.orm import Session from app.api.deps import get_current_user, get_db +from app.core.config import get_settings +from app.core.ratelimit import SlidingWindowLimiter from app.models.audit import AuditEvent from app.schemas import CurrentUser from app.services.audit import record_audit_event from app.services.knowledge import GroundedAnswer, KnowledgeHealth, get_knowledge_provider router = APIRouter(prefix="/api/v1/knowledge", tags=["knowledge"]) +settings = get_settings() +_question_limiter = ( + SlidingWindowLimiter( + max_requests=settings.knowledge_max_requests, + window_seconds=settings.knowledge_rate_limit_window_seconds, + ) + if settings.knowledge_max_requests > 0 + else None +) SupportedLanguage = Literal["nl-BE", "en-GB", "fr-BE"] @@ -32,9 +44,29 @@ class KnowledgeFeedbackRequest(BaseModel): @router.post("/questions", response_model=GroundedAnswer) def ask_question( body: AskQuestionRequest, + request: Request, db: Session = Depends(get_db), user: CurrentUser = Depends(get_current_user), ) -> GroundedAnswer: + if _question_limiter is not None: + forwarded = request.headers.get("x-forwarded-for", "") + client_ip = ( + forwarded.split(",")[-1].strip() + if forwarded + else request.client.host if request.client else "unknown" + ) + token = request.cookies.get(settings.session_cookie_name, "") + session_key = hashlib.sha256(token.encode("utf-8")).hexdigest() + retry_after = max( + _question_limiter.consume(f"ip:{client_ip}"), + _question_limiter.consume(f"session:{session_key}"), + ) + if retry_after: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail="Too many knowledge questions. Try again later.", + headers={"Retry-After": str(retry_after)}, + ) correlation_id = str(uuid.uuid4()) provider = get_knowledge_provider() answer = provider.ask(body.question, correlation_id, body.language) diff --git a/backend/app/api/routers/mcp_integrations.py b/backend/app/api/routers/mcp_integrations.py index 1680bf6..c180c87 100644 --- a/backend/app/api/routers/mcp_integrations.py +++ b/backend/app/api/routers/mcp_integrations.py @@ -44,7 +44,7 @@ def get_correlation_id( def _audit_service_request( db: Session, *, - client_id: str, + reported_client_id: str, tool: str, status_label: str, correlation_id: str, @@ -53,11 +53,19 @@ def _audit_service_request( record_audit_event( db, actor_type="service", - actor_label=client_id, + # The shared service token authenticates the Hub, not the caller identity that + # the Hub reports in a header. Keep attribution authoritative and retain the + # reported value only as explicitly non-authenticated diagnostic metadata. + actor_label="itworx-mcp-hub", action="mcp_tool_request", entity_type="mcp_tool", correlation_id=uuid.UUID(correlation_id), - metadata={"tool": tool, "status": status_label, **(metadata or {})}, + metadata={ + "tool": tool, + "status": status_label, + "reported_client_id": reported_client_id, + **(metadata or {}), + }, ) db.commit() @@ -79,7 +87,7 @@ def operations_summary( _set_trace_headers(response, correlation_id, client.tenant) _audit_service_request( db, - client_id=client.client_id, + reported_client_id=client.reported_client_id, tool="fleet_ops_get_operations_summary", status_label="ok", correlation_id=correlation_id, @@ -103,7 +111,7 @@ def attention_vehicles( _set_trace_headers(response, correlation_id, client.tenant) _audit_service_request( db, - client_id=client.client_id, + reported_client_id=client.reported_client_id, tool="fleet_ops_list_attention_vehicles", status_label="ok", correlation_id=correlation_id, @@ -124,7 +132,7 @@ def vehicle_details( if vehicle is None: _audit_service_request( db, - client_id=client.client_id, + reported_client_id=client.reported_client_id, tool="fleet_ops_get_vehicle_details", status_label="not_found", correlation_id=correlation_id, @@ -146,7 +154,7 @@ def vehicle_details( _audit_service_request( db, - client_id=client.client_id, + reported_client_id=client.reported_client_id, tool="fleet_ops_get_vehicle_details", status_label="ok", correlation_id=correlation_id, @@ -182,7 +190,7 @@ def search_knowledge( response.headers["X-Sources-Returned"] = str(len(answer.sources)) _audit_service_request( db, - client_id=client.client_id, + reported_client_id=client.reported_client_id, tool="fleet_ops_search_knowledge", status_label=answer.evidence_state, correlation_id=correlation_id, diff --git a/backend/app/core/config.py b/backend/app/core/config.py index a0475f2..c9bc7df 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -23,6 +23,8 @@ class Settings(BaseSettings): ragcore_api_token: str = "" ragcore_space_id: str = "" ragcore_http_timeout_seconds: float = 5.0 + # Search fallback is only labelled grounded above this explicit retrieval threshold. + ragcore_min_search_score: float = 0.05 n8n_webhook_url: str = "http://n8n:5678/webhook/mobilityops-return" n8n_webhook_trigger_token: str = "replace-me-n8n-webhook-trigger-token" n8n_callback_token: str = "replace-me-n8n-callback-token" @@ -67,6 +69,8 @@ class Settings(BaseSettings): # Failed password logins per client IP before a temporary 429 (0 disables). login_max_failures: int = 10 login_failure_window_seconds: int = 900 + knowledge_max_requests: int = 30 + knowledge_rate_limit_window_seconds: int = 60 metrics_bearer_token: str = "" privacy_minimum_booking_retention_days: int = 30 privacy_audit_retention_days: int = 2555 @@ -86,13 +90,11 @@ INSECURE_DEFAULT_SECRETS: tuple[tuple[str, str], ...] = ( def insecure_default_secrets(settings: "Settings") -> list[str]: """Return the names of secret settings that still carry their placeholder value. - Only secrets that actually guard something in the given deployment are reported: - ``mcp_hub_service_token`` is irrelevant while MCP Hub registration is disabled. + MCP routes are always mounted, independently of the Hub reachability-status flag, so + their inbound token must always be non-placeholder in production. """ insecure: list[str] = [] for name, placeholder in INSECURE_DEFAULT_SECRETS: - if name == "mcp_hub_service_token" and not settings.mcp_hub_registration_enabled: - continue value = getattr(settings, name) if not value or value == placeholder or value.startswith("replace-me"): insecure.append(name) @@ -112,4 +114,8 @@ def get_settings() -> Settings: + ", ".join(insecure) + ". Set real values in the environment (see .env.example)." ) + if not settings.mobilityops_public_url.lower().startswith("https://"): + raise RuntimeError("Production MOBILITYOPS_PUBLIC_URL must use HTTPS.") + if not settings.session_cookie_secure: + raise RuntimeError("Production SESSION_COOKIE_SECURE must be true.") return settings diff --git a/backend/app/core/ratelimit.py b/backend/app/core/ratelimit.py index 58fa385..6e7a39d 100644 --- a/backend/app/core/ratelimit.py +++ b/backend/app/core/ratelimit.py @@ -47,3 +47,26 @@ class FailedAttemptLimiter: def reset(self, key: str) -> None: with self._lock: self._failures.pop(key, None) + + +class SlidingWindowLimiter: + """Thread-safe request limiter where every accepted request consumes capacity.""" + + def __init__(self, *, max_requests: int, window_seconds: float) -> None: + self.max_requests = max_requests + self.window_seconds = window_seconds + self._requests: dict[str, deque[float]] = {} + self._lock = threading.Lock() + + def consume(self, key: str) -> int: + """Record an accepted request, or return the seconds until capacity is available.""" + now = time.monotonic() + with self._lock: + bucket = self._requests.setdefault(key, deque()) + cutoff = now - self.window_seconds + while bucket and bucket[0] <= cutoff: + bucket.popleft() + if len(bucket) >= self.max_requests: + return max(1, int(bucket[0] + self.window_seconds - now + 0.999)) + bucket.append(now) + return 0 diff --git a/backend/app/models/data_quality.py b/backend/app/models/data_quality.py index 2b1fff8..74502a6 100644 --- a/backend/app/models/data_quality.py +++ b/backend/app/models/data_quality.py @@ -2,7 +2,7 @@ import uuid from datetime import datetime from typing import TYPE_CHECKING -from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, String +from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, String, text from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -37,6 +37,14 @@ class DataQualityIssue(UUIDPrimaryKeyMixin, TimestampMixin, Base): name="ck_data_quality_status", ), Index("ix_data_quality_work_queue", "status", "due_at", "severity"), + Index( + "uq_data_quality_one_open_condition", + "rule_type", + "entity_type", + "entity_id", + unique=True, + postgresql_where=text("status = 'open'"), + ), ) public_ref: Mapped[str] = mapped_column(String(20), unique=True, nullable=False) diff --git a/backend/app/services/data_quality.py b/backend/app/services/data_quality.py index 3b8c125..bc34a0b 100644 --- a/backend/app/services/data_quality.py +++ b/backend/app/services/data_quality.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta from difflib import SequenceMatcher -from sqlalchemy import select, update +from sqlalchemy import func, select, update from sqlalchemy.orm import Session from app.core.errors import AppError @@ -26,6 +26,7 @@ from app.services.vehicle_status import ( REQUIRED_CUSTOMER_FIELDS = ("first_name", "last_name") REQUIRED_VEHICLE_FIELDS = ("registration_number", "make", "model", "location") DUPLICATE_THRESHOLD = 70 +DATA_QUALITY_SCAN_LOCK_ID = 6_138_493_717_091_029_491 def issue_due_at(detected_at: datetime, severity: str) -> datetime: @@ -356,6 +357,9 @@ def _scan_odometer_regressions(db: Session, scan: ScanResult) -> None: def run_scan( db: Session, *, actor_label: str | None = None, actor_type: str = "user" ) -> ScanResult: + # The check-then-insert work below spans several rules. Serialise whole scans at the + # database boundary so API and n8n triggers cannot both observe an empty condition. + db.scalar(select(func.pg_advisory_xact_lock(DATA_QUALITY_SCAN_LOCK_ID))) scan = ScanResult() _scan_duplicate_customers(db, scan) _scan_missing_required_fields(db, scan) @@ -375,8 +379,13 @@ def run_scan( return scan -def _load_open_issue(db: Session, public_ref: str) -> DataQualityIssue: - issue = db.scalar(select(DataQualityIssue).where(DataQualityIssue.public_ref == public_ref)) +def _load_open_issue( + db: Session, public_ref: str, *, lock: bool = True +) -> DataQualityIssue: + statement = select(DataQualityIssue).where(DataQualityIssue.public_ref == public_ref) + if lock: + statement = statement.with_for_update() + issue = db.scalar(statement) if issue is None: raise AppError("ISSUE_NOT_FOUND", "Data quality issue not found.", status_code=404) if issue.status != "open": @@ -707,8 +716,10 @@ def resolve_booking_overlap( return issue -def _load_vehicle_status_conflict_issue(db: Session, public_ref: str) -> DataQualityIssue: - issue = _load_open_issue(db, public_ref) +def _load_vehicle_status_conflict_issue( + db: Session, public_ref: str, *, lock: bool = True +) -> DataQualityIssue: + issue = _load_open_issue(db, public_ref, lock=lock) if issue.rule_type != "vehicle_status_conflict": raise AppError( "NOT_A_STATUS_CONFLICT_ISSUE", @@ -724,7 +735,7 @@ def preview_vehicle_status_recommendation( """Non-mutating: computes and returns the recommendation only. Never resolves the issue, never writes an audit event, never queues automation -- safe to call as often as the UI needs (e.g. every time the panel is opened) with zero side effects.""" - issue = _load_vehicle_status_conflict_issue(db, public_ref) + issue = _load_vehicle_status_conflict_issue(db, public_ref, lock=False) vehicle = db.scalar(select(Vehicle).where(Vehicle.id == issue.entity_id)) if vehicle is None: raise AppError( diff --git a/backend/app/services/knowledge/ragcore.py b/backend/app/services/knowledge/ragcore.py index df07d5e..f1193c8 100644 --- a/backend/app/services/knowledge/ragcore.py +++ b/backend/app/services/knowledge/ragcore.py @@ -92,6 +92,17 @@ def _rank_sources_for_concepts(sources: list[SourceCard], concepts: set[str]) -> ) +def _retrieval_score(result: dict) -> float | None: + scores = result.get("scores") + if not isinstance(scores, dict): + return None + for name in ("rerank", "fused"): + value = scores.get(name) + if isinstance(value, int | float) and not isinstance(value, bool): + return float(value) + return None + + class RAGcoreKnowledgeProvider: """Adapter for the central RAGcore service, against its real `/v1/*` contract (see `docs/contracts/openapi.yaml` in the RAGcore checkout -- RAGcore is built and @@ -338,6 +349,8 @@ class RAGcoreKnowledgeProvider: try: results = body.get("results", []) + if not isinstance(results, list): + raise TypeError("results must be a list") sources = [ SourceCard( document_id=str(result["citation"]["document_id"]), @@ -351,10 +364,21 @@ class RAGcoreKnowledgeProvider: except (TypeError, KeyError, ValueError): return unavailable - sources = _deduplicate_sources(sources) + all_sources = _deduplicate_sources(sources) concepts = _question_concepts(question) - sources = _rank_sources_for_concepts(sources, concepts) - if not sources: + qualified_sources = [ + SourceCard( + document_id=str(result["citation"]["document_id"]), + title=result["citation"]["title"], + version=str(result["citation"]["document_version_id"]), + section=result["citation"].get("section") or "", + excerpt=result["citation"]["excerpt"], + ) + for result in results + if (_retrieval_score(result) or 0.0) >= self._settings.ragcore_min_search_score + ] + sources = _rank_sources_for_concepts(_deduplicate_sources(qualified_sources), concepts) + if not all_sources: return GroundedAnswer( answer="", evidence_state="insufficient", @@ -363,11 +387,19 @@ class RAGcoreKnowledgeProvider: correlation_id=correlation_id, ) - if not concepts: + damage_evidence = any( + term + in ( + f"{source.document_id} {source.title} {source.section} {source.excerpt}" + ).casefold() + for source in sources + for term in _DOMAIN_CONCEPTS["damage"] + ) + if not concepts or not sources or ("damage" in concepts and not damage_evidence): return GroundedAnswer( answer="", evidence_state="insufficient", - sources=sources, + sources=all_sources, provider=self.name, correlation_id=correlation_id, ) diff --git a/backend/requirements-prod.lock b/backend/requirements-prod.lock new file mode 100644 index 0000000..8b9a72e --- /dev/null +++ b/backend/requirements-prod.lock @@ -0,0 +1,173 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --constraint=requirements.lock --output-file=requirements-prod.lock pyproject.toml +# +alembic==1.18.5 + # via + # -c requirements.lock + # mobilityops-api (pyproject.toml) +annotated-doc==0.0.5 + # via + # -c requirements.lock + # fastapi +annotated-types==0.8.0 + # via + # -c requirements.lock + # pydantic +anyio==4.14.2 + # via + # -c requirements.lock + # httpx + # starlette + # watchfiles +authlib==1.7.2 + # via + # -c requirements.lock + # mobilityops-api (pyproject.toml) +certifi==2026.7.22 + # via + # -c requirements.lock + # httpcore + # httpx +cffi==2.1.1 + # via + # -c requirements.lock + # cryptography +click==8.4.2 + # via + # -c requirements.lock + # uvicorn +cryptography==50.0.0 + # via + # -c requirements.lock + # authlib + # joserfc +fastapi==0.141.1 + # via + # -c requirements.lock + # mobilityops-api (pyproject.toml) +greenlet==3.5.4 + # via + # -c requirements.lock + # sqlalchemy +h11==0.16.0 + # via + # -c requirements.lock + # httpcore + # uvicorn +httpcore==1.0.9 + # via + # -c requirements.lock + # httpx +httptools==0.8.0 + # via + # -c requirements.lock + # uvicorn +httpx==0.28.1 + # via + # -c requirements.lock + # mobilityops-api (pyproject.toml) +idna==3.18 + # via + # -c requirements.lock + # anyio + # httpx +itsdangerous==2.2.0 + # via + # -c requirements.lock + # mobilityops-api (pyproject.toml) +joserfc==1.7.4 + # via + # -c requirements.lock + # authlib +mako==1.3.12 + # via + # -c requirements.lock + # alembic +markupsafe==3.0.3 + # via + # -c requirements.lock + # mako +prometheus-client==0.26.0 + # via + # -c requirements.lock + # mobilityops-api (pyproject.toml) +psycopg[binary]==3.3.4 + # via + # -c requirements.lock + # mobilityops-api (pyproject.toml) +psycopg-binary==3.3.4 + # via + # -c requirements.lock + # psycopg +pycparser==3.0 + # via + # -c requirements.lock + # cffi +pydantic==2.13.4 + # via + # -c requirements.lock + # fastapi + # pydantic-settings +pydantic-core==2.46.4 + # via + # -c requirements.lock + # pydantic +pydantic-settings==2.14.2 + # via + # -c requirements.lock + # mobilityops-api (pyproject.toml) +python-dotenv==1.2.2 + # via + # -c requirements.lock + # pydantic-settings + # uvicorn +pyyaml==6.0.3 + # via + # -c requirements.lock + # uvicorn +sqlalchemy==2.0.51 + # via + # -c requirements.lock + # alembic + # mobilityops-api (pyproject.toml) +starlette==1.3.1 + # via + # -c requirements.lock + # fastapi +typing-extensions==4.16.0 + # via + # -c requirements.lock + # alembic + # anyio + # fastapi + # psycopg + # pydantic + # pydantic-core + # sqlalchemy + # starlette + # typing-inspection +typing-inspection==0.4.2 + # via + # -c requirements.lock + # fastapi + # pydantic + # pydantic-settings +uvicorn[standard]==0.52.1 + # via + # -c requirements.lock + # mobilityops-api (pyproject.toml) +uvloop==0.22.1 + # via + # -c requirements.lock + # uvicorn +watchfiles==1.2.0 + # via + # -c requirements.lock + # uvicorn +websockets==17.0.1 + # via + # -c requirements.lock + # uvicorn diff --git a/backend/tests/test_auth.py b/backend/tests/test_auth.py index 8f36a69..4a0436f 100644 --- a/backend/tests/test_auth.py +++ b/backend/tests/test_auth.py @@ -46,6 +46,29 @@ def test_oidc_callback_auto_provisions_and_logs_in(client, monkeypatch): assert session.json()["role"] == "rental_employee" +def test_oidc_callback_rejects_missing_email_verification_claim(client, monkeypatch): + import app.api.routers.auth as auth_router + + class FakeClient: + async def authorize_access_token(self, _request): + return {"userinfo": {"sub": "unverified-subject", "email": "new@example.test"}} + + class FakeOAuth: + def create_client(self, _name): + return FakeClient() + + monkeypatch.setattr(auth_router.settings, "oidc_enabled", True) + monkeypatch.setattr(auth_router.settings, "oidc_issuer_url", "https://id.example.test") + monkeypatch.setattr(auth_router.settings, "oidc_client_id", "client") + monkeypatch.setattr(auth_router.settings, "oidc_client_secret", "secret") + monkeypatch.setattr(auth_router.settings, "oidc_allowed_email_domains", "example.test") + monkeypatch.setattr(auth_router, "oauth", FakeOAuth()) + + response = client.get("/api/v1/auth/oidc/callback", follow_redirects=False) + assert response.status_code == 401 + assert client.get("/api/v1/auth/session").status_code == 401 + + def test_demo_login_grants_access(ops_client): response = ops_client.get("/api/v1/dashboard") assert response.status_code == 200 @@ -173,4 +196,8 @@ def test_demo_reset_preserves_integration_telemetry(ops_client, client): assert client.post("/api/v1/demo/login", json={"role": "operations_manager"}).status_code == 200 events = client.get("/api/v1/audit", params={"action": "mcp_tool_request"}).json() - assert any(event["actor_label"].endswith(":reset-probe") for event in events) + assert any( + event["actor_label"] == "itworx-mcp-hub" + and event["metadata"].get("reported_client_id", "").endswith(":reset-probe") + for event in events + ) diff --git a/backend/tests/test_hardening.py b/backend/tests/test_hardening.py index e8863ad..9b36c45 100644 --- a/backend/tests/test_hardening.py +++ b/backend/tests/test_hardening.py @@ -3,33 +3,37 @@ from __future__ import annotations import uuid +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime +from threading import Barrier import pytest -from sqlalchemy import delete, select +from sqlalchemy import delete, func, select from app.core.config import Settings, get_settings, insecure_default_secrets from app.core.db import SessionLocal +from app.core.errors import AppError from app.core.observability import UNMATCHED_ROUTE_LABEL -from app.core.ratelimit import FailedAttemptLimiter +from app.core.ratelimit import FailedAttemptLimiter, SlidingWindowLimiter from app.models.audit import AuditEvent from app.models.customer import Customer from app.models.data_quality import DataQualityIssue -from app.services.data_quality import run_scan +from app.schemas import CurrentUser +from app.services.data_quality import defer_issue, run_scan from tests.test_return import _activate_booking, _return_body -def test_insecure_defaults_are_detected_only_for_relevant_secrets(): +def test_insecure_defaults_include_mounted_mcp_boundary_even_when_status_check_is_disabled(): defaults = Settings(_env_file=None) assert "app_secret" in insecure_default_secrets(defaults) - assert "mcp_hub_service_token" not in insecure_default_secrets(defaults) + assert "mcp_hub_service_token" in insecure_default_secrets(defaults) hardened = Settings( _env_file=None, app_secret="x" * 32, n8n_callback_token="c" * 32, - mcp_hub_registration_enabled=True, + mcp_hub_service_token="m" * 32, ) - assert insecure_default_secrets(hardened) == ["mcp_hub_service_token"] + assert insecure_default_secrets(hardened) == [] def test_production_refuses_to_start_with_placeholder_secrets(monkeypatch): @@ -47,6 +51,37 @@ def test_production_refuses_to_start_with_placeholder_secrets(monkeypatch): assert get_settings().mobilityops_env == "test" +@pytest.mark.parametrize( + ("public_url", "secure_cookie", "message"), + [ + ("http://fleetops.example.test", "true", "must use HTTPS"), + ("https://fleetops.example.test", "false", "must be true"), + ], +) +def test_production_refuses_cleartext_or_insecure_session_cookie( + monkeypatch, public_url, secure_cookie, message +): + values = { + "MOBILITYOPS_ENV": "production", + "APP_SECRET": "a" * 32, + "N8N_CALLBACK_TOKEN": "c" * 32, + "MCP_HUB_SERVICE_TOKEN": "m" * 32, + "MOBILITYOPS_PUBLIC_URL": public_url, + "SESSION_COOKIE_SECURE": secure_cookie, + } + for name, value in values.items(): + monkeypatch.setenv(name, value) + get_settings.cache_clear() + try: + with pytest.raises(RuntimeError, match=message): + get_settings() + finally: + get_settings.cache_clear() + monkeypatch.undo() + get_settings.cache_clear() + assert get_settings().mobilityops_env == "test" + + def test_failed_attempt_limiter_blocks_after_threshold_and_resets(): limiter = FailedAttemptLimiter(max_failures=3, window_seconds=60) for _ in range(3): @@ -58,6 +93,29 @@ def test_failed_attempt_limiter_blocks_after_threshold_and_resets(): assert limiter.retry_after_seconds("1.2.3.4") == 0 +def test_sliding_window_limiter_counts_successes_and_isolates_keys(): + limiter = SlidingWindowLimiter(max_requests=2, window_seconds=60) + assert limiter.consume("session-a") == 0 + assert limiter.consume("session-a") == 0 + assert limiter.consume("session-a") > 0 + assert limiter.consume("session-b") == 0 + + +def test_login_client_key_uses_proxy_appended_peer_not_spoofed_leading_value(): + from starlette.requests import Request + + from app.api.routers.auth import _client_key + + request = Request( + { + "type": "http", + "headers": [(b"x-forwarded-for", b"203.0.113.99, 198.51.100.7")], + "client": ("172.20.0.3", 12345), + } + ) + assert _client_key(request) == "198.51.100.7" + + def test_audit_export_accepts_naive_datetimes(ops_client): response = ops_client.get( "/api/v1/audit/export.csv", @@ -194,3 +252,85 @@ def test_scan_does_not_flag_anonymised_customers_as_missing_fields(ops_client): db.execute(delete(AuditEvent).where(AuditEvent.entity_id == customer_id)) db.execute(delete(Customer).where(Customer.id == customer_id)) db.commit() + + +def test_concurrent_quality_scans_leave_only_one_open_issue_per_condition(): + barrier = Barrier(2) + + def scan() -> None: + with SessionLocal() as db: + barrier.wait() + run_scan(db) + + with ThreadPoolExecutor(max_workers=2) as executor: + list(executor.map(lambda _index: scan(), range(2))) + + with SessionLocal() as db: + duplicates = db.execute( + select( + DataQualityIssue.rule_type, + DataQualityIssue.entity_type, + DataQualityIssue.entity_id, + func.count(DataQualityIssue.id), + ) + .where(DataQualityIssue.status == "open") + .group_by( + DataQualityIssue.rule_type, + DataQualityIssue.entity_type, + DataQualityIssue.entity_id, + ) + .having(func.count(DataQualityIssue.id) > 1) + ).all() + assert duplicates == [] + + +def test_concurrent_issue_resolution_records_exactly_one_decision(): + issue_id = uuid.uuid4() + issue_ref = f"DQ-RACE-{uuid.uuid4().hex[:8].upper()}" + with SessionLocal() as db: + db.add( + DataQualityIssue( + id=issue_id, + public_ref=issue_ref, + rule_type="missing_required_field", + entity_type="customer", + entity_id=uuid.uuid4(), + severity="low", + status="open", + evidence_json={}, + proposed_action_json={}, + detected_at=datetime.now(UTC), + ) + ) + db.commit() + + actor = CurrentUser( + public_ref="USR-RACE", display_name="Race Manager", role="operations_manager" + ) + barrier = Barrier(2) + + def resolve() -> str: + with SessionLocal() as db: + barrier.wait() + try: + return defer_issue(db, issue_ref, actor).status + except AppError as exc: + return exc.code + + try: + with ThreadPoolExecutor(max_workers=2) as executor: + outcomes = list(executor.map(lambda _index: resolve(), range(2))) + assert sorted(outcomes) == ["ISSUE_NOT_OPEN", "deferred"] + with SessionLocal() as db: + decisions = db.scalar( + select(func.count(AuditEvent.id)).where( + AuditEvent.entity_id == issue_id, + AuditEvent.action == "data_quality_issue_deferred", + ) + ) + assert decisions == 1 + finally: + with SessionLocal() as db: + db.execute(delete(AuditEvent).where(AuditEvent.entity_id == issue_id)) + db.execute(delete(DataQualityIssue).where(DataQualityIssue.id == issue_id)) + db.commit() diff --git a/backend/tests/test_knowledge.py b/backend/tests/test_knowledge.py index 62f6507..3ebd7c1 100644 --- a/backend/tests/test_knowledge.py +++ b/backend/tests/test_knowledge.py @@ -7,6 +7,7 @@ import httpx from app.api.routers import knowledge as knowledge_router from app.core.config import get_settings +from app.core.ratelimit import SlidingWindowLimiter from app.services.knowledge import KnowledgeHealth from app.services.knowledge.demo import DemoKnowledgeProvider from app.services.knowledge.procedures import iter_procedure_documents @@ -140,6 +141,16 @@ def test_ask_question_requires_authentication(client): assert response.status_code == 401 +def test_knowledge_questions_are_bounded_per_session_and_client(ops_client, monkeypatch): + limiter = SlidingWindowLimiter(max_requests=1, window_seconds=60) + monkeypatch.setattr(knowledge_router, "_question_limiter", limiter) + body = {"question": "What is the vehicle return procedure?"} + assert ops_client.post("/api/v1/knowledge/questions", json=body).status_code == 200 + blocked = ops_client.post("/api/v1/knowledge/questions", json=body) + assert blocked.status_code == 429 + assert int(blocked.headers["retry-after"]) >= 1 + + def test_ask_question_is_audited_without_leaking_full_text(ops_client): ops_client.post( "/api/v1/knowledge/questions", @@ -695,5 +706,38 @@ def test_ragcore_search_fallback_prefers_damage_procedure(monkeypatch): ), ) answer = provider.ask("Wat moet ik doen bij schade?", "test-correlation-damage", "nl-BE") + assert answer.evidence_state == "insufficient" + assert answer.answer == "" + assert any(source.document_id == "damage-procedure" for source in answer.sources) + + +def test_ragcore_search_fallback_accepts_relevant_damage_evidence_above_threshold(monkeypatch): + provider = RAGcoreKnowledgeProvider() + monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") + search = _search_body() + search["results"].append( + { + "citation": { + "document_id": "damage-procedure", + "document_version_id": "version-2", + "title": "damage-procedure.md", + "section": "Damage", + "excerpt": "Record damage and keep the vehicle blocked.", + }, + "rank": 1, + "scores": {"fused": 0.5}, + } + ) + monkeypatch.setattr( + provider, + "_client", + lambda: _FakeClient( + post_responses={ + "/v1/answers": _FakeResponse(503, {}), + "/v1/search": _FakeResponse(200, search), + } + ), + ) + answer = provider.ask("Wat moet ik doen bij schade?", "strong-damage", "nl-BE") assert answer.evidence_state == "grounded" assert answer.sources[0].document_id == "damage-procedure" diff --git a/backend/tests/test_mcp_integrations.py b/backend/tests/test_mcp_integrations.py index 6429e58..a12fb49 100644 --- a/backend/tests/test_mcp_integrations.py +++ b/backend/tests/test_mcp_integrations.py @@ -96,6 +96,8 @@ def test_mcp_tool_requests_are_audited(client, ops_client): events = ops_client.get("/api/v1/audit", params={"action": "mcp_tool_request"}).json() assert len(events) >= 1 assert events[0]["actor_type"] == "service" + assert events[0]["actor_label"] == "itworx-mcp-hub" + assert events[0]["metadata"]["reported_client_id"].endswith(":probe-1") def test_search_knowledge_respects_requested_locale(client): @@ -135,7 +137,11 @@ def test_operations_summary_mints_correlation_id_when_none_supplied(client, ops_ ) assert response.status_code == 200 events = ops_client.get("/api/v1/audit", params={"action": "mcp_tool_request"}).json() - matching = [e for e in events if e["actor_label"].endswith(":no-correlation-probe")] + matching = [ + e + for e in events + if e["metadata"].get("reported_client_id", "").endswith(":no-correlation-probe") + ] assert len(matching) >= 1 assert matching[0]["correlation_id"] # a fresh UUID was minted, not left empty diff --git a/compose.observability.yaml b/compose.observability.yaml index 2551488..e8906f7 100644 --- a/compose.observability.yaml +++ b/compose.observability.yaml @@ -1,6 +1,6 @@ services: prometheus: - image: prom/prometheus:v3.7.1 + image: prom/prometheus:v3.7.1@sha256:ff7e389acbe064a4823212a500393d40a28a8f362e4b05cbf6742a9a3ef736b2 profiles: ["observability"] command: - --config.file=/etc/prometheus/prometheus.yml @@ -16,7 +16,7 @@ services: networks: [mobilityops] grafana: - image: grafana/grafana:12.2.0 + image: grafana/grafana:12.2.0@sha256:74144189b38447facf737dfd0f3906e42e0776212bf575dc3334c3609183adf7 profiles: ["observability"] environment: GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} diff --git a/compose.test.yaml b/compose.test.yaml index 622eaa7..0bf6b2e 100644 --- a/compose.test.yaml +++ b/compose.test.yaml @@ -8,6 +8,8 @@ services: test: ["CMD-SHELL", "pg_isready -U mobilityops_test -d mobilityops_test"] api: + build: + target: test environment: MOBILITYOPS_ENV: test MOBILITYOPS_DEMO_MODE: "true" diff --git a/compose.unraid.yaml b/compose.unraid.yaml index 5a19f61..beb67b5 100644 --- a/compose.unraid.yaml +++ b/compose.unraid.yaml @@ -36,7 +36,7 @@ services: retries: 20 backup: - image: postgres:16-alpine + image: postgres:16-alpine@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685 restart: unless-stopped environment: POSTGRES_DB: ${POSTGRES_DB:-mobilityops} diff --git a/compose.yaml b/compose.yaml index baf7f94..43b9423 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,6 @@ services: db: - image: postgres:16-alpine + image: postgres:16-alpine@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685 environment: POSTGRES_DB: ${POSTGRES_DB:-mobilityops} POSTGRES_USER: ${POSTGRES_USER:-mobilityops} @@ -91,7 +91,7 @@ services: n8n: # Explicit fallback only; production reuses the server's existing central n8n. - image: n8nio/n8n:2.33.7 + image: n8nio/n8n:2.33.7@sha256:3989d9b8ebb77b4ee8f604519eb73e44f4384bfaa689526e0104eed79a237d30 environment: TZ: ${TZ:-Europe/Brussels} GENERIC_TIMEZONE: ${TZ:-Europe/Brussels} diff --git a/deploy/unraid/README.md b/deploy/unraid/README.md index 195e90b..e387238 100644 --- a/deploy/unraid/README.md +++ b/deploy/unraid/README.md @@ -9,7 +9,7 @@ existing shared n8n remains available on its established port 5678. - Directory: `/mnt/user/appdata/mobilityops` - Compose project: `mobilityops` -- Web: `http://192.168.10.150:1236` (`1236` on the host to `80` in `web`) +- Public web: `https://fleetops.itworx.tech` (TLS reverse proxy to host port `1236`) - API and PostgreSQL: Compose network only - Shared n8n: `http://192.168.10.150:5678` (outside the MobilityOps Compose project) @@ -17,14 +17,15 @@ existing shared n8n remains available on its established port 5678. Create `.env` from `.env.example`, replace every placeholder secret, set `MOBILITYOPS_ENV=production`, set both public URLs to -`http://192.168.10.150:1236`, and retain `KNOWLEDGE_PROVIDER=demo` while RAGcore is not -available. Keep `MCP_HUB_REGISTRATION_ENABLED=false` until the central Hub is ready. +`https://fleetops.itworx.tech`, set `SESSION_COOKIE_SECURE=true`, and retain +`KNOWLEDGE_PROVIDER=demo` while RAGcore is not available. The internal `1236` listener is +an upstream for the TLS proxy, not a user-facing URL. ```bash cd /mnt/user/appdata/mobilityops ./deploy/unraid/configure-env.sh \ - http://192.168.10.150:1236 \ - http://192.168.10.150:5678/webhook/mobilityops-return + https://fleetops.itworx.tech \ + https://n8n.itworx.tech/webhook/mobilityops-return docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml up --build -d db api web backup docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml exec api \ python -m app.cli seed --reset @@ -36,7 +37,7 @@ workflow into the existing n8n container: ```bash ./deploy/unraid/setup-existing-n8n.sh \ n8n \ - http://192.168.10.150:1236/api/v1/integrations/n8n/return-callback + https://fleetops.itworx.tech/api/v1/integrations/n8n/return-callback ``` The callback token remains server-side and is never written to the repository. The diff --git a/deploy/unraid/configure-env.sh b/deploy/unraid/configure-env.sh index b72347d..f3fea5b 100755 --- a/deploy/unraid/configure-env.sh +++ b/deploy/unraid/configure-env.sh @@ -1,8 +1,13 @@ #!/bin/sh set -eu -public_url="${1:-http://192.168.10.150:1236}" -n8n_webhook_url="${2:-http://192.168.10.150:5678/webhook/mobilityops-return}" +public_url="${1:-https://fleetops.itworx.tech}" +n8n_webhook_url="${2:-https://n8n.itworx.tech/webhook/mobilityops-return}" + +case "$public_url" in + https://*) ;; + *) echo "Production public URL must use HTTPS" >&2; exit 1 ;; +esac if [ -e .env ]; then echo "Refusing to overwrite existing .env" >&2 @@ -26,6 +31,7 @@ sed -i \ -e "s|^DATABASE_URL=.*|DATABASE_URL=postgresql+psycopg://mobilityops:${db_password}@db:5432/mobilityops|" \ -e "s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=${db_password}|" \ -e "s|^APP_SECRET=.*|APP_SECRET=${app_secret}|" \ + -e "s|^SESSION_COOKIE_SECURE=.*|SESSION_COOKIE_SECURE=true|" \ -e "s|^N8N_WEBHOOK_URL=.*|N8N_WEBHOOK_URL=${n8n_webhook_url}|" \ -e "s|^N8N_ENCRYPTION_KEY=.*|N8N_ENCRYPTION_KEY=${n8n_key}|" \ -e "s|^N8N_BASIC_AUTH_PASSWORD=.*|N8N_BASIC_AUTH_PASSWORD=${n8n_password}|" \ diff --git a/deploy/unraid/setup-existing-n8n.sh b/deploy/unraid/setup-existing-n8n.sh index 3e4a373..b81fbd3 100755 --- a/deploy/unraid/setup-existing-n8n.sh +++ b/deploy/unraid/setup-existing-n8n.sh @@ -2,7 +2,7 @@ set -eu container_name="${1:-n8n}" -callback_url="${2:-http://192.168.10.150:1236/api/v1/integrations/n8n/return-callback}" +callback_url="${2:-https://fleetops.itworx.tech/api/v1/integrations/n8n/return-callback}" source_workflow="${3:-n8n/workflows/fleet-ops-vehicle-return.json}" if [ ! -f .env ]; then diff --git a/deploy/unraid/setup-scheduled-scan.sh b/deploy/unraid/setup-scheduled-scan.sh index ea3c2dd..fe30cb1 100755 --- a/deploy/unraid/setup-scheduled-scan.sh +++ b/deploy/unraid/setup-scheduled-scan.sh @@ -2,7 +2,7 @@ set -eu container_name="${1:-n8n}" -scan_url="${2:-http://192.168.10.150:1236/api/v1/integrations/n8n/scheduled-scan}" +scan_url="${2:-https://fleetops.itworx.tech/api/v1/integrations/n8n/scheduled-scan}" source_workflow="${3:-n8n/workflows/fleet-ops-data-quality-scan.json}" if [ ! -f .env ]; then diff --git a/docs/08-return-workflow.md b/docs/08-return-workflow.md index 67f44a2..0aa3c6f 100644 --- a/docs/08-return-workflow.md +++ b/docs/08-return-workflow.md @@ -27,7 +27,8 @@ created, and next-booking risk. `register_vehicle_return` (below) calls the same 1. Authorize Rental Employee or Operations Manager. 2. Lock booking and vehicle rows. 3. Reject cancelled/already-returned booking unless idempotency replay matches. -4. Validate required fields and submitted reading against booking start reading. +4. Validate required fields; compare the submitted reading only with the vehicle's + canonical odometer so a lower reading is recorded as evidence instead of rejected. 5. Create a return inspection. 6. Set booking to returned and store submitted end reading. 7. If submitted reading >= canonical odometer, update canonical odometer. diff --git a/docs/11-n8n-integration.md b/docs/11-n8n-integration.md index edeb13d..dca0ebd 100644 --- a/docs/11-n8n-integration.md +++ b/docs/11-n8n-integration.md @@ -20,10 +20,6 @@ The canonical, live-validated definition is `n8n/workflows/fleet-ops-vehicle-ret ## Second live workflow: scheduled quality scan -RAGcore is not connected in this environment, so the originally sketched "knowledge sync" -workflow below remains deferred (see "Deferred: knowledge sync"). The second implemented -workflow does not depend on RAGcore or MCP Hub, so it is not blocked by them. - Input: hourly schedule trigger, or a manual trigger for on-demand testing. Steps: @@ -45,16 +41,15 @@ workflow (see `deploy/unraid/setup-scheduled-scan.sh` and `docs/17-runbook.md`). active on the live instance; a fresh import ships inactive until credentials are wired up and it is deliberately published. -## RAGcore procedure sync (in progress) +## RAGcore procedure sync -RAGcore is now reachable in this environment; a live inspection of its real contract is -recorded in `docs/live-ai-integration/n8n-current-state.md`. Workflow 3, "Fleet Ops — -RAGcore Procedure Sync", is being built against that real contract (not the sketch -originally in this section) — see `n8n/workflows/MANIFEST.md` for current status. +Workflow 3, "Fleet Ops — RAGcore Procedure Sync", is published and active against the +live RAGcore contract. See `n8n/workflows/MANIFEST.md` for its bounded upload contract, +credentials and live validation evidence. -## Workflow error handler (in progress) +## Workflow error handler -Workflow 4, "Fleet Ops — Workflow Error Handler", is a central technical workflow attached +Workflow 4, "Fleet Ops — Workflow Error Handler", is active and attached to workflows 1-3 via n8n's per-workflow "Error Workflow" setting, reporting bounded, secret-free failure details to Fleet Ops. See `n8n/workflows/MANIFEST.md` for status. diff --git a/docs/17-runbook.md b/docs/17-runbook.md index cb65405..15c6ee6 100644 --- a/docs/17-runbook.md +++ b/docs/17-runbook.md @@ -71,8 +71,8 @@ On startup the API creates the first active Operations Manager only when no user email exists. The sign-in page then accepts email/password instead of exposing demo roles; demo reset, the guided tour and the synthetic-data badge are hidden. -Use a long unique `APP_SECRET`, set `SESSION_COOKIE_SECURE=true` once the public endpoint -uses HTTPS, and keep `INITIAL_ADMIN_PASSWORD` out of Git and logs. Existing sessions are +Use a long unique `APP_SECRET`; production requires an HTTPS public endpoint and +`SESSION_COOKIE_SECURE=true`. Keep `INITIAL_ADMIN_PASSWORD` out of Git and logs. Existing sessions are revalidated against the current user record on every request, so deactivating an account invalidates its next request. @@ -168,17 +168,17 @@ deliberately published with a real service token. ### Existing shared n8n on the Unraid review server -The Unraid deployment uses the existing n8n at `http://192.168.10.150:5678`; it does not +The Unraid deployment uses the existing n8n at `https://n8n.itworx.tech`; it does not start MobilityOps's bundled n8n service. `compose.unraid.yaml` places that fallback behind the opt-in `bundled-n8n` profile. Configure the API target and publish the workflow with: ```bash sed -i \ - 's|^N8N_WEBHOOK_URL=.*|N8N_WEBHOOK_URL=http://192.168.10.150:5678/webhook/mobilityops-return|' \ + 's|^N8N_WEBHOOK_URL=.*|N8N_WEBHOOK_URL=https://n8n.itworx.tech/webhook/mobilityops-return|' \ .env ./deploy/unraid/setup-existing-n8n.sh \ n8n \ - http://192.168.10.150:1236/api/v1/integrations/n8n/return-callback + https://fleetops.itworx.tech/api/v1/integrations/n8n/return-callback docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml up -d db api web ``` @@ -215,7 +215,7 @@ Publish the scheduled quality-scan workflow the same way: ```bash ./deploy/unraid/setup-scheduled-scan.sh \ n8n \ - http://192.168.10.150:1236/api/v1/integrations/n8n/scheduled-scan + https://fleetops.itworx.tech/api/v1/integrations/n8n/scheduled-scan ``` ## Required operational checks diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 5c6adc4..b0e8016 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,4 +1,4 @@ -FROM node:22-alpine AS build +FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32 AS build WORKDIR /app COPY package.json package-lock.json tsconfig.json vite.config.ts index.html ./ RUN npm ci --no-audit --no-fund @@ -10,7 +10,7 @@ ARG VITE_API_BASE_URL="" ENV VITE_API_BASE_URL=$VITE_API_BASE_URL RUN npm run build -FROM nginx:1.27-alpine +FROM nginx:1.27-alpine@sha256:65645c7bb6a0661892a8b03b89d0743208a18dd2f3f17a54ef4b76fb8e2f2a10 COPY nginx.conf /etc/nginx/conf.d/default.conf COPY --from=build /app/dist /usr/share/nginx/html EXPOSE 80 diff --git a/frontend/nginx.conf b/frontend/nginx.conf index b13b5a0..daf9acf 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -24,21 +24,21 @@ server { limit_req zone=demo_login burst=100 nodelay; proxy_pass http://api:8000; proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; } location = /api/v1/demo/reset { proxy_pass http://api:8000; proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; } location /api/ { proxy_pass http://api:8000; proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; } diff --git a/n8n/workflows/MANIFEST.md b/n8n/workflows/MANIFEST.md index eaa1e27..5534d12 100644 --- a/n8n/workflows/MANIFEST.md +++ b/n8n/workflows/MANIFEST.md @@ -19,7 +19,7 @@ credential values are never embedded; nodes reference named n8n credentials inst | Error Workflow | `Fleet Ops — Workflow Error Handler` (wired) | | Timeouts / bounded retries | `Record follow-up` and heartbeat HTTP nodes: 15s timeout, retry on fail (3 tries, 1000ms wait) | | Execution telemetry | Successful runs POST execution ID and status to `/api/v1/integrations/n8n/heartbeat`; failed runs are registered by the Error Workflow. | -| Checksum (sha256) | `17b8dd5b8e7ab5d0c0856a40f4199a401ad38f63895f611a096a80c9740f1e99` (heartbeat-enabled definition, 2026-08-10) | +| Checksum (sha256) | `19f233b6de90a3474010d5495a53c75e969fb5de591e84dd3abcf9635304c1f1` (HTTPS callback definition, 2026-08-21) | ## 2. Fleet Ops — Scheduled Data Quality Scan @@ -35,7 +35,7 @@ credential values are never embedded; nodes reference named n8n credentials inst | Error Workflow | `Fleet Ops — Workflow Error Handler` (wired) | | Timeouts / bounded retries | Scan and heartbeat HTTP nodes: 15s timeout, retry on fail (3 tries, 1000ms wait) | | Execution telemetry | Every successful scheduled/manual run posts an idempotent heartbeat with its n8n execution ID. | -| Checksum (sha256) | `f0bda8b0fa99d1a2403e970e00bd1f896cd4d10eaa626147fcca577a7ad7cb46` | +| Checksum (sha256) | `057bfe27ea7e2b497c57dec2a5967cad2960170a5c17535021f4480e63a6073d` | ## 3. Fleet Ops — RAGcore Procedure Sync @@ -56,8 +56,9 @@ trailing `}}` (e.g. `={{ $json.execution_id }} }}` instead of `={{ $json.executi }}`), which would have sent malformed values on every real run. Fixed via `n8n import:workflow` (the safe CLI path — not the REST API, which caused a prior wipe incident in this environment) against the same live workflow ID, keeping it inactive; -re-exported and verified the fix applied with no other change (still 6 real nodes, -`active: false`). Also found and fixed a second gap: `settings.errorWorkflow` was unset +re-exported and verified the fix applied with no other change (7 real nodes; the cleaned +repository export intentionally remains `active: false` so importing cannot start work +before credentials are wired). Also found and fixed a second gap: `settings.errorWorkflow` was unset (workflows 1-2 wire `"errorWorkflow": "Xppn2rAEqUuyiCJF"`, workflow 3 did not) — wired it the same way via the same CLI import path, re-verified. @@ -72,7 +73,7 @@ the same way via the same CLI import path, re-verified. | Active status (as of 2026-08-05) | **Active / Published** | | Error Workflow | `Fleet Ops — Workflow Error Handler` (wired) | | Execution telemetry | Successful syncs report both the bounded sync result and the canonical workflow heartbeat. | -| Checksum (sha256) | `8644d9faec4d17a43ba797e300b91fff976881c9b259e457fadd5a7cfaf39c6f` | +| Checksum (sha256) | `4afd46b6ef57b7e0a4705611732506d7fcc2a37567266847991b85050e1d37c3` | ## 4. Fleet Ops — Workflow Error Handler @@ -93,7 +94,7 @@ Ops, which registers an audit event idempotently keyed on `execution_id`. | Active status (as of 2026-08-04) | Active / Published | | Error Workflow (on itself) | `- No Workflow -` (deliberately unset — prevents a recursive error loop) | | Execution telemetry | A successfully handled failure posts its own execution heartbeat after registering the target failure. | -| Checksum (sha256) | `1e50cf8b679b9e9f9b5cf4a1594bda459597ade40f44e8a68f540890f9c4b61d` | +| Checksum (sha256) | `57e6c1206b1be4522d88a5983f53b2ddf94f8d6e55c1b7be8c397f75e46412da` | Validated this round: mock-data run (Error Trigger pinned to a realistic payload) produced a real `200 {"status":"registered", ...}` from the live Fleet Ops server; diff --git a/n8n/workflows/fleet-ops-data-quality-scan.json b/n8n/workflows/fleet-ops-data-quality-scan.json index 8e6b59f..73ad965 100644 --- a/n8n/workflows/fleet-ops-data-quality-scan.json +++ b/n8n/workflows/fleet-ops-data-quality-scan.json @@ -30,7 +30,7 @@ { "parameters": { "method": "POST", - "url": "http://192.168.10.150:1236/api/v1/integrations/n8n/scheduled-scan", + "url": "https://fleetops.itworx.tech/api/v1/integrations/n8n/scheduled-scan", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendHeaders": true, @@ -68,7 +68,7 @@ { "parameters": { "method": "POST", - "url": "http://192.168.10.150:1236/api/v1/integrations/n8n/heartbeat", + "url": "https://fleetops.itworx.tech/api/v1/integrations/n8n/heartbeat", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendBody": true, diff --git a/n8n/workflows/fleet-ops-error-handler.json b/n8n/workflows/fleet-ops-error-handler.json index e5fb9b3..b4c6872 100644 --- a/n8n/workflows/fleet-ops-error-handler.json +++ b/n8n/workflows/fleet-ops-error-handler.json @@ -24,7 +24,7 @@ { "parameters": { "method": "POST", - "url": "http://192.168.10.150:1236/api/v1/integrations/n8n/workflow-error", + "url": "https://fleetops.itworx.tech/api/v1/integrations/n8n/workflow-error", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendBody": true, @@ -58,7 +58,7 @@ { "parameters": { "method": "POST", - "url": "http://192.168.10.150:1236/api/v1/integrations/n8n/heartbeat", + "url": "https://fleetops.itworx.tech/api/v1/integrations/n8n/heartbeat", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendBody": true, diff --git a/n8n/workflows/fleet-ops-ragcore-procedure-sync.json b/n8n/workflows/fleet-ops-ragcore-procedure-sync.json index 85e1656..29145e5 100644 --- a/n8n/workflows/fleet-ops-ragcore-procedure-sync.json +++ b/n8n/workflows/fleet-ops-ragcore-procedure-sync.json @@ -32,7 +32,7 @@ }, { "parameters": { - "url": "http://192.168.10.150:1236/api/v1/integrations/n8n/procedures", + "url": "https://fleetops.itworx.tech/api/v1/integrations/n8n/procedures", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "options": { @@ -57,7 +57,7 @@ { "parameters": { "method": "POST", - "url": "http://192.168.10.150:1236/api/v1/integrations/n8n/heartbeat", + "url": "https://fleetops.itworx.tech/api/v1/integrations/n8n/heartbeat", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendBody": true, @@ -164,7 +164,7 @@ { "parameters": { "method": "POST", - "url": "http://192.168.10.150:1236/api/v1/integrations/n8n/procedures-sync-result", + "url": "https://fleetops.itworx.tech/api/v1/integrations/n8n/procedures-sync-result", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendBody": true, diff --git a/n8n/workflows/fleet-ops-vehicle-return.json b/n8n/workflows/fleet-ops-vehicle-return.json index ab3411d..5a4560a 100644 --- a/n8n/workflows/fleet-ops-vehicle-return.json +++ b/n8n/workflows/fleet-ops-vehicle-return.json @@ -35,7 +35,7 @@ { "parameters": { "method": "POST", - "url": "http://192.168.10.150:1236/api/v1/integrations/n8n/return-callback", + "url": "https://fleetops.itworx.tech/api/v1/integrations/n8n/return-callback", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendHeaders": true, @@ -72,7 +72,7 @@ { "parameters": { "method": "POST", - "url": "http://192.168.10.150:1236/api/v1/integrations/n8n/heartbeat", + "url": "https://fleetops.itworx.tech/api/v1/integrations/n8n/heartbeat", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendBody": true,