from __future__ import annotations from collections.abc import Iterator import pytest from fastapi.testclient import TestClient from sqlalchemy import create_engine from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session from sqlalchemy.pool import StaticPool from modelforge_api.api.routes.observability import get_service from modelforge_api.main import app from modelforge_api.persistence.models import Base from modelforge_api.services.observability import ObservabilityService from modelforge_api.settings import Settings, get_settings @pytest.fixture def session() -> Iterator[Session]: engine = create_engine( "sqlite+pysqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool, ) Base.metadata.create_all(engine) with Session(engine) as value: yield value @pytest.fixture(autouse=True) def clean_overrides() -> Iterator[None]: yield app.dependency_overrides.clear() def test_operations_routes_require_the_operator_credential(session: Session) -> None: subject = ObservabilityService(session, Settings(observability_monitor_enabled=False)) subject.ensure_defaults() app.dependency_overrides[get_service] = lambda: subject app.dependency_overrides[get_settings] = lambda: Settings( operator_api_key="operator-secret", observability_monitor_enabled=False ) client = TestClient(app) assert client.get("/api/v1/admin/operations/overview").status_code == 401 assert client.get( "/api/v1/admin/operations/overview", headers={"Authorization": "Bearer capability-client-secret"}, ).status_code == 401 response = client.get( "/api/v1/admin/operations/overview", headers={"X-ModelForge-Admin-Token": "operator-secret"}, ) assert response.status_code == 200 assert response.json()["status"] == "HEALTHY" def test_prometheus_endpoint_is_bounded_and_contains_no_request_payload( session: Session, ) -> None: subject = ObservabilityService(session, Settings(observability_monitor_enabled=False)) subject.ensure_defaults() app.dependency_overrides[get_service] = lambda: subject app.dependency_overrides[get_settings] = lambda: Settings( operator_api_key="operator-secret", observability_monitor_enabled=False ) response = TestClient(app).get( "/metrics", headers={"X-ModelForge-Admin-Token": "operator-secret"} ) assert response.status_code == 200 assert response.headers["content-type"].startswith("text/plain") assert "modelforge_api_requests_total" in response.text assert "modelforge_api_request_duration_seconds_bucket" in response.text for forbidden in ("request_id=", "input_sha256=", "prompt=", "payload="): assert forbidden not in response.text def test_process_metrics_remain_available_when_observability_storage_fails() -> None: class BrokenSession: rolled_back = False def rollback(self) -> None: self.rolled_back = True class BrokenService: session = BrokenSession() def prometheus(self) -> str: raise SQLAlchemyError("simulated observability database outage") broken = BrokenService() app.dependency_overrides[get_service] = lambda: broken app.dependency_overrides[get_settings] = lambda: Settings( operator_api_key="operator-secret", observability_monitor_enabled=False ) response = TestClient(app).get( "/metrics", headers={"X-ModelForge-Admin-Token": "operator-secret"} ) assert response.status_code == 200 assert broken.session.rolled_back is True assert "modelforge_observability_degraded 1" in response.text