105 lines
3.6 KiB
Python
105 lines
3.6 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Iterator
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.exc import SQLAlchemyError
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from modelforge_api.api.routes.observability import get_service
|
|
from modelforge_api.main import app
|
|
from modelforge_api.persistence.models import Base
|
|
from modelforge_api.services.observability import ObservabilityService
|
|
from modelforge_api.settings import Settings, get_settings
|
|
|
|
|
|
@pytest.fixture
|
|
def session() -> Iterator[Session]:
|
|
engine = create_engine(
|
|
"sqlite+pysqlite:///:memory:",
|
|
connect_args={"check_same_thread": False},
|
|
poolclass=StaticPool,
|
|
)
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as value:
|
|
yield value
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def clean_overrides() -> Iterator[None]:
|
|
yield
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
def test_operations_routes_require_the_operator_credential(session: Session) -> None:
|
|
subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
|
|
subject.ensure_defaults()
|
|
app.dependency_overrides[get_service] = lambda: subject
|
|
app.dependency_overrides[get_settings] = lambda: Settings(
|
|
operator_api_key="operator-secret", observability_monitor_enabled=False
|
|
)
|
|
client = TestClient(app)
|
|
|
|
assert client.get("/api/v1/admin/operations/overview").status_code == 401
|
|
assert client.get(
|
|
"/api/v1/admin/operations/overview",
|
|
headers={"Authorization": "Bearer capability-client-secret"},
|
|
).status_code == 401
|
|
response = client.get(
|
|
"/api/v1/admin/operations/overview",
|
|
headers={"X-ModelForge-Admin-Token": "operator-secret"},
|
|
)
|
|
assert response.status_code == 200
|
|
assert response.json()["status"] == "HEALTHY"
|
|
|
|
|
|
def test_prometheus_endpoint_is_bounded_and_contains_no_request_payload(
|
|
session: Session,
|
|
) -> None:
|
|
subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
|
|
subject.ensure_defaults()
|
|
app.dependency_overrides[get_service] = lambda: subject
|
|
app.dependency_overrides[get_settings] = lambda: Settings(
|
|
operator_api_key="operator-secret", observability_monitor_enabled=False
|
|
)
|
|
response = TestClient(app).get(
|
|
"/metrics", headers={"X-ModelForge-Admin-Token": "operator-secret"}
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.headers["content-type"].startswith("text/plain")
|
|
assert "modelforge_api_requests_total" in response.text
|
|
assert "modelforge_api_request_duration_seconds_bucket" in response.text
|
|
for forbidden in ("request_id=", "input_sha256=", "prompt=", "payload="):
|
|
assert forbidden not in response.text
|
|
|
|
|
|
def test_process_metrics_remain_available_when_observability_storage_fails() -> None:
|
|
class BrokenSession:
|
|
rolled_back = False
|
|
|
|
def rollback(self) -> None:
|
|
self.rolled_back = True
|
|
|
|
class BrokenService:
|
|
session = BrokenSession()
|
|
|
|
def prometheus(self) -> str:
|
|
raise SQLAlchemyError("simulated observability database outage")
|
|
|
|
broken = BrokenService()
|
|
app.dependency_overrides[get_service] = lambda: broken
|
|
app.dependency_overrides[get_settings] = lambda: Settings(
|
|
operator_api_key="operator-secret", observability_monitor_enabled=False
|
|
)
|
|
response = TestClient(app).get(
|
|
"/metrics", headers={"X-ModelForge-Admin-Token": "operator-secret"}
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert broken.session.rolled_back is True
|
|
assert "modelforge_observability_degraded 1" in response.text
|