71 lines
2.7 KiB
Python
71 lines
2.7 KiB
Python
import base64
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from modelforge_api.domain.capability_evaluation import CapabilityEvaluationSuiteCreate
|
|
from modelforge_api.domain.serving import (
|
|
OCRInvokeRequest,
|
|
SpeechTranscriptionInvokeRequest,
|
|
VisionEmbeddingInvokeRequest,
|
|
)
|
|
from modelforge_api.services.manifest_registry import ManifestRegistry
|
|
|
|
|
|
def test_m9_capability_estate_is_typed_and_complete() -> None:
|
|
manifests = {item.capability: item for item in ManifestRegistry().capabilities()}
|
|
assert manifests["document.ocr"].estate.category == "DOCUMENT"
|
|
assert manifests["vision.embedding"].estate.evaluation_type == "visual-retrieval"
|
|
assert manifests["speech.transcription"].estate.payload_limits.max_duration_seconds == 120
|
|
assert manifests["speech.transcription"].privacy.allow_persistence is False
|
|
assert manifests["speech.transcription"].privacy.allow_network_egress is False
|
|
|
|
|
|
def test_modality_requests_reject_ambiguous_or_unbounded_content() -> None:
|
|
encoded = base64.b64encode(b"safe-local-fixture").decode()
|
|
OCRInvokeRequest(content_base64=encoded, media_type="image/png")
|
|
SpeechTranscriptionInvokeRequest(audio_base64=encoded, media_type="audio/wav")
|
|
VisionEmbeddingInvokeRequest(items=[{"text": "local visual query"}])
|
|
with pytest.raises(ValidationError, match="exactly one"):
|
|
VisionEmbeddingInvokeRequest(
|
|
items=[
|
|
{
|
|
"text": "ambiguous",
|
|
"image_base64": encoded,
|
|
"media_type": "image/png",
|
|
}
|
|
]
|
|
)
|
|
with pytest.raises(ValidationError, match="canonical base64"):
|
|
OCRInvokeRequest(content_base64="not-base64", media_type="image/png")
|
|
|
|
|
|
def test_evaluation_metrics_are_capability_type_specific() -> None:
|
|
common = {
|
|
"capability": "document.ocr",
|
|
"key": "ocr-local",
|
|
"evaluation_type": "ocr",
|
|
"revision": "v1",
|
|
"dataset_revision": "generated-v1",
|
|
"cases": [
|
|
{
|
|
"key": "nl-clean",
|
|
"fixture_ref": "generated/nl-clean.png",
|
|
"fixture_sha256": "a" * 64,
|
|
"ground_truth": {"text": "veilige lokale tekst"},
|
|
}
|
|
],
|
|
}
|
|
suite = CapabilityEvaluationSuiteCreate(
|
|
**common,
|
|
metrics=[{"name": "cer", "direction": "lower_is_better", "unit": "ratio"}],
|
|
)
|
|
assert suite.evaluation_type == "ocr"
|
|
with pytest.raises(ValidationError, match="invalid for ocr"):
|
|
CapabilityEvaluationSuiteCreate(
|
|
**common,
|
|
metrics=[
|
|
{"name": "recall_at_10", "direction": "higher_is_better", "unit": "ratio"}
|
|
],
|
|
)
|