Initial public ModelForge release

2026-09-01 21:30:16 +02:00
commit 7082ab955a
490 changed files with 104252 additions and 0 deletions
@@ -0,0 +1,70 @@
+import base64
+
+import pytest
+from pydantic import ValidationError
+
+from modelforge_api.domain.capability_evaluation import CapabilityEvaluationSuiteCreate
+from modelforge_api.domain.serving import (
+    OCRInvokeRequest,
+    SpeechTranscriptionInvokeRequest,
+    VisionEmbeddingInvokeRequest,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+
+
+def test_m9_capability_estate_is_typed_and_complete() -> None:
+    manifests = {item.capability: item for item in ManifestRegistry().capabilities()}
+    assert manifests["document.ocr"].estate.category == "DOCUMENT"
+    assert manifests["vision.embedding"].estate.evaluation_type == "visual-retrieval"
+    assert manifests["speech.transcription"].estate.payload_limits.max_duration_seconds == 120
+    assert manifests["speech.transcription"].privacy.allow_persistence is False
+    assert manifests["speech.transcription"].privacy.allow_network_egress is False
+
+
+def test_modality_requests_reject_ambiguous_or_unbounded_content() -> None:
+    encoded = base64.b64encode(b"safe-local-fixture").decode()
+    OCRInvokeRequest(content_base64=encoded, media_type="image/png")
+    SpeechTranscriptionInvokeRequest(audio_base64=encoded, media_type="audio/wav")
+    VisionEmbeddingInvokeRequest(items=[{"text": "local visual query"}])
+    with pytest.raises(ValidationError, match="exactly one"):
+        VisionEmbeddingInvokeRequest(
+            items=[
+                {
+                    "text": "ambiguous",
+                    "image_base64": encoded,
+                    "media_type": "image/png",
+                }
+            ]
+        )
+    with pytest.raises(ValidationError, match="canonical base64"):
+        OCRInvokeRequest(content_base64="not-base64", media_type="image/png")
+
+
+def test_evaluation_metrics_are_capability_type_specific() -> None:
+    common = {
+        "capability": "document.ocr",
+        "key": "ocr-local",
+        "evaluation_type": "ocr",
+        "revision": "v1",
+        "dataset_revision": "generated-v1",
+        "cases": [
+            {
+                "key": "nl-clean",
+                "fixture_ref": "generated/nl-clean.png",
+                "fixture_sha256": "a" * 64,
+                "ground_truth": {"text": "veilige lokale tekst"},
+            }
+        ],
+    }
+    suite = CapabilityEvaluationSuiteCreate(
+        **common,
+        metrics=[{"name": "cer", "direction": "lower_is_better", "unit": "ratio"}],
+    )
+    assert suite.evaluation_type == "ocr"
+    with pytest.raises(ValidationError, match="invalid for ocr"):
+        CapabilityEvaluationSuiteCreate(
+            **common,
+            metrics=[
+                {"name": "recall_at_10", "direction": "higher_is_better", "unit": "ratio"}
+            ],
+        )