Initial public ModelForge release
This commit is contained in:
Jens
@@ -0,0 +1,662 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.domain.evaluation import (
|
||||
AdvisorRecommendationCreate,
|
||||
CandidatePoolEntry,
|
||||
EmbeddingMigrationCreate,
|
||||
EvaluationCaseCreate,
|
||||
EvaluationCaseResultCreate,
|
||||
EvaluationComparisonCreate,
|
||||
EvaluationRevisionCreate,
|
||||
EvaluationRunComplete,
|
||||
EvaluationRunCreate,
|
||||
EvaluationSuiteCreate,
|
||||
MigrationUpdate,
|
||||
ModelComparisonCandidateCreate,
|
||||
ModelComparisonCreate,
|
||||
RankedResult,
|
||||
RerankingCaseResultCreate,
|
||||
RerankingRunComplete,
|
||||
RerankingRunCreate,
|
||||
RetrievalCandidatePoolCreate,
|
||||
RetrievalPipelineIdentityCreate,
|
||||
)
|
||||
from modelforge_api.persistence.models import (
|
||||
Base,
|
||||
Capability,
|
||||
CapabilityContract,
|
||||
CapabilityDeployment,
|
||||
EmbeddingMigration,
|
||||
EmbeddingSpace,
|
||||
EvaluationCase,
|
||||
Project,
|
||||
RetrievalCandidatePool,
|
||||
)
|
||||
from modelforge_api.services.evaluation import EvaluationError, EvaluationService
|
||||
|
||||
|
||||
def _session() -> Session:
|
||||
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
return Session(engine)
|
||||
|
||||
|
||||
def _suite(service: EvaluationService, project_id: UUID, relevant: UUID):
|
||||
return service.create_suite(
|
||||
EvaluationSuiteCreate(
|
||||
project_id=project_id,
|
||||
key="examplerag-retrieval",
|
||||
name="ExampleRAG retrieval",
|
||||
description="Reviewed local retrieval cases.",
|
||||
revision=EvaluationRevisionCreate(
|
||||
revision="v1",
|
||||
dataset_revision="corpus-v1",
|
||||
retrieval_settings={"hybrid": True},
|
||||
cases=[
|
||||
EvaluationCaseCreate(
|
||||
case_key="known-answer",
|
||||
query="Where is the answer?",
|
||||
relevant_chunk_ids=[relevant],
|
||||
label_provenance={"source": "manual-review"},
|
||||
critical=True,
|
||||
review_status="approved",
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _run(
|
||||
service: EvaluationService,
|
||||
project_id: UUID,
|
||||
revision_id: UUID,
|
||||
case_id: UUID,
|
||||
relevant: UUID,
|
||||
*,
|
||||
target: str,
|
||||
rank: int,
|
||||
):
|
||||
run = service.create_run(
|
||||
EvaluationRunCreate(
|
||||
project_id=project_id,
|
||||
suite_revision_id=revision_id,
|
||||
target_kind=target,
|
||||
target_index_ref=f"index-{target}",
|
||||
embedding_space_ref=f"space-{target}",
|
||||
corpus_revision="corpus-v1",
|
||||
retrieval_config_digest="a" * 64,
|
||||
environment_fingerprint={"engine": "test-v1"},
|
||||
)
|
||||
)
|
||||
ranked = [RankedResult(chunk_id=uuid4(), score=1.0 - index / 100) for index in range(rank - 1)]
|
||||
ranked.append(RankedResult(chunk_id=relevant, score=0.5))
|
||||
return service.complete_run(
|
||||
run.id,
|
||||
EvaluationRunComplete(
|
||||
results=[
|
||||
EvaluationCaseResultCreate(case_id=case_id, ranked_results=ranked, latency_ms=10.0)
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_have_hand_calculable_values_and_per_case_evidence() -> None:
|
||||
with _session() as session:
|
||||
project = Project(key="examplerag", name="ExampleRAG", description="test")
|
||||
session.add(project)
|
||||
session.commit()
|
||||
relevant = uuid4()
|
||||
service = EvaluationService(session)
|
||||
suite = _suite(service, project.id, relevant)
|
||||
case_id = session.query(EvaluationCase.id).scalar()
|
||||
assert case_id is not None
|
||||
run = _run(
|
||||
service,
|
||||
project.id,
|
||||
suite.latest_revision_id,
|
||||
case_id,
|
||||
relevant,
|
||||
target="current",
|
||||
rank=2,
|
||||
)
|
||||
assert run.aggregate_metrics == pytest.approx(
|
||||
{
|
||||
"recall_at_5": 1.0,
|
||||
"recall_at_10": 1.0,
|
||||
"mrr": 0.5,
|
||||
"ndcg_at_10": 1 / 1.584962500721156,
|
||||
}
|
||||
)
|
||||
evidence = service.case_results(run.id)[0]
|
||||
assert evidence.first_relevant_rank == 2
|
||||
assert evidence.critical is True
|
||||
assert len(evidence.ranked_results) == 2
|
||||
|
||||
|
||||
def test_comparison_blocks_critical_regression() -> None:
|
||||
with _session() as session:
|
||||
project = Project(key="examplerag", name="ExampleRAG", description="test")
|
||||
session.add(project)
|
||||
session.commit()
|
||||
relevant = uuid4()
|
||||
service = EvaluationService(session)
|
||||
suite = _suite(service, project.id, relevant)
|
||||
case_id = session.query(EvaluationCase.id).scalar()
|
||||
assert case_id is not None
|
||||
baseline = _run(
|
||||
service,
|
||||
project.id,
|
||||
suite.latest_revision_id,
|
||||
case_id,
|
||||
relevant,
|
||||
target="current",
|
||||
rank=1,
|
||||
)
|
||||
candidate = _run(
|
||||
service,
|
||||
project.id,
|
||||
suite.latest_revision_id,
|
||||
case_id,
|
||||
relevant,
|
||||
target="shadow",
|
||||
rank=5,
|
||||
)
|
||||
comparison = service.compare(
|
||||
EvaluationComparisonCreate(
|
||||
baseline_run_id=baseline.id,
|
||||
candidate_run_id=candidate.id,
|
||||
)
|
||||
)
|
||||
assert comparison.comparability == "comparable"
|
||||
assert comparison.regressed_cases == 1
|
||||
assert comparison.critical_regressions == 1
|
||||
assert comparison.promotion_eligibility == "not_eligible"
|
||||
assert comparison.eligibility_evidence["promotion_performed"] is False
|
||||
|
||||
|
||||
def test_suite_revision_and_cases_are_immutable() -> None:
|
||||
with _session() as session:
|
||||
project = Project(key="examplerag", name="ExampleRAG", description="test")
|
||||
session.add(project)
|
||||
session.commit()
|
||||
service = EvaluationService(session)
|
||||
suite = _suite(service, project.id, uuid4())
|
||||
from modelforge_api.persistence.models import EvaluationSuiteRevision
|
||||
|
||||
revision = session.get(EvaluationSuiteRevision, suite.latest_revision_id)
|
||||
assert revision is not None
|
||||
revision.top_k = 20
|
||||
with pytest.raises(ValueError, match="immutable approved fields"):
|
||||
session.commit()
|
||||
|
||||
|
||||
def _pool_entries(
|
||||
relevant: UUID, relevant_rank: int = 5, count: int = 40
|
||||
) -> list[CandidatePoolEntry]:
|
||||
identifiers = [uuid4() for _ in range(count)]
|
||||
identifiers[relevant_rank - 1] = relevant
|
||||
return [
|
||||
CandidatePoolEntry(
|
||||
id=str(identifier),
|
||||
document_id=str(uuid4()),
|
||||
score=1.0 - index / 100,
|
||||
content_sha256=hashlib.sha256(f"chunk-{identifier}".encode()).hexdigest(),
|
||||
)
|
||||
for index, identifier in enumerate(identifiers)
|
||||
]
|
||||
|
||||
|
||||
def test_frozen_candidate_pool_is_fingerprinted_content_free_and_immutable() -> None:
|
||||
with _session() as session:
|
||||
project = Project(key="examplerag", name="ExampleRAG", description="test")
|
||||
session.add(project)
|
||||
session.commit()
|
||||
relevant = uuid4()
|
||||
service = EvaluationService(session)
|
||||
suite = _suite(service, project.id, relevant)
|
||||
case = session.query(EvaluationCase).one()
|
||||
request = RetrievalCandidatePoolCreate(
|
||||
project_id=project.id,
|
||||
suite_revision_id=suite.latest_revision_id,
|
||||
evaluation_case_id=case.id,
|
||||
source_embedding_space="space-current",
|
||||
source_index_ref="rag_dense_nomic_v1",
|
||||
corpus_revision="corpus-v1",
|
||||
retrieval_config_digest="a" * 64,
|
||||
ordered_candidates=_pool_entries(relevant),
|
||||
)
|
||||
first = service.create_candidate_pool(request)
|
||||
repeated = service.create_candidate_pool(request)
|
||||
assert first.id == repeated.id
|
||||
assert first.candidate_count == 40
|
||||
assert all("text" not in candidate for candidate in first.ordered_candidates)
|
||||
stored = session.get(RetrievalCandidatePool, first.id)
|
||||
assert stored is not None
|
||||
stored.source_index_ref = "mutated"
|
||||
with pytest.raises(ValueError, match="immutable approved fields"):
|
||||
session.commit()
|
||||
|
||||
|
||||
def test_pipeline_identity_and_reranking_run_enforce_same_frozen_pool() -> None:
|
||||
with _session() as session:
|
||||
project = Project(key="examplerag", name="ExampleRAG", description="test")
|
||||
capability = Capability(key="rag.reranking", description="Reranking")
|
||||
session.add_all([project, capability])
|
||||
session.flush()
|
||||
contract = CapabilityContract(
|
||||
capability_id=capability.id,
|
||||
version=1,
|
||||
input_schema={},
|
||||
output_schema={},
|
||||
contract={},
|
||||
upgrade_class="behavioral",
|
||||
)
|
||||
session.add(contract)
|
||||
session.flush()
|
||||
deployment = CapabilityDeployment(
|
||||
capability_contract_id=contract.id,
|
||||
deployment_candidate_id=uuid4(),
|
||||
embedding_space_id=None,
|
||||
artifact_set_id=uuid4(),
|
||||
runtime_profile_id=uuid4(),
|
||||
compute_node_id=uuid4(),
|
||||
accelerator_id=uuid4(),
|
||||
channel="experiment",
|
||||
status="candidate",
|
||||
production=False,
|
||||
fallback_policy={"allowed": False},
|
||||
config_fingerprint="b" * 64,
|
||||
provenance={"test": True},
|
||||
rollback_policy={},
|
||||
)
|
||||
session.add(deployment)
|
||||
session.commit()
|
||||
relevant = uuid4()
|
||||
service = EvaluationService(session)
|
||||
suite = _suite(service, project.id, relevant)
|
||||
case = session.query(EvaluationCase).one()
|
||||
pool = service.create_candidate_pool(
|
||||
RetrievalCandidatePoolCreate(
|
||||
project_id=project.id,
|
||||
suite_revision_id=suite.latest_revision_id,
|
||||
evaluation_case_id=case.id,
|
||||
source_embedding_space="space-current",
|
||||
source_index_ref="rag_dense_nomic_v1",
|
||||
corpus_revision="corpus-v1",
|
||||
retrieval_config_digest="a" * 64,
|
||||
ordered_candidates=_pool_entries(relevant, count=39),
|
||||
)
|
||||
)
|
||||
assert pool.candidate_count == 39
|
||||
common = {
|
||||
"project_id": project.id,
|
||||
"embedding_space_ref": "space-current",
|
||||
"sparse_config_digest": "c" * 64,
|
||||
"fusion_config_digest": "d" * 64,
|
||||
"configuration": {"rrf": "v1"},
|
||||
}
|
||||
control = service.create_pipeline_identity(RetrievalPipelineIdentityCreate(**common))
|
||||
candidate = service.create_pipeline_identity(
|
||||
RetrievalPipelineIdentityCreate(
|
||||
**common,
|
||||
reranker_deployment_id=deployment.id,
|
||||
reranker_config_digest="e" * 64,
|
||||
)
|
||||
)
|
||||
assert control.identity_digest != candidate.identity_digest
|
||||
assert candidate.migration_class == "behavioral"
|
||||
run = service.create_reranking_run(
|
||||
RerankingRunCreate(
|
||||
project_id=project.id,
|
||||
suite_revision_id=suite.latest_revision_id,
|
||||
pipeline_identity_id=candidate.id,
|
||||
control_pipeline_identity_id=control.id,
|
||||
candidate_pool_ids=[pool.id],
|
||||
corpus_revision="corpus-v1",
|
||||
environment_fingerprint={"runtime": "offline"},
|
||||
)
|
||||
)
|
||||
ranked_ids = [UUID(str(item["id"])) for item in pool.ordered_candidates[:10]]
|
||||
ranked_ids.remove(relevant)
|
||||
ranked_ids.insert(0, relevant)
|
||||
completed = service.complete_reranking_run(
|
||||
run.id,
|
||||
RerankingRunComplete(
|
||||
results=[
|
||||
RerankingCaseResultCreate(
|
||||
case_id=case.id,
|
||||
candidate_pool_id=pool.id,
|
||||
ranked_results=[
|
||||
RankedResult(chunk_id=identifier, score=1.0 - rank / 100)
|
||||
for rank, identifier in enumerate(ranked_ids)
|
||||
],
|
||||
retrieval_latency_ms=90,
|
||||
rerank_latency_ms=25,
|
||||
total_latency_ms=115,
|
||||
)
|
||||
]
|
||||
),
|
||||
)
|
||||
assert completed.aggregate_metrics["mrr"] == 1.0
|
||||
assert completed.latency_metrics["rerank_p95_ms"] == 25
|
||||
evidence = service.reranking_case_results(run.id)[0]
|
||||
assert evidence.first_relevant_rank == 1
|
||||
assert evidence.critical is True
|
||||
|
||||
control_metrics = _run(
|
||||
service,
|
||||
project.id,
|
||||
suite.latest_revision_id,
|
||||
case.id,
|
||||
relevant,
|
||||
target="current",
|
||||
rank=2,
|
||||
)
|
||||
matrix = service.create_model_comparison(
|
||||
ModelComparisonCreate(
|
||||
project_id=project.id,
|
||||
capability_contract_id=contract.id,
|
||||
suite_revision_id=suite.latest_revision_id,
|
||||
current_run_id=control_metrics.id,
|
||||
title="Current retrieval plus Qwen reranker",
|
||||
candidates=[
|
||||
ModelComparisonCandidateCreate(
|
||||
candidate_key="nomic-plus-qwen-reranker",
|
||||
label="nomic + Qwen3-Reranker-0.6B",
|
||||
status="evaluated",
|
||||
candidate_kind="retrieval_pipeline",
|
||||
pipeline_identity_id=candidate.id,
|
||||
latency_ms={"baseline_p95": 90.0, "p95": 115.0},
|
||||
resource_evidence={
|
||||
"runtime_compatible": True,
|
||||
"gpu_fit": True,
|
||||
"measured": True,
|
||||
"stale": False,
|
||||
"resident_vram_bytes": 2_000,
|
||||
},
|
||||
security_state={
|
||||
"supply_chain_status": "verified",
|
||||
"license_status": "approved",
|
||||
},
|
||||
provenance={"evidence_level": "A"},
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
candidate_evidence = matrix.candidates[0]
|
||||
assert candidate_evidence["reranking_run_id"] == str(run.id)
|
||||
assert candidate_evidence["metric_deltas"]["mrr"] == pytest.approx(0.8)
|
||||
recommendation = service.recommend(
|
||||
matrix.id,
|
||||
AdvisorRecommendationCreate(candidate_key="nomic-plus-qwen-reranker"),
|
||||
)
|
||||
assert recommendation.target_kind == "retrieval_pipeline"
|
||||
assert recommendation.current_pipeline_identity_id == control.id
|
||||
assert recommendation.candidate_pipeline_identity_id == candidate.id
|
||||
assert recommendation.candidate_deployment_id == deployment.id
|
||||
assert recommendation.verdict == "KEEP_CURRENT_EMBEDDING_ADD_RERANKER_CANDIDATE"
|
||||
assert recommendation.migration_impact == {
|
||||
"class": "behavioral",
|
||||
"requires_reindex": False,
|
||||
}
|
||||
|
||||
with pytest.raises(EvaluationError, match="pool provenance"):
|
||||
service.create_reranking_run(
|
||||
RerankingRunCreate(
|
||||
project_id=project.id,
|
||||
suite_revision_id=suite.latest_revision_id,
|
||||
pipeline_identity_id=candidate.id,
|
||||
control_pipeline_identity_id=control.id,
|
||||
candidate_pool_ids=[pool.id],
|
||||
corpus_revision="different-corpus",
|
||||
environment_fingerprint={},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_migration_requires_preflight_and_complete_validation() -> None:
|
||||
with _session() as session:
|
||||
project = Project(key="examplerag", name="ExampleRAG", description="test")
|
||||
session.add(project)
|
||||
session.flush()
|
||||
migration = EmbeddingMigration(
|
||||
project_id=project.id,
|
||||
source_embedding_space="legacy-observed",
|
||||
target_embedding_space_id=uuid4(),
|
||||
source_index_ref="current",
|
||||
target_index_ref="shadow",
|
||||
corpus_revision=hashlib.sha256(b"corpus").hexdigest(),
|
||||
total_chunks=10,
|
||||
completed_chunks=0,
|
||||
failed_chunks=0,
|
||||
retried_chunks=0,
|
||||
batch_size=2,
|
||||
concurrency=1,
|
||||
priority="background",
|
||||
preflight_evidence={},
|
||||
progress_evidence={},
|
||||
validation_evidence={},
|
||||
operational_metrics={},
|
||||
)
|
||||
session.add(migration)
|
||||
session.commit()
|
||||
service = EvaluationService(session)
|
||||
# PLANNED -> PREFLIGHT is valid; BACKFILLING then requires passed evidence.
|
||||
item = service.update_migration(
|
||||
migration.id,
|
||||
MigrationUpdate(
|
||||
status="preflight", completed_chunks=0, failed_chunks=0, retried_chunks=0
|
||||
),
|
||||
)
|
||||
assert item.status == "preflight"
|
||||
with pytest.raises(EvaluationError, match="preflight"):
|
||||
service.update_migration(
|
||||
migration.id,
|
||||
MigrationUpdate(
|
||||
status="backfilling", completed_chunks=0, failed_chunks=0, retried_chunks=0
|
||||
),
|
||||
)
|
||||
item = service.update_migration(
|
||||
migration.id,
|
||||
MigrationUpdate(
|
||||
status="backfilling",
|
||||
completed_chunks=0,
|
||||
failed_chunks=0,
|
||||
retried_chunks=0,
|
||||
preflight_evidence={"passed": True},
|
||||
),
|
||||
)
|
||||
assert item.priority == "background"
|
||||
item = service.cancel_migration(migration.id)
|
||||
assert item.cancel_requested is True
|
||||
|
||||
|
||||
def test_suite_cases_are_addressable_and_migration_start_is_explicit() -> None:
|
||||
with _session() as session:
|
||||
project = Project(key="examplerag", name="ExampleRAG", description="test")
|
||||
space = EmbeddingSpace(
|
||||
capability_contract_id=uuid4(),
|
||||
artifact_set_id=uuid4(),
|
||||
runtime_profile_id=uuid4(),
|
||||
identity_digest="4" * 64,
|
||||
dimension=1024,
|
||||
normalized=True,
|
||||
identity_facts={"distance_metric": "cosine"},
|
||||
)
|
||||
session.add_all([project, space])
|
||||
session.commit()
|
||||
service = EvaluationService(session)
|
||||
suite = _suite(service, project.id, uuid4())
|
||||
case = session.query(EvaluationCase).one()
|
||||
|
||||
definitions = service.suite_cases(suite.id, suite.latest_revision_id)
|
||||
assert [item.id for item in definitions] == [case.id]
|
||||
assert definitions[0].label_provenance == {"source": "manual-review"}
|
||||
|
||||
migration = service.create_migration(
|
||||
project.id,
|
||||
EmbeddingMigrationCreate(
|
||||
source_embedding_space="legacy-observed",
|
||||
target_embedding_space_id=space.id,
|
||||
source_index_ref="current",
|
||||
target_index_ref="shadow-explicit-start",
|
||||
corpus_revision="corpus-v1",
|
||||
total_chunks=1,
|
||||
),
|
||||
)
|
||||
started = service.start_migration(migration.id)
|
||||
assert started.status == "preflight"
|
||||
assert started.priority == "background"
|
||||
|
||||
|
||||
def test_model_comparison_and_advisor_hard_block_critical_regression() -> None:
|
||||
with _session() as session:
|
||||
project = Project(key="examplerag", name="ExampleRAG", description="test")
|
||||
capability = Capability(key="rag.embedding", description="Dense embedding")
|
||||
session.add_all([project, capability])
|
||||
session.flush()
|
||||
contract = CapabilityContract(
|
||||
capability_id=capability.id,
|
||||
version=1,
|
||||
input_schema={},
|
||||
output_schema={},
|
||||
contract={},
|
||||
upgrade_class="behavioral",
|
||||
)
|
||||
session.add(contract)
|
||||
session.commit()
|
||||
relevant = uuid4()
|
||||
service = EvaluationService(session)
|
||||
suite = _suite(service, project.id, relevant)
|
||||
case_id = session.query(EvaluationCase.id).scalar()
|
||||
assert case_id is not None
|
||||
baseline = _run(
|
||||
service,
|
||||
project.id,
|
||||
suite.latest_revision_id,
|
||||
case_id,
|
||||
relevant,
|
||||
target="current",
|
||||
rank=1,
|
||||
)
|
||||
candidate = _run(
|
||||
service,
|
||||
project.id,
|
||||
suite.latest_revision_id,
|
||||
case_id,
|
||||
relevant,
|
||||
target="shadow",
|
||||
rank=2,
|
||||
)
|
||||
|
||||
matrix = service.create_model_comparison(
|
||||
ModelComparisonCreate(
|
||||
project_id=project.id,
|
||||
capability_contract_id=contract.id,
|
||||
suite_revision_id=suite.latest_revision_id,
|
||||
current_run_id=baseline.id,
|
||||
title="ExampleRAG embedding candidates",
|
||||
candidates=[
|
||||
ModelComparisonCandidateCreate(
|
||||
candidate_key="qwen-retrieval",
|
||||
label="Qwen retrieval-aware",
|
||||
status="evaluated",
|
||||
evaluation_run_id=candidate.id,
|
||||
embedding_space="space-qwen",
|
||||
latency_ms={"baseline_p95": 100.0, "p95": 150.0},
|
||||
resource_evidence={
|
||||
"runtime_compatible": True,
|
||||
"gpu_fit": True,
|
||||
"measured": True,
|
||||
"stale": False,
|
||||
"resident_vram_bytes": 1_000,
|
||||
},
|
||||
migration_impact={"class": "requires_reindex", "chunks": 597},
|
||||
security_state={
|
||||
"supply_chain_status": "verified",
|
||||
"license_status": "approved",
|
||||
},
|
||||
provenance={"evidence_level": "A"},
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
assert matrix.candidates[0]["quality_metrics"]["recall_at_10"] == 1.0
|
||||
assert matrix.candidates[0]["critical_regressions"] == 1
|
||||
|
||||
recommendation = service.recommend(
|
||||
matrix.id, AdvisorRecommendationCreate(candidate_key="qwen-retrieval")
|
||||
)
|
||||
assert recommendation.verdict == "KEEP_CURRENT"
|
||||
assert recommendation.confidence == "HIGH"
|
||||
assert recommendation.evidence_level == "A"
|
||||
assert "critical_regression" in recommendation.blockers
|
||||
assert recommendation.migration_impact["class"] == "requires_reindex"
|
||||
assert recommendation.policy_snapshot["critical_regression_hard_block"] is True
|
||||
|
||||
|
||||
def test_advisor_requires_more_evidence_for_capacity_blocked_candidate() -> None:
|
||||
with _session() as session:
|
||||
project = Project(key="examplerag", name="ExampleRAG", description="test")
|
||||
capability = Capability(key="rag.embedding", description="Dense embedding")
|
||||
session.add_all([project, capability])
|
||||
session.flush()
|
||||
contract = CapabilityContract(
|
||||
capability_id=capability.id,
|
||||
version=1,
|
||||
input_schema={},
|
||||
output_schema={},
|
||||
contract={},
|
||||
upgrade_class="behavioral",
|
||||
)
|
||||
session.add(contract)
|
||||
session.commit()
|
||||
service = EvaluationService(session)
|
||||
suite = _suite(service, project.id, uuid4())
|
||||
case = session.query(EvaluationCase).one()
|
||||
baseline = _run(
|
||||
service,
|
||||
project.id,
|
||||
suite.latest_revision_id,
|
||||
case.id,
|
||||
UUID(case.relevant_chunk_ids[0]),
|
||||
target="current",
|
||||
rank=1,
|
||||
)
|
||||
matrix = service.create_model_comparison(
|
||||
ModelComparisonCreate(
|
||||
project_id=project.id,
|
||||
capability_contract_id=contract.id,
|
||||
suite_revision_id=suite.latest_revision_id,
|
||||
current_run_id=baseline.id,
|
||||
title="Blocked candidates",
|
||||
candidates=[
|
||||
ModelComparisonCandidateCreate(
|
||||
candidate_key="qwen-4b",
|
||||
label="Qwen3-Embedding-4B",
|
||||
status="blocked",
|
||||
blockers=["insufficient_vram"],
|
||||
provenance={"evidence_level": "B"},
|
||||
resource_evidence={"runtime_compatible": True, "gpu_fit": False},
|
||||
security_state={
|
||||
"supply_chain_status": "planned",
|
||||
"license_status": "compatible",
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
recommendation = service.recommend(
|
||||
matrix.id, AdvisorRecommendationCreate(candidate_key="qwen-4b")
|
||||
)
|
||||
assert recommendation.verdict == "REQUIRES_MORE_EVIDENCE"
|
||||
assert recommendation.confidence == "LOW"
|
||||
assert "local_project_evaluation_missing" in recommendation.blockers
|
||||
Reference in New Issue
Block a user