geointel/backend/tests/test_accuracy_phase4_evaluator_hardening.py

from __future__ import annotations

import copy
import hashlib
import json
import math
import sys
from pathlib import Path

import pytest


ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
if str(SCRIPTS) not in sys.path:
    sys.path.insert(0, str(SCRIPTS))

from accuracy_phase4_evaluator import (  # noqa: E402
    TASKS,
    EXPECTED_PROTECTED_POLICY,
    canonical_hash,
    count_metrics,
    detection_ap,
    evaluate_cases,
    evaluate_object_detection,
    evaluate_footprint_segmentation,
    evaluate_raster_classification,
    evaluate_terrain,
    evaluate_validation,
    evaluate_vector_comparison,
    subgroup_report,
    task_inventory,
)


METADATA = {
    "region": "flanders",
    "municipality": "Mol",
    "urbanity": "urban",
    "object_size": "medium",
    "source": "synthetic-source",
    "sensor": "synthetic-sensor",
    "resolution_m": 0.25,
    "context": "dense_urban",
    "season": "summer",
    "date": "2026-01-01",
    "vegetation": "partial",
    "occlusion": "none",
    "difficulty": "normal",
}


def lineage(sample_id: str) -> dict:
    return {
        "reference": {
            "source_id": f"synthetic:{sample_id}:reference",
            "source_version": "1",
            "derivation": "hand_authored_contract_fixture",
        },
        "prediction": {
            "source_id": f"synthetic:{sample_id}:prediction",
            "source_version": "1",
            "derivation": "hand_authored_fixed_output",
        },
    }


def detection_case(sample_id: str = "det-1") -> dict:
    return {
        "sample_id": sample_id,
        "task": "object_detection",
        "split": "test",
        "metadata": copy.deepcopy(METADATA),
        "config": {"confidence_threshold": 0.5, "match_iou": 0.5},
        "lineage": lineage(sample_id),
        "classes": ["building", "tank"],
        "references": [{"id": "r-building", "class": "building", "bbox": [0, 0, 4, 4]}],
        "predictions": [
            {
                "id": "p-building",
                "class": "building",
                "bbox": [0, 0, 4, 4],
                "confidence": 0.8,
            },
            {
                "id": "p-filtered",
                "class": "building",
                "bbox": [10, 10, 12, 12],
                "confidence": 0.2,
            },
        ],
    }


def raster_case(sample_id: str = "raster-1") -> dict:
    reference_side = {
        "crs": "EPSG:31370",
        "transform": [1, 0, 100000, 0, -1, 200000],
        "shape": [2, 2],
        "nodata": -9999,
        "mask": [[True, True], [True, True]],
    }
    return {
        "sample_id": sample_id,
        "task": "raster_classification",
        "split": "test",
        "metadata": copy.deepcopy(METADATA),
        "config": {},
        "lineage": lineage(sample_id),
        "classes": [0, 1],
        "references": [[0, 1], [1, 0]],
        "predictions": [[0, 1], [1, 0]],
        "raster_context": {
            "reference": reference_side,
            "prediction": copy.deepcopy(reference_side),
        },
    }


def polygon_case(task: str = "vector_comparison") -> dict:
    sample_id = f"{task}-1"
    config = {"match_iou": 0.5}
    if task == "footprint_segmentation":
        config["boundary_tolerance_m"] = 1.0
    polygon = [
        [100000, 200000],
        [100010, 200000],
        [100010, 200010],
        [100000, 200010],
        [100000, 200000],
    ]
    return {
        "sample_id": sample_id,
        "task": task,
        "split": "test",
        "metadata": copy.deepcopy(METADATA),
        "config": config,
        "lineage": lineage(sample_id),
        "classes": ["building"],
        "spatial_context": {
            "crs": "EPSG:31370",
            "coordinate_units": "m",
            "metric": True,
        },
        "references": [{"id": "reference", "class": "building", "polygon": polygon}],
        "predictions": [{"id": "prediction", "class": "building", "polygon": polygon}],
    }


def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> None:
    case = detection_case()
    portfolio = {
        "schema_version": 2,
        "portfolio_kind": "synthetic_contract",
        "portfolio_id": "synthetic-hardening-test",
        "portfolio_lineage": {
            "origin": "repository_fixture",
            "source_path": "synthetic.json",
            "version": "1",
        },
        "split_roles": ["test"],
        "selection_policy": "Fixed before evaluation; no selection.",
        "claim_boundary": "Synthetic evaluator test; not product accuracy.",
        "protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
        "cases": [case],
    }
    path = tmp_path / "portfolio.json"
    path.write_text(
        json.dumps(portfolio, ensure_ascii=False, indent=2) + "\n",
        encoding="utf-8",
    )

    report = evaluate_cases(path, {case["sample_id"]})
    raw = report["results"][0]["raw"]

    assert raw["references"] == case["references"]
    assert raw["predictions_pre_filter"] == case["predictions"]
    assert raw["predictions_post_filter"] == case["predictions"][:1]
    assert raw["config"] == case["config"]
    assert raw["split"] == "test"
    assert raw["input_lineage"] == case["lineage"]
    assert raw["portfolio_lineage"]["declared"] == portfolio["portfolio_lineage"]
    assert raw["hashes"]["case_input_canonical_json_sha256"] == canonical_hash(case)
    assert raw["hashes"]["references_canonical_json_sha256"] == canonical_hash(
        case["references"]
    )
    assert (
        report["portfolio_file_sha256"] == hashlib.sha256(path.read_bytes()).hexdigest()
    )
    assert report["portfolio_canonical_json_sha256"] == canonical_hash(portfolio)
    assert report["results_canonical_json_sha256"] == canonical_hash(report["results"])
    high_threshold = next(
        row
        for row in report["results"][0]["metrics"]["coverage_risk"]
        if row["threshold"] == 0.9
    )
    assert high_threshold["retained_prediction_coverage"] == 0.0
    assert high_threshold["reference_coverage"] == 0.0
    assert high_threshold["false_negative_count"] == 1
    assert high_threshold["risk"] == 1.0

    challenge_exposed = copy.deepcopy(portfolio)
    challenge_exposed["challenge_labels"] = []
    path.write_text(json.dumps(challenge_exposed, ensure_ascii=False), encoding="utf-8")
    with pytest.raises(ValueError, match="Challenge cases and labels"):
        evaluate_cases(path, {case["sample_id"]})


def test_portfolio_schema_policy_metadata_and_lineage_are_strict(
    tmp_path: Path,
) -> None:
    case = detection_case("strict-contract")
    portfolio = {
        "schema_version": 2,
        "portfolio_kind": "synthetic_contract",
        "portfolio_id": "synthetic-strict-contract",
        "portfolio_lineage": {
            "origin": "repository_fixture",
            "source_path": "synthetic.json",
            "version": "1",
        },
        "split_roles": ["test"],
        "selection_policy": "Fixed before evaluation; no selection.",
        "claim_boundary": "Synthetic evaluator test; not product accuracy.",
        "protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
        "cases": [case],
    }
    path = tmp_path / "strict.json"

    def evaluate(value: dict) -> dict:
        path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
        return evaluate_cases(path, {case["sample_id"]})

    assert evaluate(portfolio)["case_count"] == 1

    for invalid_version in (1, True, "2"):
        invalid = copy.deepcopy(portfolio)
        invalid["schema_version"] = invalid_version
        with pytest.raises(
            ValueError, match="schema_version must be exactly integer 2"
        ):
            evaluate(invalid)

    invalid_policy = copy.deepcopy(portfolio)
    invalid_policy["protected_policy"]["test_feedback_allowed"] = True
    with pytest.raises(ValueError, match="protected_policy must exactly equal"):
        evaluate(invalid_policy)

    invalid_metadata = copy.deepcopy(portfolio)
    invalid_metadata["cases"][0]["metadata"]["source"] = "unknown"
    with pytest.raises(ValueError, match="metadata.source must be a meaningful"):
        evaluate(invalid_metadata)

    invalid_resolution = copy.deepcopy(portfolio)
    invalid_resolution["cases"][0]["metadata"]["resolution_m"] = 0
    with pytest.raises(ValueError, match="metadata.resolution_m must be positive"):
        evaluate(invalid_resolution)

    invalid_lineage = copy.deepcopy(portfolio)
    del invalid_lineage["cases"][0]["lineage"]["prediction"]["derivation"]
    with pytest.raises(ValueError, match="lineage.prediction missing"):
        evaluate(invalid_lineage)


def test_portfolio_kind_separates_synthetic_and_governed_product_claims(
    tmp_path: Path,
) -> None:
    fixture_path = (
        ROOT / "fixtures" / "accuracy" / "p4" / "protected-baseline-cases.json"
    )
    synthetic = json.loads(fixture_path.read_text(encoding="utf-8"))
    allowed = {item["sample_id"] for item in synthetic["cases"]}
    path = tmp_path / "portfolio.json"

    missing_kind = copy.deepcopy(synthetic)
    del missing_kind["portfolio_kind"]
    path.write_text(json.dumps(missing_kind), encoding="utf-8")
    with pytest.raises(ValueError, match="portfolio_kind"):
        evaluate_cases(path, allowed)

    confused = copy.deepcopy(synthetic)
    confused["claim_boundary"] = "Governed product baseline accuracy evidence."
    path.write_text(json.dumps(confused), encoding="utf-8")
    with pytest.raises(ValueError, match="synthetic_contract"):
        evaluate_cases(path, allowed)

    governed = json.loads(
        json.dumps(synthetic)
        .replace("Synthetic", "Governed")
        .replace("synthetic", "governed")
        .replace("repository_fixture", "governed_product_evaluation")
    )
    governed["portfolio_kind"] = "governed_product_baseline"
    governed["portfolio_id"] = "governed-product-baseline-test"
    governed["claim_boundary"] = (
        "Governed product baseline metrics recomputed from protected raw cases; "
        "inference provenance is validated separately."
    )
    path.write_text(json.dumps(governed), encoding="utf-8")
    report = evaluate_cases(path, allowed)
    assert report["portfolio_kind"] == "governed_product_baseline"
    assert set(report["evaluated_task_families"]) == TASKS

    governed["cases"] = governed["cases"][:-1]
    path.write_text(json.dumps(governed), encoding="utf-8")


def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
    references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}]
    predictions = [
        {
            "id": "z-true",
            "class": "building",
            "bbox": [0, 0, 4, 4],
            "confidence": 0.8,
        },
        {
            "id": "a-false",
            "class": "building",
            "bbox": [10, 10, 12, 12],
            "confidence": 0.8,
        },
    ]
    forward = detection_ap(predictions, references, 0.5)
    reverse = detection_ap(list(reversed(predictions)), references, 0.5)
    assert forward == reverse == pytest.approx(0.5)

    wrong_class = copy.deepcopy(predictions)
    wrong_class[1] = {
        "id": "a-tank",
        "class": "tank",
        "bbox": [0, 0, 4, 4],
        "confidence": 0.95,
    }
    assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5)


def test_detection_ap_and_calibration_are_pooled_globally_and_per_subgroup(
    tmp_path: Path,
) -> None:
    first = detection_case("a-case")
    first["predictions"] = [
        {
            "id": "p-true",
            "class": "building",
            "bbox": [0, 0, 4, 4],
            "confidence": 0.9,
        }
    ]
    second = detection_case("b-case")
    second["predictions"] = [
        {
            "id": "p-false",
            "class": "building",
            "bbox": [10, 10, 12, 12],
            "confidence": 0.9,
        },
        {
            "id": "p-true",
            "class": "building",
            "bbox": [0, 0, 4, 4],
            "confidence": 0.8,
        },
    ]
    portfolio = {
        "schema_version": 2,
        "portfolio_kind": "synthetic_contract",
        "portfolio_id": "synthetic-pooled-detection",
        "portfolio_lineage": {
            "origin": "repository_fixture",
            "source_path": "pooled.json",
            "version": "1",
        },
        "split_roles": ["test"],
        "selection_policy": "Fixed before evaluation; no selection.",
        "claim_boundary": "Synthetic evaluator test; not product accuracy.",
        "protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
        "cases": [first, second],
    }
    path = tmp_path / "pooled.json"
    path.write_text(json.dumps(portfolio, ensure_ascii=False), encoding="utf-8")
    report = evaluate_cases(path, {"a-case", "b-case"})

    case_ap = [item["metrics"]["ap50"] for item in report["results"]]
    pooled = report["portfolio_metrics"]["object_detection"]["micro"]
    expected_pooled = detection_ap(
        [
            {**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
            for item in first["predictions"]
        ]
        + [
            {**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
            for item in second["predictions"]
        ],
        [
            {**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
            for item in first["references"]
        ]
        + [
            {**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
            for item in second["references"]
        ],
        0.5,
    )
    assert pooled["ap50"] == expected_pooled
    assert pooled["ap50"] != pytest.approx(sum(case_ap) / len(case_ap))
    assert sum(item["count"] for item in pooled["calibration"]["bins"]) == 3
    subgroup = report["subgroups"]["dimensions"]["region"]["strata"]["flanders"]
    subgroup_calibration = subgroup["task_metrics"]["object_detection"]["micro"][
        "calibration"
    ]
    assert sum(item["count"] for item in subgroup_calibration["bins"]) == 3


def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None:
    invalid_case = detection_case("invalid-class")
    invalid_case["predictions"][0]["class"] = "road"
    with pytest.raises(ValueError, match="outside the declared ontology"):
        evaluate_object_detection(invalid_case)

    valid = raster_case()
    valid["predictions"][0][1] = -9999
    valid["raster_context"]["prediction"]["mask"][0][1] = False
    result = evaluate_raster_classification(valid)
    assert result["metrics"]["prediction_coverage"] == pytest.approx(0.75)
    assert result["metrics"]["per_class"]["1"]["false_negative"] == 1

    jagged = raster_case("jagged")
    jagged["predictions"][1].pop()
    with pytest.raises(ValueError, match="exactly rectangular"):
        evaluate_raster_classification(jagged)

    missing_metadata = raster_case("missing-metadata")
    del missing_metadata["raster_context"]["prediction"]["crs"]
    with pytest.raises(ValueError, match="missing"):
        evaluate_raster_classification(missing_metadata)

    shifted = raster_case("shifted")
    shifted["raster_context"]["prediction"]["transform"][2] += 1
    with pytest.raises(ValueError, match="affine alignment differs"):
        evaluate_raster_classification(shifted)

    invalid_class = raster_case("invalid-class")
    invalid_class["predictions"][0][0] = 3
    with pytest.raises(ValueError, match="prediction class outside ontology"):
        evaluate_raster_classification(invalid_class)

    invalid_nodata = raster_case("invalid-nodata")
    invalid_nodata["predictions"][0][0] = -9999
    with pytest.raises(ValueError, match="marks nodata as valid"):
        evaluate_raster_classification(invalid_nodata)
    singular = raster_case("singular")
    for side in ("reference", "prediction"):
        singular["raster_context"][side]["transform"] = [1, 2, 0, 2, 4, 0]
    with pytest.raises(ValueError, match="affine transform is singular"):
        evaluate_raster_classification(singular)


def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None:
    assert evaluate_vector_comparison(polygon_case())["metrics"]["f1"] == 1.0
    assert (
        evaluate_footprint_segmentation(polygon_case("footprint_segmentation"))[
            "metrics"
        ]["mean_iou"]
        == 1.0
    )
    outer = [
        [100000, 200000],
        [100020, 200000],
        [100020, 200020],
        [100000, 200020],
        [100000, 200000],
    ]
    hole = [
        [100005, 200005],
        [100010, 200005],
        [100010, 200010],
        [100005, 200010],
        [100005, 200005],
    ]
    polygon_geometry = {"type": "Polygon", "coordinates": [outer, hole]}
    geojson_polygon = polygon_case()
    for side in ("references", "predictions"):
        del geojson_polygon[side][0]["polygon"]
        geojson_polygon[side][0]["geometry"] = copy.deepcopy(polygon_geometry)
    polygon_result = evaluate_vector_comparison(geojson_polygon)
    assert polygon_result["metrics"]["mean_iou"] == 1.0
    assert polygon_result["raw"]["references"][0]["geometry"] == polygon_geometry

    second = [
        [100030, 200000],
        [100040, 200000],
        [100040, 200010],
        [100030, 200010],
        [100030, 200000],
    ]
    multipolygon_geometry = {
        "type": "MultiPolygon",
        "coordinates": [[outer, hole], [second]],
    }
    geojson_multi = polygon_case()
    for side in ("references", "predictions"):
        del geojson_multi[side][0]["polygon"]
        geojson_multi[side][0]["geometry"] = copy.deepcopy(multipolygon_geometry)
    assert evaluate_vector_comparison(geojson_multi)["metrics"]["mean_iou"] == 1.0

    geographic = polygon_case()
    geographic["spatial_context"]["crs"] = "EPSG:4326"
    with pytest.raises(ValueError, match="projected CRS"):
        evaluate_vector_comparison(geographic)
    mercator = polygon_case()
    mercator["spatial_context"]["crs"] = "EPSG:3857"
    with pytest.raises(ValueError, match="Mercator is unsuitable"):
        evaluate_vector_comparison(mercator)

    wrong_geography = polygon_case()
    wrong_geography["spatial_context"]["crs"] = "EPSG:32660"
    with pytest.raises(ValueError, match="does not overlap"):
        evaluate_vector_comparison(wrong_geography)

    wrong_units = polygon_case()
    wrong_units["spatial_context"]["coordinate_units"] = "degree"
    with pytest.raises(ValueError, match="must be 'm'"):
        evaluate_vector_comparison(wrong_units)

    bowtie = polygon_case()
    bowtie["predictions"][0]["polygon"] = [
        [100000, 200000],
        [100010, 200010],
        [100010, 200000],
        [100000, 200010],
        [100000, 200000],
    ]
    with pytest.raises(ValueError, match="positive-area and valid"):
        evaluate_vector_comparison(bowtie)


def test_failure_gallery_covers_geometry_raster_calibration_and_contexts() -> None:
    segmentation = polygon_case("footprint_segmentation")
    segmentation["predictions"][0]["polygon"] = [
        [100000, 200000],
        [100012, 200000],
        [100012, 200010],
        [100000, 200010],
        [100000, 200000],
    ]
    segmentation_result = evaluate_footprint_segmentation(segmentation)
    segmentation_codes = {
        item["error_code"] for item in segmentation_result["failures"]
    }
    assert {"M-BOUNDARY", "M-AREA-BIAS"} <= segmentation_codes

    raster = raster_case("raster-taxonomy")
    raster["metadata"]["tile_edge"] = True
    raster["predictions"][0][0] = 1
    raster_result = evaluate_raster_classification(raster)
    raster_failure = next(
        item
        for item in raster_result["failures"]
        if item["kind"] == "raster_misclassification"
    )
    assert raster_failure["error_code"] == "M-CLASS"
    assert "tile_edge" in raster_failure["contexts"]

    detection = detection_case("context-taxonomy")
    detection["references"] = []
    detection["predictions"] = [
        {
            "id": "high-confidence-fp",
            "class": "building",
            "bbox": [10, 10, 12, 12],
            "confidence": 0.95,
        }
    ]
    detection["config"]["fixed_diagnostic_risk_thresholds"] = [0.5, 0.9]
    detection["metadata"]["tile_edge"] = True
    detection["metadata"]["ood"] = True
    detection_result = evaluate_object_detection(detection)
    false_positive = next(
        item
        for item in detection_result["failures"]
        if item["kind"] == "false_positive"
    )
    assert {"tile_edge", "high_confidence", "out_of_distribution"} <= set(
        false_positive["contexts"]
    )
    assert {"M-MISCALIBRATED", "M-OOD"} <= set(false_positive["secondary_error_codes"])
    assert any(
        item["error_code"] == "M-MISCALIBRATED" for item in detection_result["failures"]
    )


def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> (
    None
):
    terrain = {
        "sample_id": "terrain",
        "task": "terrain_interpretation",
        "split": "test",
        "metadata": copy.deepcopy(METADATA),
        "config": {},
        "lineage": lineage("terrain"),
        "units": "m_TAW",
        "references": [1.0, 2.0],
        "predictions": [1.1, None],
    }
    assert evaluate_terrain(terrain)["metrics"]["coverage"] == 0.5
    for field, value in (("references", math.nan), ("predictions", math.inf)):
        invalid = copy.deepcopy(terrain)
        invalid[field][0] = value
        with pytest.raises(ValueError, match="finite number"):
            evaluate_terrain(invalid)

    validation = {
        "sample_id": "validation",
        "task": "geospatial_data_validation",
        "split": "test",
        "metadata": copy.deepcopy(METADATA),
        "config": {},
        "lineage": lineage("validation"),
        "expected_anomalies": [{"code": "D-MAJOR", "severity": "major"}],
        "observed_anomalies": [],
    }
    assert (
        evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
        == 0
    )
    validation["expected_anomalies"].append(
        {"code": "D-CRITICAL", "severity": "critical"}
    )
    assert (
        evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
        == 1
    )
    validation["expected_anomalies"] = [{"code": "D-SEVERITY", "severity": "critical"}]
    validation["observed_anomalies"] = [{"code": "D-SEVERITY", "severity": "minor"}]
    severity_result = evaluate_validation(validation)
    assert severity_result["metrics"]["true_positive"] == 0
    assert severity_result["metrics"]["false_positive"] == 1
    assert severity_result["metrics"]["false_negative"] == 1
    assert severity_result["metrics"]["severity_mismatch_count"] == 1
    assert severity_result["metrics"]["blocker_or_critical_miss_count"] == 1

    validation["expected_anomalies"] = ["D-NO-SEVERITY"]
    with pytest.raises(ValueError, match="include code and severity"):
        evaluate_validation(validation)


def _subgroup_result(region: str, index: int, tp: int, fp: int, fn: int) -> dict:
    metadata = copy.deepcopy(METADATA)
    metadata["region"] = region
    sample_id = f"{region}-{index}"
    reference = {"id": "r", "class": "building", "bbox": [0, 0, 1, 1]}
    prediction = {
        "id": "p",
        "class": "building",
        "bbox": [0, 0, 1, 1],
        "confidence": 0.8,
    }
    return {
        "sample_id": sample_id,
        "task": "object_detection",
        "metadata": metadata,
        "metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4},
        "raw": {
            "sample_id": sample_id,
            "classes": ["building"],
            "references": [reference],
            "predictions_pre_filter": [prediction],
            "predictions_post_filter": [prediction],
            "matches": [
                {
                    "prediction_id": "p",
                    "reference_id": "r",
                    "overlap": 1.0,
                    "confidence": 0.8,
                    "class": "building",
                }
            ],
        },
        "failures": [],
    }


def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
    results = [
        *[_subgroup_result("strong", index, 10, 0, 0) for index in range(5)],
        *[_subgroup_result("weak", index, 1, 4, 4) for index in range(5)],
    ]
    report = subgroup_report(results)
    region = report["dimensions"]["region"]
    weak = region["strata"]["weak"]["task_metrics"]["object_detection"]

    assert weak["status"] == "evaluable"
    assert weak["case_support"] == 5
    assert weak["micro"]["precision_ci95_wilson"]["status"] == "computed"
    assert weak["macro"]["f1_case_support"] == 5
    assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak"

    insufficient = subgroup_report([_subgroup_result("thin", 0, 1, 0, 0)])
    thin = insufficient["dimensions"]["region"]["strata"]["thin"]
    assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support"
    assert thin["release_gate_status"] == "not_evaluable"
    assert insufficient["overall_status"] == "not_evaluable"


def test_capability_inventory_is_comprehensive_and_honest() -> None:
    inventory = task_inventory()
    assert {item["task"] for item in inventory} == TASKS
    assert len(inventory) >= 15
    assert all(item["implementation_paths"] for item in inventory)
    assert all(item["suitable_metrics"] for item in inventory)
    assistant = next(
        item
        for item in inventory
        if item["capability_id"] == "geo_assistant_orchestration"
    )
    assert assistant["evaluation_status"].startswith("no_independent_accuracy_score")