from __future__ import annotations import copy import hashlib import json import math import sys from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] SCRIPTS = ROOT / "scripts" if str(SCRIPTS) not in sys.path: sys.path.insert(0, str(SCRIPTS)) from accuracy_phase4_evaluator import ( # noqa: E402 TASKS, canonical_hash, count_metrics, detection_ap, evaluate_cases, evaluate_object_detection, evaluate_footprint_segmentation, evaluate_raster_classification, evaluate_terrain, evaluate_validation, evaluate_vector_comparison, subgroup_report, task_inventory, ) METADATA = { "region": "flanders", "municipality": "Mol", "urbanity": "urban", "object_size": "medium", "source": "synthetic-source", "sensor": "synthetic-sensor", "resolution_m": 0.25, "season": "summer", "date": "2026-01-01", "vegetation": "partial", "occlusion": "none", "difficulty": "normal", } def lineage(sample_id: str) -> dict: return { "reference": { "source_id": f"synthetic:{sample_id}:reference", "source_version": "1", "derivation": "hand_authored_contract_fixture", }, "prediction": { "source_id": f"synthetic:{sample_id}:prediction", "source_version": "1", "derivation": "hand_authored_fixed_output", }, } def detection_case(sample_id: str = "det-1") -> dict: return { "sample_id": sample_id, "task": "object_detection", "split": "test", "metadata": copy.deepcopy(METADATA), "config": {"confidence_threshold": 0.5, "match_iou": 0.5}, "lineage": lineage(sample_id), "classes": ["building", "tank"], "references": [{"id": "r-building", "class": "building", "bbox": [0, 0, 4, 4]}], "predictions": [ { "id": "p-building", "class": "building", "bbox": [0, 0, 4, 4], "confidence": 0.8, }, { "id": "p-filtered", "class": "building", "bbox": [10, 10, 12, 12], "confidence": 0.2, }, ], } def raster_case(sample_id: str = "raster-1") -> dict: reference_side = { "crs": "EPSG:31370", "transform": [1, 0, 100000, 0, -1, 200000], "shape": [2, 2], "nodata": -9999, "mask": [[True, True], [True, True]], } return { "sample_id": sample_id, "task": "raster_classification", "split": "test", "metadata": copy.deepcopy(METADATA), "config": {}, "lineage": lineage(sample_id), "classes": [0, 1], "references": [[0, 1], [1, 0]], "predictions": [[0, 1], [1, 0]], "raster_context": { "reference": reference_side, "prediction": copy.deepcopy(reference_side), }, } def polygon_case(task: str = "vector_comparison") -> dict: sample_id = f"{task}-1" config = {"match_iou": 0.5} if task == "footprint_segmentation": config["boundary_tolerance_m"] = 1.0 polygon = [ [100000, 200000], [100010, 200000], [100010, 200010], [100000, 200010], [100000, 200000], ] return { "sample_id": sample_id, "task": task, "split": "test", "metadata": copy.deepcopy(METADATA), "config": config, "lineage": lineage(sample_id), "classes": ["building"], "spatial_context": { "crs": "EPSG:31370", "coordinate_units": "m", "metric": True, }, "references": [{"id": "reference", "class": "building", "polygon": polygon}], "predictions": [{"id": "prediction", "class": "building", "polygon": polygon}], } def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> None: case = detection_case() portfolio = { "schema_version": 2, "portfolio_id": "synthetic-hardening-test", "portfolio_lineage": { "origin": "repository_fixture", "source_path": "synthetic.json", "version": "1", }, "split_roles": ["test"], "selection_policy": "Fixed before evaluation; no selection.", "claim_boundary": "Synthetic evaluator test; not product accuracy.", "protected_policy": {"threshold_selection_allowed": False}, "cases": [case], } path = tmp_path / "portfolio.json" path.write_text( json.dumps(portfolio, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) report = evaluate_cases(path, {case["sample_id"]}) raw = report["results"][0]["raw"] assert raw["references"] == case["references"] assert raw["predictions_pre_filter"] == case["predictions"] assert raw["predictions_post_filter"] == case["predictions"][:1] assert raw["config"] == case["config"] assert raw["split"] == "test" assert raw["input_lineage"] == case["lineage"] assert raw["portfolio_lineage"]["declared"] == portfolio["portfolio_lineage"] assert raw["hashes"]["case_input_canonical_json_sha256"] == canonical_hash(case) assert raw["hashes"]["references_canonical_json_sha256"] == canonical_hash( case["references"] ) assert ( report["portfolio_file_sha256"] == hashlib.sha256(path.read_bytes()).hexdigest() ) assert report["portfolio_canonical_json_sha256"] == canonical_hash(portfolio) assert report["results_canonical_json_sha256"] == canonical_hash(report["results"]) high_threshold = next( row for row in report["results"][0]["metrics"]["coverage_risk"] if row["threshold"] == 0.9 ) assert high_threshold["retained_prediction_coverage"] == 0.0 assert high_threshold["reference_coverage"] == 0.0 assert high_threshold["false_negative_count"] == 1 assert high_threshold["risk"] == 1.0 challenge_exposed = copy.deepcopy(portfolio) challenge_exposed["challenge_labels"] = [] path.write_text(json.dumps(challenge_exposed, ensure_ascii=False), encoding="utf-8") with pytest.raises(ValueError, match="Challenge cases and labels"): evaluate_cases(path, {case["sample_id"]}) def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None: references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}] predictions = [ { "id": "z-true", "class": "building", "bbox": [0, 0, 4, 4], "confidence": 0.8, }, { "id": "a-false", "class": "building", "bbox": [10, 10, 12, 12], "confidence": 0.8, }, ] forward = detection_ap(predictions, references, 0.5) reverse = detection_ap(list(reversed(predictions)), references, 0.5) assert forward == reverse == pytest.approx(0.5) wrong_class = copy.deepcopy(predictions) wrong_class[1] = { "id": "a-tank", "class": "tank", "bbox": [0, 0, 4, 4], "confidence": 0.95, } assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5) def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None: invalid_case = detection_case("invalid-class") invalid_case["predictions"][0]["class"] = "road" with pytest.raises(ValueError, match="outside the declared ontology"): evaluate_object_detection(invalid_case) valid = raster_case() valid["predictions"][0][1] = -9999 valid["raster_context"]["prediction"]["mask"][0][1] = False result = evaluate_raster_classification(valid) assert result["metrics"]["prediction_coverage"] == pytest.approx(0.75) assert result["metrics"]["per_class"]["1"]["false_negative"] == 1 jagged = raster_case("jagged") jagged["predictions"][1].pop() with pytest.raises(ValueError, match="exactly rectangular"): evaluate_raster_classification(jagged) missing_metadata = raster_case("missing-metadata") del missing_metadata["raster_context"]["prediction"]["crs"] with pytest.raises(ValueError, match="missing"): evaluate_raster_classification(missing_metadata) shifted = raster_case("shifted") shifted["raster_context"]["prediction"]["transform"][2] += 1 with pytest.raises(ValueError, match="affine alignment differs"): evaluate_raster_classification(shifted) invalid_class = raster_case("invalid-class") invalid_class["predictions"][0][0] = 3 with pytest.raises(ValueError, match="prediction class outside ontology"): evaluate_raster_classification(invalid_class) invalid_nodata = raster_case("invalid-nodata") invalid_nodata["predictions"][0][0] = -9999 with pytest.raises(ValueError, match="marks nodata as valid"): evaluate_raster_classification(invalid_nodata) def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None: assert evaluate_vector_comparison(polygon_case())["metrics"]["f1"] == 1.0 assert ( evaluate_footprint_segmentation(polygon_case("footprint_segmentation"))[ "metrics" ]["mean_iou"] == 1.0 ) geographic = polygon_case() geographic["spatial_context"]["crs"] = "EPSG:4326" with pytest.raises(ValueError, match="projected CRS"): evaluate_vector_comparison(geographic) wrong_units = polygon_case() wrong_units["spatial_context"]["coordinate_units"] = "degree" with pytest.raises(ValueError, match="must be 'm'"): evaluate_vector_comparison(wrong_units) bowtie = polygon_case() bowtie["predictions"][0]["polygon"] = [ [100000, 200000], [100010, 200010], [100010, 200000], [100000, 200010], [100000, 200000], ] with pytest.raises(ValueError, match="positive-area and valid"): evaluate_vector_comparison(bowtie) def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> ( None ): terrain = { "sample_id": "terrain", "task": "terrain_interpretation", "split": "test", "metadata": copy.deepcopy(METADATA), "config": {}, "lineage": lineage("terrain"), "units": "m_TAW", "references": [1.0, 2.0], "predictions": [1.1, None], } assert evaluate_terrain(terrain)["metrics"]["coverage"] == 0.5 for field, value in (("references", math.nan), ("predictions", math.inf)): invalid = copy.deepcopy(terrain) invalid[field][0] = value with pytest.raises(ValueError, match="finite number"): evaluate_terrain(invalid) validation = { "sample_id": "validation", "task": "geospatial_data_validation", "split": "test", "metadata": copy.deepcopy(METADATA), "config": {}, "lineage": lineage("validation"), "expected_anomalies": [{"code": "D-MAJOR", "severity": "major"}], "observed_anomalies": [], } assert ( evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"] == 0 ) validation["expected_anomalies"].append( {"code": "D-CRITICAL", "severity": "critical"} ) assert ( evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"] == 1 ) validation["expected_anomalies"] = ["D-NO-SEVERITY"] with pytest.raises(ValueError, match="include code and severity"): evaluate_validation(validation) def _subgroup_result(region: str, tp: int, fp: int, fn: int) -> dict: metadata = copy.deepcopy(METADATA) metadata["region"] = region return { "task": "object_detection", "metadata": metadata, "metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4}, "failures": [], } def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None: results = [ *[_subgroup_result("strong", 10, 0, 0) for _ in range(5)], *[_subgroup_result("weak", 1, 4, 4) for _ in range(5)], ] report = subgroup_report(results) region = report["dimensions"]["region"] weak = region["strata"]["weak"]["task_metrics"]["object_detection"] assert weak["status"] == "evaluable" assert weak["case_support"] == 5 assert weak["micro"]["precision_ci95_wilson"]["status"] == "computed" assert weak["macro"]["f1_case_support"] == 5 assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak" insufficient = subgroup_report([_subgroup_result("thin", 1, 0, 0)]) thin = insufficient["dimensions"]["region"]["strata"]["thin"] assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support" assert thin["release_gate_status"] == "not_evaluable" assert insufficient["overall_status"] == "not_evaluable" def test_capability_inventory_is_comprehensive_and_honest() -> None: inventory = task_inventory() assert {item["task"] for item in inventory} == TASKS assert len(inventory) >= 15 assert all(item["implementation_paths"] for item in inventory) assert all(item["suitable_metrics"] for item in inventory) assistant = next( item for item in inventory if item["capability_id"] == "geo_assistant_orchestration" ) assert assistant["evaluation_status"].startswith("no_independent_accuracy_score")