diff --git a/.gitignore b/.gitignore index fec18197..260f3229 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,12 @@ build/ /artifacts/evidence/accuracy/* !/artifacts/evidence/accuracy/P1/ !/artifacts/evidence/accuracy/P1/** +!/artifacts/evidence/accuracy/P3/ +!/artifacts/evidence/accuracy/P3/** +!/artifacts/evidence/accuracy/P4/ +/artifacts/evidence/accuracy/P4/* +!/artifacts/evidence/accuracy/P4/reference-harness-v2/ +!/artifacts/evidence/accuracy/P4/reference-harness-v2/** !/artifacts/evidence/accuracy/P2/ !/artifacts/evidence/accuracy/P2/** /.cache/ diff --git a/backend/tests/test_accuracy_phase4_evaluation.py b/backend/tests/test_accuracy_phase4_evaluation.py new file mode 100644 index 00000000..d65561b1 --- /dev/null +++ b/backend/tests/test_accuracy_phase4_evaluation.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from accuracy_phase4_evaluator import evaluate_cases # noqa: E402 +from generate_accuracy_phase4_splits import ( # noqa: E402 + LeakageError, + assert_training_inputs_safe, + build_manifests, +) +from run_accuracy_phase4_benchmark import ( # noqa: E402 + EvidenceConflictError, + build_release_gate_report, + canonical_golden_baseline, + firewall_contract_checks, + run_workflow, +) + + +SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json" +CASES = ROOT / "fixtures/accuracy/p4/protected-baseline-cases.json" + + +def load_source() -> dict: + return json.loads(SOURCE.read_text(encoding="utf-8")) + + +def evaluation_inputs() -> tuple[dict, dict, dict, dict]: + source = load_source() + development, protected, leakage = build_manifests(source) + split_result = { + "development": development, + "protected": protected, + "leakage": leakage, + } + allowed = { + item["sample_id"] + for item in protected["samples"] + if item["split"] in {"test", "background-test"} + } + evaluation = evaluate_cases(CASES, allowed) + portfolio = json.loads(CASES.read_text(encoding="utf-8")) + firewall = firewall_contract_checks(split_result, CASES) + return split_result, evaluation, portfolio, firewall + + +def test_split_fixture_is_order_independent_and_has_all_roles() -> None: + source = load_source() + development, protected, leakage = build_manifests(source) + reversed_source = copy.deepcopy(source) + reversed_source["samples"].reverse() + reversed_development, reversed_protected, reversed_leakage = build_manifests( + reversed_source + ) + + assert leakage["status"] == "pass" + assert leakage["finding_count"] == 0 + assert leakage["split_counts"] == { + "background-test": 2, + "calibration": 2, + "challenge": 4, + "test": 7, + "train": 3, + "val": 3, + } + assert reversed_development["manifest_sha256"] == development["manifest_sha256"] + assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"] + assert reversed_leakage == leakage + + +def test_training_firewall_rejects_non_train_and_protected_lineage() -> None: + development, protected, leakage = build_manifests(load_source()) + assert leakage["status"] == "pass" + train = [item for item in development["samples"] if item["split"] == "train"] + validation = next(item for item in development["samples"] if item["split"] == "val") + + assert_training_inputs_safe([], train, protected) + with pytest.raises(LeakageError, match="non_train_role"): + assert_training_inputs_safe([], [validation], protected) + with pytest.raises(LeakageError, match="protected_identity"): + disguised = dict(train[0]) + disguised["source_family"] = protected["samples"][0]["source_family"] + assert_training_inputs_safe([], [disguised], protected) + with pytest.raises(LeakageError, match="protected_path"): + assert_training_inputs_safe([CASES], [], protected) + + +def test_task_evaluator_retains_exact_raw_inputs_metrics_and_failures() -> None: + _split_result, report, _portfolio, _firewall = evaluation_inputs() + + assert report["task_count"] == 7 + assert report["case_count"] == 9 + assert len(report["task_inventory"]) >= 15 + assert len(report["failures"]) == 11 + assert report["subgroups"]["overall_status"] == "not_evaluable" + assert all( + { + "references", + "predictions_pre_filter", + "predictions_post_filter", + "config", + "input_lineage", + "portfolio_lineage", + } + <= set(item["raw"]) + for item in report["results"] + ) + detection = next( + item + for item in report["results"] + if item["sample_id"] == "det-test-flanders-urban" + ) + assert len(detection["raw"]["predictions_pre_filter"]) == 4 + assert len(detection["raw"]["predictions_post_filter"]) == 3 + assert detection["metrics"]["true_positive"] == 2 + assert detection["metrics"]["false_positive"] == 1 + assert detection["metrics"]["ap50"] is not None + empty = next( + item + for item in report["results"] + if item["sample_id"] == "background-test-pure-empty" + ) + assert empty["metrics"]["precision"] is None + assert empty["metrics"]["recall"] is None + assert empty["metrics"]["f1"] is None + + +def test_product_prerequisites_cannot_pass_without_executed_baseline() -> None: + split_result, evaluation, portfolio, firewall = evaluation_inputs() + product_gates = { + "active_model_available_and_hash_verified": {"status": "pass"}, + "authoritative_reference_portfolio_available": {"status": "pass"}, + "human_review_complete": {"status": "pass"}, + "split_independence": {"status": "pass"}, + "phase3_leakage_resolved": {"status": "pass"}, + "protected_storage_isolation": {"status": "pass"}, + "executed_product_incumbent_baseline": { + "status": "not_evaluable", + "reason": "no raw active-model inference", + }, + "representative_product_subgroup_support": {"status": "pass"}, + } + report = build_release_gate_report( + split_result, + evaluation, + portfolio, + canonical_golden_baseline(), + firewall, + product_gates, + ) + + assert report["local_harness_status"] == "pass" + assert report["product_benchmark_status"] == "not_evaluable" + assert report["status"] == "not_evaluable" + assert report["phase_decision"] == "blocked" + + +def test_one_workflow_is_byte_reproducible_complete_and_fail_closed( + tmp_path: Path, +) -> None: + output = tmp_path / "p4" + first = run_workflow(ROOT, output) + first_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")} + second = run_workflow(ROOT, output) + second_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")} + + assert first == second + assert first_bytes == second_bytes + assert first["local_harness_status"] == "pass" + assert first["product_benchmark_status"] == "not_evaluable" + assert first["status"] == "not_evaluable" + assert first["phase4_done"] is False + assert first["phase5_ready"] is False + required = { + "acceptance-gates.json", + "aoi-metrics.json", + "baseline-raw-predictions.json", + "benchmark-manifest.json", + "calibration-metrics.json", + "candidate-vs-incumbent.json", + "development-split-manifest.json", + "error-taxonomy.json", + "evaluation-contract.json", + "failure-gallery.json", + "generation-status.json", + "human-review-summary.json", + "input-manifest.json", + "latency-and-reliability.json", + "leakage-gate-report.json", + "metric-report.json", + "object-metrics.json", + "protected-split-manifest.json", + "reference-implementation-baseline.json", + "release-gate-report.json", + "split-and-leakage-audit.json", + "stratified-metrics.json", + "tile-metrics.json", + "workflow-summary.json", + "evidence-manifest.json", + } + assert required == set(first_bytes) + manifest = json.loads( + (output / "evidence-manifest.json").read_text(encoding="utf-8") + ) + assert manifest["artifact_count"] == len(required) - 1 + for item in manifest["artifacts"]: + path = output / item["path"] + assert path.stat().st_size == item["size_bytes"] + assert hashlib.sha256(path.read_bytes()).hexdigest() == item["sha256"] + gates = json.loads( + (output / "release-gate-report.json").read_text(encoding="utf-8") + ) + assert gates["promotion_allowed"] is False + assert all(item["status"] == "pass" for item in gates["local_gates"].values()) + assert {item["status"] for item in gates["product_gates"].values()} <= { + "pass", + "fail", + "not_evaluable", + } + + +def test_immutable_workflow_refuses_to_replace_changed_evidence(tmp_path: Path) -> None: + output = tmp_path / "p4" + run_workflow(ROOT, output) + (output / "workflow-summary.json").write_text("{}\n", encoding="utf-8") + + with pytest.raises(EvidenceConflictError, match="Refusing to overwrite"): + run_workflow(ROOT, output) diff --git a/backend/tests/test_accuracy_phase4_evaluator_hardening.py b/backend/tests/test_accuracy_phase4_evaluator_hardening.py new file mode 100644 index 00000000..7cedbe4b --- /dev/null +++ b/backend/tests/test_accuracy_phase4_evaluator_hardening.py @@ -0,0 +1,397 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import math +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from accuracy_phase4_evaluator import ( # noqa: E402 + TASKS, + canonical_hash, + count_metrics, + detection_ap, + evaluate_cases, + evaluate_object_detection, + evaluate_footprint_segmentation, + evaluate_raster_classification, + evaluate_terrain, + evaluate_validation, + evaluate_vector_comparison, + subgroup_report, + task_inventory, +) + + +METADATA = { + "region": "flanders", + "municipality": "Mol", + "urbanity": "urban", + "object_size": "medium", + "source": "synthetic-source", + "sensor": "synthetic-sensor", + "resolution_m": 0.25, + "season": "summer", + "date": "2026-01-01", + "vegetation": "partial", + "occlusion": "none", + "difficulty": "normal", +} + + +def lineage(sample_id: str) -> dict: + return { + "reference": { + "source_id": f"synthetic:{sample_id}:reference", + "source_version": "1", + "derivation": "hand_authored_contract_fixture", + }, + "prediction": { + "source_id": f"synthetic:{sample_id}:prediction", + "source_version": "1", + "derivation": "hand_authored_fixed_output", + }, + } + + +def detection_case(sample_id: str = "det-1") -> dict: + return { + "sample_id": sample_id, + "task": "object_detection", + "split": "test", + "metadata": copy.deepcopy(METADATA), + "config": {"confidence_threshold": 0.5, "match_iou": 0.5}, + "lineage": lineage(sample_id), + "classes": ["building", "tank"], + "references": [{"id": "r-building", "class": "building", "bbox": [0, 0, 4, 4]}], + "predictions": [ + { + "id": "p-building", + "class": "building", + "bbox": [0, 0, 4, 4], + "confidence": 0.8, + }, + { + "id": "p-filtered", + "class": "building", + "bbox": [10, 10, 12, 12], + "confidence": 0.2, + }, + ], + } + + +def raster_case(sample_id: str = "raster-1") -> dict: + reference_side = { + "crs": "EPSG:31370", + "transform": [1, 0, 100000, 0, -1, 200000], + "shape": [2, 2], + "nodata": -9999, + "mask": [[True, True], [True, True]], + } + return { + "sample_id": sample_id, + "task": "raster_classification", + "split": "test", + "metadata": copy.deepcopy(METADATA), + "config": {}, + "lineage": lineage(sample_id), + "classes": [0, 1], + "references": [[0, 1], [1, 0]], + "predictions": [[0, 1], [1, 0]], + "raster_context": { + "reference": reference_side, + "prediction": copy.deepcopy(reference_side), + }, + } + + +def polygon_case(task: str = "vector_comparison") -> dict: + sample_id = f"{task}-1" + config = {"match_iou": 0.5} + if task == "footprint_segmentation": + config["boundary_tolerance_m"] = 1.0 + polygon = [ + [100000, 200000], + [100010, 200000], + [100010, 200010], + [100000, 200010], + [100000, 200000], + ] + return { + "sample_id": sample_id, + "task": task, + "split": "test", + "metadata": copy.deepcopy(METADATA), + "config": config, + "lineage": lineage(sample_id), + "classes": ["building"], + "spatial_context": { + "crs": "EPSG:31370", + "coordinate_units": "m", + "metric": True, + }, + "references": [{"id": "reference", "class": "building", "polygon": polygon}], + "predictions": [{"id": "prediction", "class": "building", "polygon": polygon}], + } + + +def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> None: + case = detection_case() + portfolio = { + "schema_version": 2, + "portfolio_id": "synthetic-hardening-test", + "portfolio_lineage": { + "origin": "repository_fixture", + "source_path": "synthetic.json", + "version": "1", + }, + "split_roles": ["test"], + "selection_policy": "Fixed before evaluation; no selection.", + "claim_boundary": "Synthetic evaluator test; not product accuracy.", + "protected_policy": {"threshold_selection_allowed": False}, + "cases": [case], + } + path = tmp_path / "portfolio.json" + path.write_text( + json.dumps(portfolio, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + report = evaluate_cases(path, {case["sample_id"]}) + raw = report["results"][0]["raw"] + + assert raw["references"] == case["references"] + assert raw["predictions_pre_filter"] == case["predictions"] + assert raw["predictions_post_filter"] == case["predictions"][:1] + assert raw["config"] == case["config"] + assert raw["split"] == "test" + assert raw["input_lineage"] == case["lineage"] + assert raw["portfolio_lineage"]["declared"] == portfolio["portfolio_lineage"] + assert raw["hashes"]["case_input_canonical_json_sha256"] == canonical_hash(case) + assert raw["hashes"]["references_canonical_json_sha256"] == canonical_hash( + case["references"] + ) + assert ( + report["portfolio_file_sha256"] == hashlib.sha256(path.read_bytes()).hexdigest() + ) + assert report["portfolio_canonical_json_sha256"] == canonical_hash(portfolio) + assert report["results_canonical_json_sha256"] == canonical_hash(report["results"]) + high_threshold = next( + row + for row in report["results"][0]["metrics"]["coverage_risk"] + if row["threshold"] == 0.9 + ) + assert high_threshold["retained_prediction_coverage"] == 0.0 + assert high_threshold["reference_coverage"] == 0.0 + assert high_threshold["false_negative_count"] == 1 + assert high_threshold["risk"] == 1.0 + + challenge_exposed = copy.deepcopy(portfolio) + challenge_exposed["challenge_labels"] = [] + path.write_text(json.dumps(challenge_exposed, ensure_ascii=False), encoding="utf-8") + with pytest.raises(ValueError, match="Challenge cases and labels"): + evaluate_cases(path, {case["sample_id"]}) + + +def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None: + references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}] + predictions = [ + { + "id": "z-true", + "class": "building", + "bbox": [0, 0, 4, 4], + "confidence": 0.8, + }, + { + "id": "a-false", + "class": "building", + "bbox": [10, 10, 12, 12], + "confidence": 0.8, + }, + ] + forward = detection_ap(predictions, references, 0.5) + reverse = detection_ap(list(reversed(predictions)), references, 0.5) + assert forward == reverse == pytest.approx(0.5) + + wrong_class = copy.deepcopy(predictions) + wrong_class[1] = { + "id": "a-tank", + "class": "tank", + "bbox": [0, 0, 4, 4], + "confidence": 0.95, + } + assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5) + + +def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None: + invalid_case = detection_case("invalid-class") + invalid_case["predictions"][0]["class"] = "road" + with pytest.raises(ValueError, match="outside the declared ontology"): + evaluate_object_detection(invalid_case) + + valid = raster_case() + valid["predictions"][0][1] = -9999 + valid["raster_context"]["prediction"]["mask"][0][1] = False + result = evaluate_raster_classification(valid) + assert result["metrics"]["prediction_coverage"] == pytest.approx(0.75) + assert result["metrics"]["per_class"]["1"]["false_negative"] == 1 + + jagged = raster_case("jagged") + jagged["predictions"][1].pop() + with pytest.raises(ValueError, match="exactly rectangular"): + evaluate_raster_classification(jagged) + + missing_metadata = raster_case("missing-metadata") + del missing_metadata["raster_context"]["prediction"]["crs"] + with pytest.raises(ValueError, match="missing"): + evaluate_raster_classification(missing_metadata) + + shifted = raster_case("shifted") + shifted["raster_context"]["prediction"]["transform"][2] += 1 + with pytest.raises(ValueError, match="affine alignment differs"): + evaluate_raster_classification(shifted) + + invalid_class = raster_case("invalid-class") + invalid_class["predictions"][0][0] = 3 + with pytest.raises(ValueError, match="prediction class outside ontology"): + evaluate_raster_classification(invalid_class) + + invalid_nodata = raster_case("invalid-nodata") + invalid_nodata["predictions"][0][0] = -9999 + with pytest.raises(ValueError, match="marks nodata as valid"): + evaluate_raster_classification(invalid_nodata) + + +def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None: + assert evaluate_vector_comparison(polygon_case())["metrics"]["f1"] == 1.0 + assert ( + evaluate_footprint_segmentation(polygon_case("footprint_segmentation"))[ + "metrics" + ]["mean_iou"] + == 1.0 + ) + + geographic = polygon_case() + geographic["spatial_context"]["crs"] = "EPSG:4326" + with pytest.raises(ValueError, match="projected CRS"): + evaluate_vector_comparison(geographic) + + wrong_units = polygon_case() + wrong_units["spatial_context"]["coordinate_units"] = "degree" + with pytest.raises(ValueError, match="must be 'm'"): + evaluate_vector_comparison(wrong_units) + + bowtie = polygon_case() + bowtie["predictions"][0]["polygon"] = [ + [100000, 200000], + [100010, 200010], + [100010, 200000], + [100000, 200010], + [100000, 200000], + ] + with pytest.raises(ValueError, match="positive-area and valid"): + evaluate_vector_comparison(bowtie) + + +def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> ( + None +): + terrain = { + "sample_id": "terrain", + "task": "terrain_interpretation", + "split": "test", + "metadata": copy.deepcopy(METADATA), + "config": {}, + "lineage": lineage("terrain"), + "units": "m_TAW", + "references": [1.0, 2.0], + "predictions": [1.1, None], + } + assert evaluate_terrain(terrain)["metrics"]["coverage"] == 0.5 + for field, value in (("references", math.nan), ("predictions", math.inf)): + invalid = copy.deepcopy(terrain) + invalid[field][0] = value + with pytest.raises(ValueError, match="finite number"): + evaluate_terrain(invalid) + + validation = { + "sample_id": "validation", + "task": "geospatial_data_validation", + "split": "test", + "metadata": copy.deepcopy(METADATA), + "config": {}, + "lineage": lineage("validation"), + "expected_anomalies": [{"code": "D-MAJOR", "severity": "major"}], + "observed_anomalies": [], + } + assert ( + evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"] + == 0 + ) + validation["expected_anomalies"].append( + {"code": "D-CRITICAL", "severity": "critical"} + ) + assert ( + evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"] + == 1 + ) + validation["expected_anomalies"] = ["D-NO-SEVERITY"] + with pytest.raises(ValueError, match="include code and severity"): + evaluate_validation(validation) + + +def _subgroup_result(region: str, tp: int, fp: int, fn: int) -> dict: + metadata = copy.deepcopy(METADATA) + metadata["region"] = region + return { + "task": "object_detection", + "metadata": metadata, + "metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4}, + "failures": [], + } + + +def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None: + results = [ + *[_subgroup_result("strong", 10, 0, 0) for _ in range(5)], + *[_subgroup_result("weak", 1, 4, 4) for _ in range(5)], + ] + report = subgroup_report(results) + region = report["dimensions"]["region"] + weak = region["strata"]["weak"]["task_metrics"]["object_detection"] + + assert weak["status"] == "evaluable" + assert weak["case_support"] == 5 + assert weak["micro"]["precision_ci95_wilson"]["status"] == "computed" + assert weak["macro"]["f1_case_support"] == 5 + assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak" + + insufficient = subgroup_report([_subgroup_result("thin", 1, 0, 0)]) + thin = insufficient["dimensions"]["region"]["strata"]["thin"] + assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support" + assert thin["release_gate_status"] == "not_evaluable" + assert insufficient["overall_status"] == "not_evaluable" + + +def test_capability_inventory_is_comprehensive_and_honest() -> None: + inventory = task_inventory() + assert {item["task"] for item in inventory} == TASKS + assert len(inventory) >= 15 + assert all(item["implementation_paths"] for item in inventory) + assert all(item["suitable_metrics"] for item in inventory) + assistant = next( + item + for item in inventory + if item["capability_id"] == "geo_assistant_orchestration" + ) + assert assistant["evaluation_status"].startswith("no_independent_accuracy_score") diff --git a/backend/tests/test_accuracy_phase4_split_hardening.py b/backend/tests/test_accuracy_phase4_split_hardening.py new file mode 100644 index 00000000..a0caf1fc --- /dev/null +++ b/backend/tests/test_accuracy_phase4_split_hardening.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import copy +import json +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from generate_accuracy_phase4_splits import ( # noqa: E402 + LeakageError, + assert_training_inputs_safe, + build_manifests, + generate, +) + + +SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json" + + +def load_source() -> dict: + return json.loads(SOURCE.read_text(encoding="utf-8")) + + +def test_normative_roles_hashes_and_source_order_are_enforced() -> None: + source = load_source() + development, protected, leakage = build_manifests(source) + reversed_source = copy.deepcopy(source) + reversed_source["samples"].reverse() + reversed_development, reversed_protected, reversed_leakage = build_manifests( + reversed_source + ) + + assert leakage["status"] == "pass" + assert leakage["finding_count"] == 0 + assert leakage["split_counts"] == { + "background-test": 2, + "calibration": 2, + "challenge": 4, + "test": 7, + "train": 3, + "val": 3, + } + assert leakage["crs_validation"] == { + "status": "pass", + "crs": "EPSG:31370", + "distance_units": "m", + } + assert development["training_access_allowed_by_split"] == { + "train": True, + "val": False, + "calibration": False, + } + assert protected["labels_available_by_split"]["challenge"] == "sealed_external" + assert reversed_development["manifest_sha256"] == development["manifest_sha256"] + assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"] + assert reversed_leakage == leakage + + +@pytest.mark.parametrize( + ("field", "expected_code"), + [ + ("group_id", "S-SPATIAL-GROUP"), + ("source_family", "S-SOURCE-FAMILY"), + ("temporal_family", "S-TEMPORAL-FAMILY"), + ("raw_image_sha256", "S-RAW-IMAGE-DUPLICATE"), + ("processed_image_sha256", "S-PROCESSED-IMAGE-DUPLICATE"), + ("label_sha256", "S-LABEL-DUPLICATE"), + ("label_geometry_hash", "S-LABEL-GEOMETRY-DUPLICATE"), + ("parent_raster_id", "S-PARENT-RASTER"), + ("acquisition_id", "S-ACQUISITION"), + ], +) +def test_cross_split_lineage_and_content_collisions_fail( + field: str, expected_code: str +) -> None: + source = load_source() + source["samples"][8][field] = source["samples"][0][field] + _development, _protected, leakage = build_manifests(source) + + assert leakage["status"] == "fail" + assert expected_code in {item["code"] for item in leakage["findings"]} + + +@pytest.mark.parametrize( + ("field", "expected_code"), + [ + ("perceptual_image_hash", "S-PERCEPTUAL-IMAGE-NEAR-DUPLICATE"), + ("label_geometry_fingerprint", "S-LABEL-GEOMETRY-NEAR-DUPLICATE"), + ], +) +def test_near_duplicate_fingerprints_fail(field: str, expected_code: str) -> None: + source = load_source() + source["samples"][8][field] = source["samples"][0][field] + _development, _protected, leakage = build_manifests(source) + + assert leakage["status"] == "fail" + assert expected_code in {item["code"] for item in leakage["findings"]} + + +def test_object_native_feature_and_spatial_collisions_fail() -> None: + source = load_source() + source["samples"][8]["object_ids"] = source["samples"][0]["object_ids"] + source["samples"][9]["native_feature_ids"] = source["samples"][1][ + "native_feature_ids" + ] + source["samples"][10]["bbox"] = source["samples"][2]["bbox"] + _development, _protected, leakage = build_manifests(source) + codes = {item["code"] for item in leakage["findings"]} + + assert {"S-OBJECT-INSTANCE", "S-NATIVE-FEATURE", "S-SPATIAL-OVERLAP"} <= codes + + +def test_non_metric_crs_and_missing_normative_role_fail_closed() -> None: + geographic = load_source() + geographic["crs"] = "EPSG:4326" + with pytest.raises(LeakageError, match="projected in metres"): + build_manifests(geographic) + + missing = load_source() + missing["samples"] = [ + item for item in missing["samples"] if item["split"] != "calibration" + ] + with pytest.raises(LeakageError, match="Required splits are absent"): + build_manifests(missing) + + +def test_training_firewall_only_allows_train_and_binds_protected_lineage() -> None: + development, protected, leakage = build_manifests(load_source()) + assert leakage["status"] == "pass" + train = [item for item in development["samples"] if item["split"] == "train"] + validation = next(item for item in development["samples"] if item["split"] == "val") + protected_item = protected["samples"][0] + + assert_training_inputs_safe([], train, protected) + with pytest.raises(LeakageError, match="non_train_role"): + assert_training_inputs_safe([], [validation], protected) + with pytest.raises(LeakageError, match="protected_identity"): + disguised = copy.deepcopy(train[0]) + disguised["source_family"] = protected_item["source_family"] + assert_training_inputs_safe([], [disguised], protected) + with pytest.raises(LeakageError, match="protected_path"): + assert_training_inputs_safe([Path("vault/protected/test.json")], [], protected) + + +def test_failed_generation_writes_status_but_no_consumable_manifests( + tmp_path: Path, +) -> None: + source = load_source() + source["samples"][8]["group_id"] = source["samples"][0]["group_id"] + source_path = tmp_path / "source.json" + source_path.write_text(json.dumps(source), encoding="utf-8") + output = tmp_path / "out" + + with pytest.raises(LeakageError, match="Leakage gate failed"): + generate(source_path, output) + + status = json.loads((output / "generation-status.json").read_text(encoding="utf-8")) + assert status["status"] == "fail" + assert not (output / "development-split-manifest.json").exists() + assert not (output / "protected-split-manifest.json").exists() + + +def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together() -> ( + None +): + source = load_source() + source["assignment_mode"] = "deterministic_grouped" + source["split_assignment"] = { + "seed": "fixed-phase4-test-seed", + "roles": [ + "train", + "val", + "calibration", + "test", + "background-test", + "challenge", + ], + "weights": { + "train": 6, + "val": 2, + "calibration": 1, + "test": 2, + "background-test": 1, + "challenge": 1, + }, + "stratify_by": ["task"], + } + for item in source["samples"]: + item.pop("split") + source["samples"][1]["group_id"] = source["samples"][0]["group_id"] + + development, protected, leakage = build_manifests(source) + reversed_source = copy.deepcopy(source) + reversed_source["samples"].reverse() + reversed_development, reversed_protected, reversed_leakage = build_manifests( + reversed_source + ) + + assigned = { + item["sample_id"]: item["split"] + for item in development["samples"] + protected["samples"] + } + assert assigned["det-train-a"] == assigned["seg-train-a"] + assert set(leakage["split_counts"]) == { + "train", + "val", + "calibration", + "test", + "background-test", + "challenge", + } + assert leakage["status"] == "pass" + assert reversed_development["manifest_sha256"] == development["manifest_sha256"] + assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"] + assert reversed_leakage == leakage diff --git a/fixtures/accuracy/p4/protected-baseline-cases.json b/fixtures/accuracy/p4/protected-baseline-cases.json new file mode 100644 index 00000000..cd214e49 --- /dev/null +++ b/fixtures/accuracy/p4/protected-baseline-cases.json @@ -0,0 +1,881 @@ +{ + "schema_version": 2, + "portfolio_id": "geointel-p4-reference-harness-v2", + "split_roles": [ + "test", + "background-test" + ], + "selection_policy": "No data-dependent threshold, model or gate selection is performed from protected cases.", + "protected_policy": { + "operating_point_selection_allowed": false, + "diagnostic_curves_select_operating_point": false, + "test_feedback_allowed": false, + "threshold_selection_source": "pre_registered_configuration_only" + }, + "portfolio_lineage": { + "origin": "repository_fixture", + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "version": "1" + }, + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "cases": [ + { + "sample_id": "det-test-flanders-urban", + "split": "test", + "task": "object_detection", + "metadata": { + "region": "flanders", + "municipality": "Antwerpen", + "urbanity": "urban", + "object_size": "mixed", + "source": "fixture-orthophoto", + "sensor": "synthetic-rgb", + "resolution_m": 0.25, + "season": "summer", + "occlusion": "partial", + "vegetation": "moderate", + "difficulty": "hard", + "context": "dense_urban", + "tile_edge": false, + "label_review_state": "fixture_reviewed", + "ood": false, + "date": "2025-06-15" + }, + "config": { + "confidence_threshold": 0.5, + "match_iou": 0.5, + "fixed_diagnostic_risk_thresholds": [ + 0, + 0.5, + 0.7, + 0.9 + ] + }, + "lineage": { + "reference": { + "source_id": "synthetic:det-test-flanders-urban:reference", + "source_version": "1", + "derivation": "hand_authored_contract_fixture" + }, + "prediction": { + "source_id": "synthetic:det-test-flanders-urban:prediction", + "source_version": "1", + "derivation": "hand_authored_fixed_output" + } + }, + "classes": [ + "building" + ], + "references": [ + { + "id": "r1", + "class": "building", + "bbox": [ + 10, + 10, + 30, + 30 + ] + }, + { + "id": "r2", + "class": "building", + "bbox": [ + 50, + 50, + 80, + 80 + ] + } + ], + "predictions": [ + { + "id": "p1", + "class": "building", + "bbox": [ + 10, + 10, + 30, + 30 + ], + "confidence": 0.9 + }, + { + "id": "p2", + "class": "building", + "bbox": [ + 52, + 52, + 80, + 80 + ], + "confidence": 0.7 + }, + { + "id": "p3", + "class": "building", + "bbox": [ + 85, + 85, + 95, + 95 + ], + "confidence": 0.6 + }, + { + "id": "p4-filtered", + "class": "building", + "bbox": [ + 2, + 80, + 7, + 86 + ], + "confidence": 0.2 + } + ] + }, + { + "sample_id": "seg-test-wallonia-rural", + "split": "test", + "task": "footprint_segmentation", + "metadata": { + "region": "wallonia", + "municipality": "Namur", + "urbanity": "rural", + "object_size": "large", + "source": "fixture-picc", + "sensor": "synthetic-rgb", + "resolution_m": 0.5, + "season": "spring", + "occlusion": "none", + "vegetation": "low", + "difficulty": "normal", + "context": "rural_buildings", + "tile_edge": false, + "label_review_state": "fixture_reviewed", + "ood": false, + "date": "2025-04-20" + }, + "config": { + "match_iou": 0.5, + "boundary_tolerance_m": 1 + }, + "lineage": { + "reference": { + "source_id": "synthetic:seg-test-wallonia-rural:reference", + "source_version": "1", + "derivation": "hand_authored_contract_fixture" + }, + "prediction": { + "source_id": "synthetic:seg-test-wallonia-rural:prediction", + "source_version": "1", + "derivation": "hand_authored_fixed_output" + } + }, + "spatial_context": { + "crs": "EPSG:31370", + "coordinate_units": "m", + "metric": true + }, + "classes": [ + "building" + ], + "references": [ + { + "id": "r1", + "class": "building", + "polygon": [ + [ + 0, + 0 + ], + [ + 10, + 0 + ], + [ + 10, + 8 + ], + [ + 0, + 8 + ], + [ + 0, + 0 + ] + ] + } + ], + "predictions": [ + { + "id": "p1", + "class": "building", + "polygon": [ + [ + 0.5, + 0.5 + ], + [ + 9.5, + 0.5 + ], + [ + 9.5, + 8.5 + ], + [ + 0.5, + 8.5 + ], + [ + 0.5, + 0.5 + ] + ], + "confidence": 0.8 + } + ] + }, + { + "sample_id": "raster-test-brussels-urban", + "split": "test", + "task": "raster_classification", + "metadata": { + "region": "brussels", + "municipality": "Brussel", + "urbanity": "urban", + "object_size": "not_applicable", + "source": "fixture-thematic-raster", + "sensor": "synthetic-multispectral", + "resolution_m": 1, + "season": "autumn", + "occlusion": "not_applicable", + "vegetation": "moderate", + "difficulty": "hard", + "context": "dense_urban_raster", + "tile_edge": true, + "label_review_state": "fixture_reviewed", + "ood": false, + "date": "2025-10-05" + }, + "config": { + "class_order": [ + 0, + 1, + 2 + ], + "masked_pixels_excluded": true + }, + "lineage": { + "reference": { + "source_id": "synthetic:raster-test-brussels-urban:reference", + "source_version": "1", + "derivation": "hand_authored_contract_fixture" + }, + "prediction": { + "source_id": "synthetic:raster-test-brussels-urban:prediction", + "source_version": "1", + "derivation": "hand_authored_fixed_output" + } + }, + "classes": [ + 0, + 1, + 2 + ], + "raster_context": { + "reference": { + "crs": "EPSG:31370", + "transform": [ + 1, + 0, + 0, + 0, + -1, + 3 + ], + "shape": [ + 3, + 4 + ], + "nodata": null, + "mask": [ + [ + true, + true, + true, + true + ], + [ + true, + true, + true, + true + ], + [ + true, + true, + true, + true + ] + ] + }, + "prediction": { + "crs": "EPSG:31370", + "transform": [ + 1, + 0, + 0, + 0, + -1, + 3 + ], + "shape": [ + 3, + 4 + ], + "nodata": null, + "mask": [ + [ + true, + true, + true, + true + ], + [ + true, + true, + true, + true + ], + [ + true, + true, + true, + true + ] + ] + } + }, + "references": [ + [ + 0, + 0, + 1, + 1 + ], + [ + 0, + 1, + 1, + 2 + ], + [ + 2, + 2, + 1, + 0 + ] + ], + "predictions": [ + [ + 0, + 0, + 1, + 2 + ], + [ + 0, + 1, + 1, + 2 + ], + [ + 2, + 1, + 1, + 0 + ] + ] + }, + { + "sample_id": "vector-test-flanders-suburban", + "split": "test", + "task": "vector_comparison", + "metadata": { + "region": "flanders", + "municipality": "Mol", + "urbanity": "suburban", + "object_size": "mixed", + "source": "fixture-grb", + "sensor": "vector", + "resolution_m": 0.1, + "season": "not_applicable", + "occlusion": "not_applicable", + "vegetation": "low", + "difficulty": "normal", + "context": "suburban_buildings", + "tile_edge": false, + "label_review_state": "fixture_reviewed", + "ood": false, + "date": "2025-01-15" + }, + "config": { + "match_iou": 0.5 + }, + "lineage": { + "reference": { + "source_id": "synthetic:vector-test-flanders-suburban:reference", + "source_version": "1", + "derivation": "hand_authored_contract_fixture" + }, + "prediction": { + "source_id": "synthetic:vector-test-flanders-suburban:prediction", + "source_version": "1", + "derivation": "hand_authored_fixed_output" + } + }, + "spatial_context": { + "crs": "EPSG:31370", + "coordinate_units": "m", + "metric": true + }, + "classes": [ + "building" + ], + "references": [ + { + "id": "r1", + "class": "building", + "polygon": [ + [ + 0, + 0 + ], + [ + 4, + 0 + ], + [ + 4, + 4 + ], + [ + 0, + 4 + ], + [ + 0, + 0 + ] + ] + }, + { + "id": "r2", + "class": "building", + "polygon": [ + [ + 10, + 0 + ], + [ + 14, + 0 + ], + [ + 14, + 4 + ], + [ + 10, + 4 + ], + [ + 10, + 0 + ] + ] + } + ], + "predictions": [ + { + "id": "p1", + "class": "building", + "polygon": [ + [ + 0, + 0 + ], + [ + 4, + 0 + ], + [ + 4, + 4 + ], + [ + 0, + 4 + ], + [ + 0, + 0 + ] + ] + }, + { + "id": "p2", + "class": "building", + "polygon": [ + [ + 20, + 0 + ], + [ + 24, + 0 + ], + [ + 24, + 4 + ], + [ + 20, + 4 + ], + [ + 20, + 0 + ] + ] + } + ] + }, + { + "sample_id": "change-test-wallonia-industrial", + "split": "test", + "task": "change_detection", + "metadata": { + "region": "wallonia", + "municipality": "Liège", + "urbanity": "industrial", + "object_size": "mixed", + "source": "fixture-picc-temporal", + "sensor": "vector", + "resolution_m": 0.5, + "season": "multi-date", + "occlusion": "none", + "vegetation": "low", + "difficulty": "hard", + "context": "industrial_change", + "tile_edge": false, + "label_review_state": "fixture_reviewed", + "ood": false, + "date": "2024-05-01/2025-05-01" + }, + "config": { + "match_iou": 0.5 + }, + "lineage": { + "reference": { + "source_id": "synthetic:change-test-wallonia-industrial:reference", + "source_version": "1", + "derivation": "hand_authored_contract_fixture" + }, + "prediction": { + "source_id": "synthetic:change-test-wallonia-industrial:prediction", + "source_version": "1", + "derivation": "hand_authored_fixed_output" + } + }, + "classes": [ + "added", + "removed" + ], + "references": [ + { + "id": "r1", + "class": "added", + "bbox": [ + 0, + 0, + 5, + 5 + ] + }, + { + "id": "r2", + "class": "removed", + "bbox": [ + 10, + 0, + 15, + 5 + ] + } + ], + "predictions": [ + { + "id": "p1", + "class": "added", + "bbox": [ + 0, + 0, + 5, + 5 + ], + "confidence": 0.85 + }, + { + "id": "p2", + "class": "removed", + "bbox": [ + 20, + 0, + 25, + 5 + ], + "confidence": 0.65 + } + ] + }, + { + "sample_id": "terrain-test-flanders-rural", + "split": "test", + "task": "terrain_interpretation", + "metadata": { + "region": "flanders", + "municipality": "Hasselt", + "urbanity": "rural", + "object_size": "not_applicable", + "source": "fixture-dhmv", + "sensor": "elevation-raster", + "resolution_m": 5, + "season": "not_applicable", + "occlusion": "not_applicable", + "vegetation": "moderate", + "difficulty": "normal", + "context": "terrain_profile", + "tile_edge": false, + "label_review_state": "fixture_reviewed", + "ood": false, + "date": "2023-01-01" + }, + "config": { + "missing_prediction_policy": "exclude_and_reduce_coverage" + }, + "lineage": { + "reference": { + "source_id": "synthetic:terrain-test-flanders-rural:reference", + "source_version": "1", + "derivation": "hand_authored_contract_fixture" + }, + "prediction": { + "source_id": "synthetic:terrain-test-flanders-rural:prediction", + "source_version": "1", + "derivation": "hand_authored_fixed_output" + } + }, + "units": "m_TAW", + "references": [ + 12, + 13, + 14, + 15, + 16 + ], + "predictions": [ + 12.2, + 12.8, + 14.4, + null, + 15.7 + ] + }, + { + "sample_id": "validation-test-national", + "split": "test", + "task": "geospatial_data_validation", + "metadata": { + "region": "national", + "municipality": "not_applicable", + "urbanity": "mixed", + "object_size": "not_applicable", + "source": "fixture-contracts", + "sensor": "mixed", + "resolution_m": null, + "season": "not_applicable", + "occlusion": "not_applicable", + "vegetation": "mixed", + "difficulty": "hard", + "context": "data_contract", + "tile_edge": false, + "label_review_state": "fixture_reviewed", + "ood": false, + "date": "2025-01-01" + }, + "config": { + "critical_severities": [ + "blocker", + "critical" + ] + }, + "lineage": { + "reference": { + "source_id": "synthetic:validation-test-national:reference", + "source_version": "1", + "derivation": "hand_authored_contract_fixture" + }, + "prediction": { + "source_id": "synthetic:validation-test-national:prediction", + "source_version": "1", + "derivation": "hand_authored_fixed_output" + } + }, + "expected_anomalies": [ + { + "code": "D-CRS-WRONG", + "severity": "blocker" + }, + { + "code": "D-INVALID-GEOMETRY", + "severity": "critical" + }, + { + "code": "D-MISSING-PROVENANCE", + "severity": "critical" + } + ], + "observed_anomalies": [ + { + "code": "D-CRS-WRONG", + "severity": "blocker" + }, + { + "code": "D-INVALID-GEOMETRY", + "severity": "critical" + }, + { + "code": "D-RESOLUTION", + "severity": "major" + } + ] + }, + { + "sample_id": "background-test-pure-empty", + "split": "background-test", + "task": "object_detection", + "metadata": { + "region": "brussels", + "municipality": "Brussel", + "urbanity": "urban", + "object_size": "empty", + "source": "fixture-orthophoto", + "sensor": "synthetic-rgb", + "resolution_m": 0.25, + "season": "winter", + "occlusion": "none", + "vegetation": "low", + "difficulty": "hard", + "context": "pure_background", + "tile_edge": false, + "label_review_state": "fixture_reviewed", + "ood": false, + "date": "2025-02-01" + }, + "config": { + "confidence_threshold": 0.5, + "match_iou": 0.5, + "fixed_diagnostic_risk_thresholds": [ + 0, + 0.5, + 0.7, + 0.9 + ] + }, + "lineage": { + "reference": { + "source_id": "synthetic:background-test-pure-empty:reference", + "source_version": "1", + "derivation": "hand_authored_contract_fixture" + }, + "prediction": { + "source_id": "synthetic:background-test-pure-empty:prediction", + "source_version": "1", + "derivation": "hand_authored_fixed_output" + } + }, + "classes": [ + "building" + ], + "references": [], + "predictions": [] + }, + { + "sample_id": "background-test-hard-negative", + "split": "background-test", + "task": "object_detection", + "metadata": { + "region": "flanders", + "municipality": "Oostende", + "urbanity": "urban", + "object_size": "empty", + "source": "fixture-orthophoto", + "sensor": "synthetic-rgb", + "resolution_m": 0.25, + "season": "summer", + "occlusion": "none", + "vegetation": "moderate", + "difficulty": "hard", + "context": "coastal_hard_negative", + "tile_edge": true, + "label_review_state": "fixture_reviewed", + "ood": false, + "date": "2025-07-01" + }, + "config": { + "confidence_threshold": 0.5, + "match_iou": 0.5, + "fixed_diagnostic_risk_thresholds": [ + 0, + 0.5, + 0.7, + 0.9 + ] + }, + "lineage": { + "reference": { + "source_id": "synthetic:background-test-hard-negative:reference", + "source_version": "1", + "derivation": "hand_authored_contract_fixture" + }, + "prediction": { + "source_id": "synthetic:background-test-hard-negative:prediction", + "source_version": "1", + "derivation": "hand_authored_fixed_output" + } + }, + "classes": [ + "building" + ], + "references": [], + "predictions": [ + { + "id": "p-hard-fp", + "class": "building", + "bbox": [ + 20, + 20, + 35, + 35 + ], + "confidence": 0.92 + } + ] + } + ] +} diff --git a/fixtures/accuracy/p4/split-source-manifest.json b/fixtures/accuracy/p4/split-source-manifest.json new file mode 100644 index 00000000..8650f367 --- /dev/null +++ b/fixtures/accuracy/p4/split-source-manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "dataset_version": "geointel-p4-harness-fixture-v2", + "crs": "EPSG:31370", + "independence_buffer_m": 2000.0, + "perceptual_hamming_threshold": 4, + "label_geometry_hamming_threshold": 2, + "required_splits": ["train", "val", "calibration", "test", "background-test", "challenge"], + "assignment_mode": "preassigned", + "claim_boundary": "Synthetic contract fixtures for evaluator regression only; never production accuracy evidence.", + "samples": [ + {"sample_id":"det-train-a","task":"object_detection","split":"train","group_id":"g01","source_family":"scene-01","temporal_family":"temporal-01","object_ids":["o01"],"bbox":[10000,10000,10100,10100],"raw_image_sha256":"0101010101010101010101010101010101010101010101010101010101010101","processed_image_sha256":"2121212121212121212121212121212121212121212121212121212121212121","label_sha256":"4141414141414141414141414141414141414141414141414141414141414141","perceptual_image_hash":"dcef24412401d4f9","label_geometry_hash":"6161616161616161616161616161616161616161616161616161616161616161","label_geometry_fingerprint":"c0f62e16b4033ee9","native_feature_ids":["native-01-1"],"parent_raster_id":"parent-raster-01","acquisition_id":"acquisition-01","acquisition_date":"2025-01-01"}, + {"sample_id":"seg-train-a","task":"footprint_segmentation","split":"train","group_id":"g02","source_family":"scene-02","temporal_family":"temporal-02","object_ids":["o02"],"bbox":[20000,10000,20100,10100],"raw_image_sha256":"0202020202020202020202020202020202020202020202020202020202020202","processed_image_sha256":"2222222222222222222222222222222222222222222222222222222222222222","label_sha256":"4242424242424242424242424242424242424242424242424242424242424242","perceptual_image_hash":"030bcdd3bc87412e","label_geometry_hash":"6262626262626262626262626262626262626262626262626262626262626262","label_geometry_fingerprint":"666a0fc2d26265ee","native_feature_ids":["native-02-1"],"parent_raster_id":"parent-raster-02","acquisition_id":"acquisition-02","acquisition_date":"2025-01-02"}, + {"sample_id":"raster-train-a","task":"raster_classification","split":"train","group_id":"g03","source_family":"scene-03","temporal_family":"temporal-03","object_ids":["o03"],"bbox":[30000,10000,30100,10100],"raw_image_sha256":"0303030303030303030303030303030303030303030303030303030303030303","processed_image_sha256":"2323232323232323232323232323232323232323232323232323232323232323","label_sha256":"4343434343434343434343434343434343434343434343434343434343434343","perceptual_image_hash":"75e36e008b4a6702","label_geometry_hash":"6363636363636363636363636363636363636363636363636363636363636363","label_geometry_fingerprint":"30820f2c124a3ff0","native_feature_ids":["native-03-1"],"parent_raster_id":"parent-raster-03","acquisition_id":"acquisition-03","acquisition_date":"2025-01-03"}, + {"sample_id":"val-vector-a","task":"vector_comparison","split":"val","group_id":"g04","source_family":"scene-04","temporal_family":"temporal-04","object_ids":["o04"],"bbox":[10000,30000,10100,30100],"raw_image_sha256":"0404040404040404040404040404040404040404040404040404040404040404","processed_image_sha256":"2424242424242424242424242424242424242424242424242424242424242424","label_sha256":"4444444444444444444444444444444444444444444444444444444444444444","perceptual_image_hash":"2e3b331cc01536c9","label_geometry_hash":"6464646464646464646464646464646464646464646464646464646464646464","label_geometry_fingerprint":"a559b46cba64e501","native_feature_ids":["native-04-1"],"parent_raster_id":"parent-raster-04","acquisition_id":"acquisition-04","acquisition_date":"2025-01-04"}, + {"sample_id":"val-change-b","task":"change_detection","split":"val","group_id":"g05","source_family":"scene-05","temporal_family":"temporal-05","object_ids":["o05"],"bbox":[30000,30000,30100,30100],"raw_image_sha256":"0505050505050505050505050505050505050505050505050505050505050505","processed_image_sha256":"2525252525252525252525252525252525252525252525252525252525252525","label_sha256":"4545454545454545454545454545454545454545454545454545454545454545","perceptual_image_hash":"828b99671c3f2f39","label_geometry_hash":"6565656565656565656565656565656565656565656565656565656565656565","label_geometry_fingerprint":"14ceff3640a013eb","native_feature_ids":["native-05-1"],"parent_raster_id":"parent-raster-05","acquisition_id":"acquisition-05","acquisition_date":"2025-01-05"}, + {"sample_id":"val-terrain-c","task":"terrain_interpretation","split":"val","group_id":"g06","source_family":"scene-06","temporal_family":"temporal-06","object_ids":["o06"],"bbox":[50000,30000,50100,30100],"raw_image_sha256":"0606060606060606060606060606060606060606060606060606060606060606","processed_image_sha256":"2626262626262626262626262626262626262626262626262626262626262626","label_sha256":"4646464646464646464646464646464646464646464646464646464646464646","perceptual_image_hash":"11c11105b0d10635","label_geometry_hash":"6666666666666666666666666666666666666666666666666666666666666666","label_geometry_fingerprint":"a1d9bbb93ba6d6da","native_feature_ids":["native-06-1"],"parent_raster_id":"parent-raster-06","acquisition_id":"acquisition-06","acquisition_date":"2025-01-06"}, + {"sample_id":"calibration-det-a","task":"object_detection","split":"calibration","group_id":"g07","source_family":"scene-07","temporal_family":"temporal-07","object_ids":["o21"],"bbox":[70000,30000,70100,30100],"raw_image_sha256":"0707070707070707070707070707070707070707070707070707070707070707","processed_image_sha256":"2727272727272727272727272727272727272727272727272727272727272727","label_sha256":"4747474747474747474747474747474747474747474747474747474747474747","perceptual_image_hash":"c4316514fed98f6b","label_geometry_hash":"6767676767676767676767676767676767676767676767676767676767676767","label_geometry_fingerprint":"dba609f34cb9826b","native_feature_ids":["native-07-1"],"parent_raster_id":"parent-raster-07","acquisition_id":"acquisition-07","acquisition_date":"2025-01-07"}, + {"sample_id":"calibration-seg-b","task":"footprint_segmentation","split":"calibration","group_id":"g08","source_family":"scene-08","temporal_family":"temporal-08","object_ids":["o22"],"bbox":[90000,30000,90100,30100],"raw_image_sha256":"0808080808080808080808080808080808080808080808080808080808080808","processed_image_sha256":"2828282828282828282828282828282828282828282828282828282828282828","label_sha256":"4848484848484848484848484848484848484848484848484848484848484848","perceptual_image_hash":"518ba3145333c08c","label_geometry_hash":"6868686868686868686868686868686868686868686868686868686868686868","label_geometry_fingerprint":"627df87eea48aec0","native_feature_ids":["native-08-1"],"parent_raster_id":"parent-raster-08","acquisition_id":"acquisition-08","acquisition_date":"2025-01-08"}, + {"sample_id":"det-test-flanders-urban","task":"object_detection","split":"test","group_id":"g09","source_family":"scene-09","temporal_family":"temporal-09","object_ids":["o07","o08"],"bbox":[10000,50000,10100,50100],"raw_image_sha256":"0909090909090909090909090909090909090909090909090909090909090909","processed_image_sha256":"2929292929292929292929292929292929292929292929292929292929292929","label_sha256":"4949494949494949494949494949494949494949494949494949494949494949","perceptual_image_hash":"d6d20ed34050b89e","label_geometry_hash":"6969696969696969696969696969696969696969696969696969696969696969","label_geometry_fingerprint":"7ed221e11cc6828d","native_feature_ids":["native-09-1","native-09-2"],"parent_raster_id":"parent-raster-09","acquisition_id":"acquisition-09","acquisition_date":"2025-01-09"}, + {"sample_id":"seg-test-wallonia-rural","task":"footprint_segmentation","split":"test","group_id":"g10","source_family":"scene-10","temporal_family":"temporal-10","object_ids":["o09"],"bbox":[20000,50000,20100,50100],"raw_image_sha256":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","processed_image_sha256":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","label_sha256":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","perceptual_image_hash":"6c476d18e0539687","label_geometry_hash":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","label_geometry_fingerprint":"e9fb3c93de82d0be","native_feature_ids":["native-10-1"],"parent_raster_id":"parent-raster-10","acquisition_id":"acquisition-10","acquisition_date":"2025-01-10"}, + {"sample_id":"raster-test-brussels-urban","task":"raster_classification","split":"test","group_id":"g11","source_family":"scene-11","temporal_family":"temporal-11","object_ids":["o10"],"bbox":[30000,50000,30100,50100],"raw_image_sha256":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","processed_image_sha256":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","label_sha256":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","perceptual_image_hash":"4383fa06c328f326","label_geometry_hash":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","label_geometry_fingerprint":"f4269ac17b100571","native_feature_ids":["native-11-1"],"parent_raster_id":"parent-raster-11","acquisition_id":"acquisition-11","acquisition_date":"2025-01-11"}, + {"sample_id":"vector-test-flanders-suburban","task":"vector_comparison","split":"test","group_id":"g12","source_family":"scene-12","temporal_family":"temporal-12","object_ids":["o11","o12"],"bbox":[40000,50000,40100,50100],"raw_image_sha256":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","processed_image_sha256":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","label_sha256":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","perceptual_image_hash":"1236e1aa148c25d5","label_geometry_hash":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","label_geometry_fingerprint":"2db7d069691934ed","native_feature_ids":["native-12-1","native-12-2"],"parent_raster_id":"parent-raster-12","acquisition_id":"acquisition-12","acquisition_date":"2025-01-12"}, + {"sample_id":"change-test-wallonia-industrial","task":"change_detection","split":"test","group_id":"g13","source_family":"scene-13","temporal_family":"temporal-13","object_ids":["o13","o14"],"bbox":[50000,50000,50100,50100],"raw_image_sha256":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","processed_image_sha256":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","label_sha256":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","perceptual_image_hash":"14e347040ef16eac","label_geometry_hash":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","label_geometry_fingerprint":"6c5b5f9db0d73c31","native_feature_ids":["native-13-1","native-13-2"],"parent_raster_id":"parent-raster-13","acquisition_id":"acquisition-13","acquisition_date":"2025-01-13"}, + {"sample_id":"terrain-test-flanders-rural","task":"terrain_interpretation","split":"test","group_id":"g14","source_family":"scene-14","temporal_family":"temporal-14","object_ids":["o15"],"bbox":[60000,50000,60100,50100],"raw_image_sha256":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","processed_image_sha256":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","label_sha256":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","perceptual_image_hash":"c808940b0a325e9c","label_geometry_hash":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","label_geometry_fingerprint":"4ec9f1dd6c799ec7","native_feature_ids":["native-14-1"],"parent_raster_id":"parent-raster-14","acquisition_id":"acquisition-14","acquisition_date":"2025-01-14"}, + {"sample_id":"validation-test-national","task":"geospatial_data_validation","split":"test","group_id":"g15","source_family":"scene-15","temporal_family":"temporal-15","object_ids":["o16"],"bbox":[70000,50000,70100,50100],"raw_image_sha256":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","processed_image_sha256":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","label_sha256":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","perceptual_image_hash":"4365e6b7df6acf3d","label_geometry_hash":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","label_geometry_fingerprint":"616691796f62dbbb","native_feature_ids":["native-15-1"],"parent_raster_id":"parent-raster-15","acquisition_id":"acquisition-15","acquisition_date":"2025-01-15"}, + {"sample_id":"background-test-pure-empty","task":"object_detection","split":"background-test","group_id":"g16","source_family":"scene-16","temporal_family":"temporal-16","object_ids":[],"bbox":[20000,70000,20100,70100],"raw_image_sha256":"1010101010101010101010101010101010101010101010101010101010101010","processed_image_sha256":"3030303030303030303030303030303030303030303030303030303030303030","label_sha256":"5050505050505050505050505050505050505050505050505050505050505050","perceptual_image_hash":"be4078e1ef9fa296","label_geometry_hash":"7070707070707070707070707070707070707070707070707070707070707070","label_geometry_fingerprint":"004e762f898c4fad","native_feature_ids":[],"parent_raster_id":"parent-raster-16","acquisition_id":"acquisition-16","acquisition_date":"2025-01-16"}, + {"sample_id":"background-test-hard-negative","task":"object_detection","split":"background-test","group_id":"g17","source_family":"scene-17","temporal_family":"temporal-17","object_ids":[],"bbox":[60000,70000,60100,70100],"raw_image_sha256":"1111111111111111111111111111111111111111111111111111111111111111","processed_image_sha256":"3131313131313131313131313131313131313131313131313131313131313131","label_sha256":"5151515151515151515151515151515151515151515151515151515151515151","perceptual_image_hash":"6ada1b5aa46cc261","label_geometry_hash":"7171717171717171717171717171717171717171717171717171717171717171","label_geometry_fingerprint":"2b405bcc9c95acfb","native_feature_ids":[],"parent_raster_id":"parent-raster-17","acquisition_id":"acquisition-17","acquisition_date":"2025-01-17"}, + {"sample_id":"det-challenge-coast","task":"object_detection","split":"challenge","group_id":"g18","source_family":"scene-18","temporal_family":"temporal-18","object_ids":["o17"],"bbox":[10000,90000,10100,90100],"raw_image_sha256":"1212121212121212121212121212121212121212121212121212121212121212","processed_image_sha256":"3232323232323232323232323232323232323232323232323232323232323232","label_sha256":"5252525252525252525252525252525252525252525252525252525252525252","perceptual_image_hash":"15910b3a2d056648","label_geometry_hash":"7272727272727272727272727272727272727272727272727272727272727272","label_geometry_fingerprint":"95518e2583b3a7d3","native_feature_ids":["native-18-1"],"parent_raster_id":"parent-raster-18","acquisition_id":"acquisition-18","acquisition_date":"2025-01-18"}, + {"sample_id":"seg-challenge-brussels","task":"footprint_segmentation","split":"challenge","group_id":"g19","source_family":"scene-19","temporal_family":"temporal-19","object_ids":["o18"],"bbox":[30000,90000,30100,90100],"raw_image_sha256":"1313131313131313131313131313131313131313131313131313131313131313","processed_image_sha256":"3333333333333333333333333333333333333333333333333333333333333333","label_sha256":"5353535353535353535353535353535353535353535353535353535353535353","perceptual_image_hash":"b5c6e201d687cf9e","label_geometry_hash":"7373737373737373737373737373737373737373737373737373737373737373","label_geometry_fingerprint":"2625fe765bce389e","native_feature_ids":["native-19-1"],"parent_raster_id":"parent-raster-19","acquisition_id":"acquisition-19","acquisition_date":"2025-01-19"}, + {"sample_id":"raster-challenge-seasonal","task":"raster_classification","split":"challenge","group_id":"g20","source_family":"scene-20","temporal_family":"temporal-20","object_ids":["o19"],"bbox":[50000,90000,50100,90100],"raw_image_sha256":"1414141414141414141414141414141414141414141414141414141414141414","processed_image_sha256":"3434343434343434343434343434343434343434343434343434343434343434","label_sha256":"5454545454545454545454545454545454545454545454545454545454545454","perceptual_image_hash":"85f82f2f8eb2c2f5","label_geometry_hash":"7474747474747474747474747474747474747474747474747474747474747474","label_geometry_fingerprint":"e91bcaae1021dccc","native_feature_ids":["native-20-1"],"parent_raster_id":"parent-raster-20","acquisition_id":"acquisition-20","acquisition_date":"2025-01-20"}, + {"sample_id":"validation-challenge-corrupt","task":"geospatial_data_validation","split":"challenge","group_id":"g21","source_family":"scene-21","temporal_family":"temporal-21","object_ids":["o20"],"bbox":[70000,90000,70100,90100],"raw_image_sha256":"1515151515151515151515151515151515151515151515151515151515151515","processed_image_sha256":"3535353535353535353535353535353535353535353535353535353535353535","label_sha256":"5555555555555555555555555555555555555555555555555555555555555555","perceptual_image_hash":"c71055ef6baeee8b","label_geometry_hash":"7575757575757575757575757575757575757575757575757575757575757575","label_geometry_fingerprint":"686f1d8a92703709","native_feature_ids":["native-21-1"],"parent_raster_id":"parent-raster-21","acquisition_id":"acquisition-21","acquisition_date":"2025-01-21"} + ] +} diff --git a/scripts/accuracy_phase4_evaluator.py b/scripts/accuracy_phase4_evaluator.py new file mode 100644 index 00000000..b0740056 --- /dev/null +++ b/scripts/accuracy_phase4_evaluator.py @@ -0,0 +1,1982 @@ +"""Fail-closed, task-aware metrics for the GeoIntel Phase 4 benchmark. + +The bundled portfolio is a synthetic contract fixture. Nothing emitted by +this module is, by itself, evidence of production-model accuracy. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections import defaultdict +from pathlib import Path +from statistics import mean, median +from typing import Any, Callable, Iterable + +EVALUATOR_VERSION = "2.0.0" +REPORT_SCHEMA_VERSION = 2 +SUBGROUP_MIN_CASE_SUPPORT = 5 +CANONICAL_JSON_SPEC = ( + "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), " + "ensure_ascii=false and allow_nan=false" +) + +TASKS = { + "object_detection", + "footprint_segmentation", + "raster_classification", + "vector_comparison", + "change_detection", + "terrain_interpretation", + "geospatial_data_validation", +} +VALID_ANOMALY_SEVERITIES = { + "blocker", + "critical", + "major", + "minor", + "informational", +} +ERROR_CODES = { + "false_positive": "M-FP-CONFUSER", + "false_negative": "M-FN-MISSED", + "event_false_positive": "M-FP-CONFUSER", + "event_false_negative": "M-FN-MISSED", + "validation_false_positive": "D-VALIDATION-FP", + "validation_false_negative": "D-VALIDATION-FN", + "terrain_missing": "P-PARTIAL", +} + + +def canonical_json_bytes(value: Any) -> bytes: + """Return the exact canonical byte representation used by all hashes.""" + + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def canonical_hash(value: Any) -> str: + return hashlib.sha256(canonical_json_bytes(value)).hexdigest() + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def safe_rate(numerator: int | float, denominator: int | float) -> float | None: + return float(numerator) / float(denominator) if denominator else None + + +def f1_score(precision: float | None, recall: float | None) -> float | None: + if precision is None or recall is None or precision + recall == 0: + return None + return 2 * precision * recall / (precision + recall) + + +def count_metrics(tp: int, fp: int, fn: int) -> dict[str, Any]: + precision = safe_rate(tp, tp + fp) + recall = safe_rate(tp, tp + fn) + return { + "true_positive": tp, + "false_positive": fp, + "false_negative": fn, + "prediction_count": tp + fp, + "reference_count": tp + fn, + "precision": precision, + "recall": recall, + "f1": f1_score(precision, recall), + "false_discovery_rate": safe_rate(fp, tp + fp), + "miss_rate": safe_rate(fn, tp + fn), + "precision_ci95_wilson": wilson_interval(tp, tp + fp), + "recall_ci95_wilson": wilson_interval(tp, tp + fn), + } + + +def wilson_interval( + successes: int, total: int, z: float = 1.959963984540054 +) -> dict[str, Any]: + if total <= 0: + return {"status": "undefined", "lower": None, "upper": None, "support": total} + proportion = successes / total + denominator = 1 + z * z / total + centre = (proportion + z * z / (2 * total)) / denominator + margin = ( + z + * math.sqrt((proportion * (1 - proportion) + z * z / (4 * total)) / total) + / denominator + ) + return { + "status": "computed", + "lower": max(0.0, centre - margin), + "upper": min(1.0, centre + margin), + "support": total, + "method": "Wilson score interval", + } + + +def _finite_float(value: Any, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{label} must be a finite number") + converted = float(value) + if not math.isfinite(converted): + raise ValueError(f"{label} must be a finite number") + return converted + + +def _probability(value: Any, label: str) -> float: + converted = _finite_float(value, label) + if not 0.0 <= converted <= 1.0: + raise ValueError(f"{label} must be between 0 and 1") + return converted + + +def _nonempty_mapping(value: Any, label: str) -> dict[str, Any]: + if not isinstance(value, dict) or not value: + raise ValueError(f"{label} must be a non-empty object") + return value + + +def _validate_case_common(case: dict[str, Any]) -> None: + sample_id = case.get("sample_id") + if not isinstance(sample_id, str) or not sample_id.strip(): + raise ValueError("Every case requires a non-empty sample_id") + if case.get("task") not in TASKS: + raise ValueError(f"{sample_id}: unsupported task {case.get('task')!r}") + if not isinstance(case.get("metadata"), dict): + raise ValueError(f"{sample_id}: metadata must be an object") + if not isinstance(case.get("config"), dict): + raise ValueError(f"{sample_id}: config must be an explicit object") + _nonempty_mapping(case.get("lineage"), f"{sample_id}: lineage") + + +def _stable_id(item: dict[str, Any], label: str) -> str: + identifier = item.get("id") + if not isinstance(identifier, str) or not identifier.strip(): + raise ValueError(f"{label} requires a non-empty string id") + return identifier + + +def _validate_unique_ids(items: list[dict[str, Any]], label: str) -> None: + identifiers = [ + _stable_id(item, f"{label}[{index}]") for index, item in enumerate(items) + ] + duplicates = sorted( + {identifier for identifier in identifiers if identifiers.count(identifier) > 1} + ) + if duplicates: + raise ValueError(f"{label} contains duplicate ids: {duplicates}") + + +def _validate_bbox(value: Any, label: str) -> list[float]: + if not isinstance(value, list) or len(value) != 4: + raise ValueError(f"{label} must be [min_x, min_y, max_x, max_y]") + bbox = [ + _finite_float(coordinate, f"{label}[{index}]") + for index, coordinate in enumerate(value) + ] + if bbox[2] <= bbox[0] or bbox[3] <= bbox[1]: + raise ValueError(f"{label} must have positive width and height") + return bbox + + +def _validate_classes(case: dict[str, Any]) -> list[str | int]: + sample_id = case["sample_id"] + classes = case.get("classes") + if not isinstance(classes, list) or not classes: + raise ValueError(f"{sample_id}: classes must be a non-empty list") + if any( + isinstance(value, bool) or not isinstance(value, (str, int)) + for value in classes + ): + raise ValueError(f"{sample_id}: classes may contain only strings or integers") + if len({canonical_hash(value) for value in classes}) != len(classes): + raise ValueError(f"{sample_id}: classes must be unique") + if len({type(value) for value in classes}) != 1: + raise ValueError(f"{sample_id}: class values must use one JSON scalar type") + return classes + + +def _validate_labeled_items( + case: dict[str, Any], + items: list[dict[str, Any]], + label: str, +) -> None: + classes = _validate_classes(case) + for index, item in enumerate(items): + if item.get("class") not in classes: + raise ValueError( + f"{case['sample_id']}: {label}[{index}].class is outside the " + "declared ontology" + ) + + +def _class_compatible(left: dict[str, Any], right: dict[str, Any]) -> bool: + return left.get("class") == right.get("class") + + +def bbox_iou(left: list[float], right: list[float]) -> float: + x1, y1 = max(left[0], right[0]), max(left[1], right[1]) + x2, y2 = min(left[2], right[2]), min(left[3], right[3]) + intersection = max(0.0, x2 - x1) * max(0.0, y2 - y1) + left_area = max(0.0, left[2] - left[0]) * max(0.0, left[3] - left[1]) + right_area = max(0.0, right[2] - right[0]) * max(0.0, right[3] - right[1]) + union = left_area + right_area - intersection + return intersection / union if union > 0 else 0.0 + + +def greedy_match( + predictions: list[dict[str, Any]], + references: list[dict[str, Any]], + overlap: Callable[[dict[str, Any], dict[str, Any]], float], + threshold: float, + *, + class_aware: bool = False, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]: + unmatched = set(range(len(references))) + matches: list[dict[str, Any]] = [] + false_positives: list[dict[str, Any]] = [] + ordered = sorted( + predictions, + key=lambda item: ( + -float(item.get("confidence", 1.0)), + _stable_id(item, "prediction"), + ), + ) + for prediction in ordered: + candidates: list[tuple[float, str, int]] = [] + for index in sorted(unmatched): + reference = references[index] + if class_aware and not _class_compatible(prediction, reference): + continue + candidates.append( + ( + overlap(prediction, reference), + _stable_id(reference, "reference"), + index, + ) + ) + if candidates: + score, _reference_id, index = min( + candidates, key=lambda item: (-item[0], item[1]) + ) + else: + score, index = 0.0, -1 + if index >= 0 and score >= threshold: + unmatched.remove(index) + match = { + "prediction_id": prediction["id"], + "reference_id": references[index]["id"], + "overlap": score, + "confidence": prediction.get("confidence"), + } + if class_aware: + match["class"] = prediction["class"] + matches.append(match) + else: + false_positives.append(prediction) + false_negatives = [references[index] for index in sorted(unmatched)] + return matches, false_positives, false_negatives + + +def interpolated_ap( + points: list[tuple[float, str, int]], + reference_count: int, +) -> float | None: + if reference_count <= 0: + return None + ordered = sorted(points, key=lambda item: (-item[0], item[1])) + true_positive = 0 + false_positive = 0 + curve: list[tuple[float, float]] = [] + for _confidence, _stable_prediction_id, correct in ordered: + true_positive += correct + false_positive += 1 - correct + curve.append( + ( + true_positive / reference_count, + true_positive / (true_positive + false_positive), + ) + ) + values = [] + for step in range(101): + recall_level = step / 100 + values.append( + max( + (precision for recall, precision in curve if recall >= recall_level), + default=0.0, + ) + ) + return sum(values) / len(values) + + +def detection_ap( + predictions: list[dict[str, Any]], + references: list[dict[str, Any]], + iou_threshold: float, +) -> float | None: + unmatched = set(range(len(references))) + points: list[tuple[float, str, int]] = [] + ordered = sorted( + predictions, + key=lambda item: ( + -float(item["confidence"]), + _stable_id(item, "prediction"), + ), + ) + for prediction in ordered: + candidates: list[tuple[float, str, int]] = [] + for index in sorted(unmatched): + reference = references[index] + if not _class_compatible(prediction, reference): + continue + candidates.append( + ( + bbox_iou(prediction["bbox"], reference["bbox"]), + _stable_id(reference, "reference"), + index, + ) + ) + if candidates: + score, _reference_id, index = min( + candidates, key=lambda item: (-item[0], item[1]) + ) + else: + score, index = 0.0, -1 + correct = int(index >= 0 and score >= iou_threshold) + if correct: + unmatched.remove(index) + points.append((float(prediction["confidence"]), prediction["id"], correct)) + return interpolated_ap(points, len(references)) + + +def calibration_metrics( + predictions: list[dict[str, Any]], matches: list[dict[str, Any]], bins: int = 5 +) -> dict[str, Any]: + matched_ids = {item["prediction_id"] for item in matches} + scored = [ + (float(item["confidence"]), 1 if item.get("id") in matched_ids else 0) + for item in predictions + ] + if not scored: + return {"ece": None, "brier": None, "bins": [], "status": "undefined"} + blocks = [] + ece = 0.0 + for index in range(bins): + lower = index / bins + upper = (index + 1) / bins + selected = [ + (confidence, correct) + for confidence, correct in scored + if lower <= confidence <= upper + and (index == bins - 1 or confidence < upper) + ] + if not selected: + blocks.append( + { + "lower": lower, + "upper": upper, + "count": 0, + "mean_confidence": None, + "accuracy": None, + } + ) + continue + avg_confidence = mean(item[0] for item in selected) + accuracy = mean(item[1] for item in selected) + ece += len(selected) / len(scored) * abs(accuracy - avg_confidence) + blocks.append( + { + "lower": lower, + "upper": upper, + "count": len(selected), + "mean_confidence": avg_confidence, + "accuracy": accuracy, + } + ) + return { + "status": "computed", + "ece": ece, + "brier": mean((confidence - correct) ** 2 for confidence, correct in scored), + "bins": blocks, + "binning": "five fixed equal-width bins", + } + + +def coverage_risk( + predictions: list[dict[str, Any]], + references: list[dict[str, Any]], + match_iou: float, + operating_threshold: float, +) -> list[dict[str, Any]]: + """Report retention, reference coverage and risk including every FN.""" + + thresholds = sorted({0.0, 0.5, 0.7, 0.9, operating_threshold}) + rows = [] + for threshold in thresholds: + retained = [ + item for item in predictions if float(item["confidence"]) >= threshold + ] + matches, false_positives, false_negatives = greedy_match( + retained, + references, + lambda prediction, reference: bbox_iou( + prediction["bbox"], reference["bbox"] + ), + match_iou, + class_aware=True, + ) + tp = len(matches) + fp = len(false_positives) + fn = len(false_negatives) + rows.append( + { + "threshold": threshold, + "retained_prediction_count": len(retained), + "total_prediction_count": len(predictions), + "retained_prediction_coverage": safe_rate( + len(retained), len(predictions) + ), + "matched_reference_count": tp, + "reference_count": len(references), + "reference_coverage": safe_rate(tp, len(references)), + "false_positive_count": fp, + "false_negative_count": fn, + "risk": safe_rate(fp + fn, tp + fp + fn), + "risk_definition": ( + "(false_positive + false_negative) / " + "(true_positive + false_positive + false_negative)" + ), + } + ) + return rows + + +def _mean_or_none(values: Iterable[float | None]) -> float | None: + retained = [value for value in values if value is not None] + return mean(retained) if retained else None + + +def _validate_detection_case( + case: dict[str, Any], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + references = case.get("references") + predictions = case.get("predictions") + if not isinstance(references, list) or not all( + isinstance(item, dict) for item in references + ): + raise ValueError(f"{case['sample_id']}: references must be an object list") + if not isinstance(predictions, list) or not all( + isinstance(item, dict) for item in predictions + ): + raise ValueError(f"{case['sample_id']}: predictions must be an object list") + _validate_unique_ids(references, f"{case['sample_id']}.references") + _validate_unique_ids(predictions, f"{case['sample_id']}.predictions") + _validate_labeled_items(case, references, "references") + _validate_labeled_items(case, predictions, "predictions") + for index, item in enumerate(references): + _validate_bbox( + item.get("bbox"), + f"{case['sample_id']}.references[{index}].bbox", + ) + for index, item in enumerate(predictions): + _validate_bbox( + item.get("bbox"), + f"{case['sample_id']}.predictions[{index}].bbox", + ) + _probability( + item.get("confidence"), + f"{case['sample_id']}.predictions[{index}].confidence", + ) + return references, predictions + + +def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]: + references, predictions = _validate_detection_case(case) + threshold = _probability( + case["config"].get("confidence_threshold"), + f"{case['sample_id']}.config.confidence_threshold", + ) + match_iou = _probability( + case["config"].get("match_iou"), + f"{case['sample_id']}.config.match_iou", + ) + retained = [item for item in predictions if float(item["confidence"]) >= threshold] + matches, false_positives, false_negatives = greedy_match( + retained, + references, + lambda prediction, reference: bbox_iou(prediction["bbox"], reference["bbox"]), + match_iou, + class_aware=True, + ) + per_class = {} + for class_value in case["classes"]: + class_predictions = [item for item in retained if item["class"] == class_value] + all_class_predictions = [ + item for item in predictions if item["class"] == class_value + ] + class_references = [item for item in references if item["class"] == class_value] + class_matches, class_fp, class_fn = greedy_match( + class_predictions, + class_references, + lambda prediction, reference: bbox_iou( + prediction["bbox"], reference["bbox"] + ), + match_iou, + class_aware=True, + ) + class_metrics = count_metrics(len(class_matches), len(class_fp), len(class_fn)) + class_metrics["ap50"] = detection_ap( + all_class_predictions, class_references, 0.5 + ) + per_class[str(class_value)] = class_metrics + metrics = count_metrics(len(matches), len(false_positives), len(false_negatives)) + metrics.update( + { + "operating_confidence": threshold, + "match_iou": match_iou, + "matched_iou": distribution([item["overlap"] for item in matches]), + "ap50": detection_ap(predictions, references, 0.5), + "ap50_95": _mean_or_none( + detection_ap(predictions, references, 0.5 + step * 0.05) + for step in range(10) + ), + "per_class": per_class, + "calibration": calibration_metrics(retained, matches), + "coverage_risk": coverage_risk( + predictions, references, match_iou, threshold + ), + } + ) + return result( + case, + metrics, + matches, + false_positives, + false_negatives, + post_filter_predictions=retained, + filter_description={ + "applied": True, + "field": "confidence", + "operator": ">=", + "threshold": threshold, + }, + ) + + +def polygon(item: dict[str, Any]): + from shapely.geometry import Polygon + + return Polygon(item["polygon"]) + + +def distribution(values: list[float]) -> dict[str, Any]: + if not values: + return {"count": 0, "mean": None, "median": None, "min": None, "max": None} + if any(not math.isfinite(value) for value in values): + raise ValueError("Distribution values must be finite") + ordered = sorted(values) + return { + "count": len(values), + "mean": mean(values), + "median": median(values), + "min": ordered[0], + "max": ordered[-1], + } + + +def _validate_metric_spatial_context(case: dict[str, Any]) -> dict[str, Any]: + context = _nonempty_mapping( + case.get("spatial_context"), + f"{case['sample_id']}: spatial_context", + ) + if context.get("metric") is not True: + raise ValueError(f"{case['sample_id']}: spatial_context.metric must be true") + if context.get("coordinate_units") != "m": + raise ValueError( + f"{case['sample_id']}: spatial_context.coordinate_units must be 'm'" + ) + crs_value = context.get("crs") + if not isinstance(crs_value, str) or not crs_value.strip(): + raise ValueError(f"{case['sample_id']}: a non-empty CRS is required") + try: + from pyproj import CRS + + crs = CRS.from_user_input(crs_value) + except Exception as exc: # noqa: BLE001 - invalid CRS must fail closed + raise ValueError(f"{case['sample_id']}: invalid CRS {crs_value!r}") from exc + if not crs.is_projected: + raise ValueError( + f"{case['sample_id']}: metric geometry requires a projected CRS" + ) + if not crs.axis_info or any( + axis.unit_conversion_factor is None + or not math.isclose( + axis.unit_conversion_factor, + 1.0, + rel_tol=0.0, + abs_tol=1e-12, + ) + for axis in crs.axis_info[:2] + ): + raise ValueError(f"{case['sample_id']}: CRS axes must use metres") + return context + + +def _validated_polygon(item: dict[str, Any], label: str): + coordinates = item.get("polygon") + if not isinstance(coordinates, list) or len(coordinates) < 4: + raise ValueError( + f"{label}.polygon must contain a closed ring with at least four points" + ) + normalized: list[tuple[float, float]] = [] + for index, point in enumerate(coordinates): + if not isinstance(point, list) or len(point) != 2: + raise ValueError(f"{label}.polygon[{index}] must be an [x, y] pair") + normalized.append( + ( + _finite_float(point[0], f"{label}.polygon[{index}][0]"), + _finite_float(point[1], f"{label}.polygon[{index}][1]"), + ) + ) + if normalized[0] != normalized[-1]: + raise ValueError(f"{label}.polygon ring must be explicitly closed") + from shapely.geometry import Polygon + + geometry = Polygon(normalized) + if geometry.is_empty or geometry.area <= 0 or not geometry.is_valid: + raise ValueError(f"{label}.polygon must be non-empty, positive-area and valid") + return geometry + + +def _validated_polygon_items( + case: dict[str, Any], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + _validate_metric_spatial_context(case) + references = case.get("references") + predictions = case.get("predictions") + if not isinstance(references, list) or not all( + isinstance(item, dict) for item in references + ): + raise ValueError(f"{case['sample_id']}: references must be an object list") + if not isinstance(predictions, list) or not all( + isinstance(item, dict) for item in predictions + ): + raise ValueError(f"{case['sample_id']}: predictions must be an object list") + _validate_unique_ids(references, f"{case['sample_id']}.references") + _validate_unique_ids(predictions, f"{case['sample_id']}.predictions") + _validate_labeled_items(case, references, "references") + _validate_labeled_items(case, predictions, "predictions") + validated_references = [ + dict( + item, + geometry=_validated_polygon( + item, f"{case['sample_id']}.references[{index}]" + ), + ) + for index, item in enumerate(references) + ] + validated_predictions = [ + dict( + item, + geometry=_validated_polygon( + item, f"{case['sample_id']}.predictions[{index}]" + ), + ) + for index, item in enumerate(predictions) + ] + return validated_references, validated_predictions + + +def _polygon_overlap(prediction: dict[str, Any], reference: dict[str, Any]) -> float: + intersection = prediction["geometry"].intersection(reference["geometry"]).area + union = prediction["geometry"].union(reference["geometry"]).area + return intersection / union if union > 0 else 0.0 + + +def boundary_f1(prediction, reference, tolerance: float) -> float | None: + predicted_boundary = prediction.boundary + reference_boundary = reference.boundary + predicted_length = predicted_boundary.length + reference_length = reference_boundary.length + if predicted_length <= 0 or reference_length <= 0: + return None + precision = ( + predicted_boundary.intersection(reference_boundary.buffer(tolerance)).length + / predicted_length + ) + recall = ( + reference_boundary.intersection(predicted_boundary.buffer(tolerance)).length + / reference_length + ) + return f1_score(precision, recall) + + +def evaluate_footprint_segmentation(case: dict[str, Any]) -> dict[str, Any]: + references, predictions = _validated_polygon_items(case) + match_iou = _probability( + case["config"].get("match_iou"), + f"{case['sample_id']}.config.match_iou", + ) + tolerance = _finite_float( + case["config"].get("boundary_tolerance_m"), + f"{case['sample_id']}.config.boundary_tolerance_m", + ) + if tolerance <= 0: + raise ValueError(f"{case['sample_id']}: boundary tolerance must be positive") + for index, item in enumerate(predictions): + if "confidence" in item: + _probability( + item["confidence"], + f"{case['sample_id']}.predictions[{index}].confidence", + ) + matches, false_positives, false_negatives = greedy_match( + predictions, + references, + _polygon_overlap, + match_iou, + class_aware=True, + ) + by_prediction = {item["id"]: item for item in predictions} + by_reference = {item["id"]: item for item in references} + dice = [] + boundary = [] + centroids = [] + area_errors = [] + for match in matches: + pred = by_prediction[match["prediction_id"]]["geometry"] + ref = by_reference[match["reference_id"]]["geometry"] + intersection = pred.intersection(ref).area + dice.append(2 * intersection / (pred.area + ref.area)) + boundary.append(boundary_f1(pred, ref, tolerance)) + centroids.append(pred.centroid.distance(ref.centroid)) + area_errors.append((pred.area - ref.area) / ref.area) + metrics = count_metrics(len(matches), len(false_positives), len(false_negatives)) + metrics.update( + { + "mean_iou": _mean_or_none([item["overlap"] for item in matches]), + "mean_dice": _mean_or_none(dice), + "mean_boundary_f1": _mean_or_none(boundary), + "centroid_distance_m": distribution(centroids), + "relative_area_error": distribution(area_errors), + "topologically_valid_predictions": len(predictions), + "topologically_invalid_predictions": 0, + "spatial_context_validated": True, + } + ) + return result( + case, + metrics, + matches, + false_positives, + false_negatives, + ) + + +def _validate_rectangular_grid( + value: Any, + label: str, +) -> tuple[list[list[Any]], tuple[int, int]]: + if ( + not isinstance(value, list) + or not value + or not all(isinstance(row, list) for row in value) + ): + raise ValueError(f"{label} must be a non-empty two-dimensional array") + width = len(value[0]) + if width <= 0 or any(len(row) != width for row in value): + raise ValueError(f"{label} must be exactly rectangular") + return value, (len(value), width) + + +def _validate_raster_side( + case: dict[str, Any], + side_name: str, + expected_shape: tuple[int, int], +) -> dict[str, Any]: + context = _nonempty_mapping( + case["raster_context"].get(side_name), + f"{case['sample_id']}.raster_context.{side_name}", + ) + required = {"crs", "transform", "shape", "nodata", "mask"} + missing = sorted(required - context.keys()) + if missing: + raise ValueError( + f"{case['sample_id']}.raster_context.{side_name} missing {missing}" + ) + crs_value = context["crs"] + if not isinstance(crs_value, str) or not crs_value.strip(): + raise ValueError(f"{case['sample_id']}: raster CRS must be non-empty") + try: + from pyproj import CRS + + normalized_crs = CRS.from_user_input(crs_value) + except Exception as exc: # noqa: BLE001 - invalid CRS must fail closed + raise ValueError( + f"{case['sample_id']}: invalid raster CRS {crs_value!r}" + ) from exc + transform = context["transform"] + if not isinstance(transform, list) or len(transform) != 6: + raise ValueError( + f"{case['sample_id']}: raster transform must contain six numbers" + ) + normalized_transform = tuple( + _finite_float( + value, + f"{case['sample_id']}.{side_name}.transform[{index}]", + ) + for index, value in enumerate(transform) + ) + shape = context["shape"] + if ( + not isinstance(shape, list) + or len(shape) != 2 + or any( + isinstance(value, bool) or not isinstance(value, int) or value <= 0 + for value in shape + ) + ): + raise ValueError( + f"{case['sample_id']}: raster shape must be [positive rows, " + "positive columns]" + ) + if tuple(shape) != expected_shape: + raise ValueError( + f"{case['sample_id']}: declared {side_name} shape differs from the grid" + ) + mask, mask_shape = _validate_rectangular_grid( + context["mask"], + f"{case['sample_id']}.{side_name}.mask", + ) + if mask_shape != expected_shape or any( + not isinstance(value, bool) for row in mask for value in row + ): + raise ValueError( + f"{case['sample_id']}: {side_name} mask must be a boolean grid " + "with the exact raster shape" + ) + nodata = context["nodata"] + if nodata is not None: + _finite_float(nodata, f"{case['sample_id']}.{side_name}.nodata") + return { + "crs": normalized_crs, + "transform": normalized_transform, + "shape": expected_shape, + "nodata": nodata, + "mask": mask, + } + + +def _is_nodata(value: Any, nodata: Any) -> bool: + return nodata is not None and value == nodata + + +def _validate_raster_context( + case: dict[str, Any], + reference_shape: tuple[int, int], + prediction_shape: tuple[int, int], +) -> tuple[dict[str, Any], dict[str, Any]]: + if reference_shape != prediction_shape: + raise ValueError(f"{case['sample_id']}: raster shapes differ") + _nonempty_mapping( + case.get("raster_context"), + f"{case['sample_id']}: raster_context", + ) + reference = _validate_raster_side(case, "reference", reference_shape) + prediction = _validate_raster_side(case, "prediction", prediction_shape) + if reference["crs"] != prediction["crs"]: + raise ValueError(f"{case['sample_id']}: raster CRS alignment differs") + if reference["transform"] != prediction["transform"]: + raise ValueError(f"{case['sample_id']}: raster affine alignment differs") + if reference["shape"] != prediction["shape"]: + raise ValueError(f"{case['sample_id']}: raster shape alignment differs") + return reference, prediction + + +def evaluate_raster_classification(case: dict[str, Any]) -> dict[str, Any]: + classes = _validate_classes(case) + reference_grid, reference_shape = _validate_rectangular_grid( + case.get("references"), + f"{case['sample_id']}.references", + ) + prediction_grid, prediction_shape = _validate_rectangular_grid( + case.get("predictions"), + f"{case['sample_id']}.predictions", + ) + reference_context, prediction_context = _validate_raster_context( + case, + reference_shape, + prediction_shape, + ) + labels = [str(value) for value in classes] + missing_label = "__prediction_nodata__" + confusion = { + reference: {prediction: 0 for prediction in [*labels, missing_label]} + for reference in labels + } + failures = [] + evaluated_reference_count = 0 + prediction_present_count = 0 + ignored_reference_count = 0 + prediction_outside_reference_count = 0 + correct_count = 0 + for row_index in range(reference_shape[0]): + for column_index in range(reference_shape[1]): + reference_value = reference_grid[row_index][column_index] + prediction_value = prediction_grid[row_index][column_index] + reference_valid = reference_context["mask"][row_index][column_index] + prediction_valid = prediction_context["mask"][row_index][column_index] + if reference_valid and _is_nodata( + reference_value, reference_context["nodata"] + ): + raise ValueError( + f"{case['sample_id']}: reference mask marks nodata as " + f"valid at [{row_index}, {column_index}]" + ) + if prediction_valid and _is_nodata( + prediction_value, prediction_context["nodata"] + ): + raise ValueError( + f"{case['sample_id']}: prediction mask marks nodata as " + f"valid at [{row_index}, {column_index}]" + ) + if not reference_valid: + ignored_reference_count += 1 + if prediction_valid: + prediction_outside_reference_count += 1 + continue + if reference_value not in classes: + raise ValueError( + f"{case['sample_id']}: reference class outside ontology " + f"at [{row_index}, {column_index}]" + ) + evaluated_reference_count += 1 + reference_key = str(reference_value) + if not prediction_valid: + confusion[reference_key][missing_label] += 1 + failures.append( + { + "pixel_index": row_index * reference_shape[1] + column_index, + "row": row_index, + "column": column_index, + "reference": reference_value, + "prediction": None, + "reason": "prediction_masked_or_nodata", + } + ) + continue + if prediction_value not in classes: + raise ValueError( + f"{case['sample_id']}: prediction class outside ontology " + f"at [{row_index}, {column_index}]" + ) + prediction_present_count += 1 + prediction_key = str(prediction_value) + confusion[reference_key][prediction_key] += 1 + if reference_value == prediction_value: + correct_count += 1 + else: + failures.append( + { + "pixel_index": row_index * reference_shape[1] + column_index, + "row": row_index, + "column": column_index, + "reference": reference_value, + "prediction": prediction_value, + "reason": "class_mismatch", + } + ) + per_class = {} + for class_value in classes: + class_key = str(class_value) + tp = confusion[class_key][class_key] + fp = sum( + confusion[str(other)][class_key] + for other in classes + if other != class_value + ) + fn = sum( + confusion[class_key][prediction] + for prediction in [*labels, missing_label] + if prediction != class_key + ) + values = count_metrics(tp, fp, fn) + values["iou"] = safe_rate(tp, tp + fp + fn) + per_class[class_key] = values + metrics = { + "pixel_count": reference_shape[0] * reference_shape[1], + "evaluated_reference_pixel_count": evaluated_reference_count, + "prediction_present_pixel_count": prediction_present_count, + "ignored_reference_pixel_count": ignored_reference_count, + "prediction_outside_reference_count": (prediction_outside_reference_count), + "correct_pixel_count": correct_count, + "prediction_coverage": safe_rate( + prediction_present_count, + evaluated_reference_count, + ), + "prediction_coverage_ci95_wilson": wilson_interval( + prediction_present_count, + evaluated_reference_count, + ), + "accuracy": safe_rate(correct_count, evaluated_reference_count), + "accuracy_ci95_wilson": wilson_interval( + correct_count, + evaluated_reference_count, + ), + "mean_iou": _mean_or_none([value["iou"] for value in per_class.values()]), + "macro_f1": _mean_or_none([value["f1"] for value in per_class.values()]), + "per_class": per_class, + "confusion_matrix": confusion, + "alignment_validated": True, + } + return result(case, metrics, [], failures, []) + + +def evaluate_vector_comparison(case: dict[str, Any]) -> dict[str, Any]: + references, predictions = _validated_polygon_items(case) + match_iou = _probability( + case["config"].get("match_iou"), + f"{case['sample_id']}.config.match_iou", + ) + matches, false_positives, false_negatives = greedy_match( + predictions, + references, + _polygon_overlap, + match_iou, + class_aware=True, + ) + metrics = count_metrics(len(matches), len(false_positives), len(false_negatives)) + metrics.update( + { + "mean_iou": _mean_or_none([item["overlap"] for item in matches]), + "topologically_valid": True, + "spatial_context_validated": True, + } + ) + return result(case, metrics, matches, false_positives, false_negatives) + + +def evaluate_change_detection(case: dict[str, Any]) -> dict[str, Any]: + references, predictions = _validate_detection_case(case) + match_iou = _probability( + case["config"].get("match_iou"), + f"{case['sample_id']}.config.match_iou", + ) + matches = [] + false_positives = [] + false_negatives = [] + event_metrics = {} + for event in case["classes"]: + event_predictions = [item for item in predictions if item["class"] == event] + event_references = [item for item in references if item["class"] == event] + event_matches, event_fp, event_fn = greedy_match( + event_predictions, + event_references, + lambda prediction, reference: bbox_iou( + prediction["bbox"], reference["bbox"] + ), + match_iou, + class_aware=True, + ) + matches.extend(dict(item, event=event) for item in event_matches) + false_positives.extend(event_fp) + false_negatives.extend(event_fn) + event_metrics[str(event)] = count_metrics( + len(event_matches), len(event_fp), len(event_fn) + ) + metrics = count_metrics(len(matches), len(false_positives), len(false_negatives)) + metrics["event_metrics"] = event_metrics + return result(case, metrics, matches, false_positives, false_negatives) + + +def evaluate_terrain(case: dict[str, Any]) -> dict[str, Any]: + references = case.get("references") + predictions = case.get("predictions") + if not isinstance(references, list) or not isinstance(predictions, list): + raise ValueError(f"{case['sample_id']}: terrain inputs must be lists") + if len(references) != len(predictions): + raise ValueError(f"{case['sample_id']}: terrain vector lengths differ") + units = case.get("units") + if not isinstance(units, str) or not units.strip(): + raise ValueError(f"{case['sample_id']}: terrain units must be explicit") + normalized_references = [ + _finite_float(value, f"{case['sample_id']}.references[{index}]") + for index, value in enumerate(references) + ] + normalized_predictions: list[float | None] = [] + for index, value in enumerate(predictions): + normalized_predictions.append( + None + if value is None + else _finite_float( + value, + f"{case['sample_id']}.predictions[{index}]", + ) + ) + pairs = [ + (reference, prediction) + for reference, prediction in zip( + normalized_references, + normalized_predictions, + strict=True, + ) + if prediction is not None + ] + errors = [prediction - reference for reference, prediction in pairs] + missing = [ + index for index, value in enumerate(normalized_predictions) if value is None + ] + metrics = { + "units": units, + "reference_count": len(normalized_references), + "evaluated_count": len(pairs), + "missing_count": len(missing), + "coverage": safe_rate(len(pairs), len(normalized_references)), + "coverage_ci95_wilson": wilson_interval(len(pairs), len(normalized_references)), + "mae": _mean_or_none([abs(value) for value in errors]), + "rmse": ( + math.sqrt(mean(value * value for value in errors)) if errors else None + ), + "bias": _mean_or_none(errors), + "error_sum": sum(errors), + "absolute_error_sum": sum(abs(value) for value in errors), + "squared_error_sum": sum(value * value for value in errors), + "error_distribution": distribution(errors), + } + return result( + case, + metrics, + [], + [], + [{"missing_index": index} for index in missing], + ) + + +def _normalized_anomalies( + case: dict[str, Any], + field: str, +) -> list[dict[str, str]]: + values = case.get(field) + if not isinstance(values, list): + raise ValueError(f"{case['sample_id']}: {field} must be a list") + normalized = [] + for index, value in enumerate(values): + if not isinstance(value, dict): + raise ValueError( + f"{case['sample_id']}: {field}[{index}] must include code and severity" + ) + code = value.get("code") + severity = value.get("severity") + if not isinstance(code, str) or not code.strip(): + raise ValueError(f"{case['sample_id']}: {field}[{index}].code is invalid") + if severity not in VALID_ANOMALY_SEVERITIES: + raise ValueError( + f"{case['sample_id']}: {field}[{index}].severity is invalid" + ) + normalized.append({"code": code, "severity": severity}) + codes = [item["code"] for item in normalized] + duplicates = sorted({code for code in codes if codes.count(code) > 1}) + if duplicates: + raise ValueError(f"{case['sample_id']}: duplicate {field} codes {duplicates}") + return normalized + + +def evaluate_validation(case: dict[str, Any]) -> dict[str, Any]: + expected_items = _normalized_anomalies(case, "expected_anomalies") + observed_items = _normalized_anomalies(case, "observed_anomalies") + expected = {item["code"]: item for item in expected_items} + observed = {item["code"]: item for item in observed_items} + matched_codes = sorted(expected.keys() & observed.keys()) + false_positive_codes = sorted(observed.keys() - expected.keys()) + false_negative_codes = sorted(expected.keys() - observed.keys()) + matched = [ + { + "anomaly": code, + "expected_severity": expected[code]["severity"], + "observed_severity": observed[code]["severity"], + } + for code in matched_codes + ] + false_positives = [ + {"anomaly": code, "severity": observed[code]["severity"]} + for code in false_positive_codes + ] + false_negatives = [ + {"anomaly": code, "severity": expected[code]["severity"]} + for code in false_negative_codes + ] + metrics = count_metrics(len(matched), len(false_positives), len(false_negatives)) + metrics["blocker_or_critical_miss_count"] = sum( + item["severity"] in {"blocker", "critical"} for item in false_negatives + ) + metrics["misses_by_severity"] = { + severity: sum(item["severity"] == severity for item in false_negatives) + for severity in sorted(VALID_ANOMALY_SEVERITIES) + } + return result(case, metrics, matched, false_positives, false_negatives) + + +def _case_payload_fields(case: dict[str, Any]) -> tuple[str, str]: + if case["task"] == "geospatial_data_validation": + return "expected_anomalies", "observed_anomalies" + return "references", "predictions" + + +def result( + case: dict[str, Any], + metrics: dict[str, Any], + matches: list[dict[str, Any]], + false_positives: list[dict[str, Any]], + false_negatives: list[dict[str, Any]], + *, + post_filter_predictions: Any | None = None, + filter_description: dict[str, Any] | None = None, +) -> dict[str, Any]: + reference_field, prediction_field = _case_payload_fields(case) + exact_references = serializable(case[reference_field]) + exact_predictions = serializable(case[prediction_field]) + post_filter = ( + exact_predictions + if post_filter_predictions is None + else serializable(post_filter_predictions) + ) + case_hash = canonical_hash(case) + raw = { + "sample_id": case["sample_id"], + "task": case["task"], + "metadata": serializable(case["metadata"]), + "split": case["split"], + "config": serializable(case["config"]), + "input_lineage": serializable(case["lineage"]), + "reference_input_field": reference_field, + "prediction_input_field": prediction_field, + "references": exact_references, + "predictions_pre_filter": exact_predictions, + "predictions_post_filter": post_filter, + "filter": filter_description or {"applied": False}, + "matches": serializable(matches), + "false_positives": serializable(false_positives), + "false_negatives": serializable(false_negatives), + "input_sha256": case_hash, + "hashes": { + "case_input_canonical_json_sha256": case_hash, + "references_canonical_json_sha256": canonical_hash(exact_references), + "predictions_pre_filter_canonical_json_sha256": canonical_hash( + exact_predictions + ), + "predictions_post_filter_canonical_json_sha256": canonical_hash( + post_filter + ), + "canonicalization": CANONICAL_JSON_SPEC, + }, + } + failures = [] + for item in raw["false_positives"]: + failures.append(failure_entry(case, "false_positive", item)) + for item in raw["false_negatives"]: + failures.append(failure_entry(case, "false_negative", item)) + return { + "sample_id": case["sample_id"], + "task": case["task"], + "metadata": serializable(case["metadata"]), + "metrics": metrics, + "raw": raw, + "failures": failures, + } + + +def serializable(value: Any) -> Any: + if isinstance(value, dict): + return { + key: serializable(item) for key, item in value.items() if key != "geometry" + } + if isinstance(value, list): + return [serializable(item) for item in value] + if isinstance(value, tuple): + return [serializable(item) for item in value] + return value + + +def failure_entry( + case: dict[str, Any], kind: str, evidence: dict[str, Any] +) -> dict[str, Any]: + if case["task"] == "change_detection": + kind = ( + "event_false_positive" + if kind == "false_positive" + else "event_false_negative" + ) + elif case["task"] == "geospatial_data_validation": + kind = ( + "validation_false_positive" + if kind == "false_positive" + else "validation_false_negative" + ) + elif case["task"] == "terrain_interpretation": + kind = "terrain_missing" + return { + "failure_id": canonical_hash( + { + "sample_id": case["sample_id"], + "kind": kind, + "evidence": serializable(evidence), + } + )[:20], + "sample_id": case["sample_id"], + "task": case["task"], + "error_code": ERROR_CODES[kind], + "kind": kind, + "metadata": case["metadata"], + "evidence": serializable(evidence), + } + + +EVALUATORS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = { + "object_detection": evaluate_object_detection, + "footprint_segmentation": evaluate_footprint_segmentation, + "raster_classification": evaluate_raster_classification, + "vector_comparison": evaluate_vector_comparison, + "change_detection": evaluate_change_detection, + "terrain_interpretation": evaluate_terrain, + "geospatial_data_validation": evaluate_validation, +} + + +def _capability( + capability_id: str, + task: str, + implementation_paths: list[str], + implementation_kind: str, + evaluation_status: str, + suitable_metrics: list[str], +) -> dict[str, Any]: + return { + "capability_id": capability_id, + "task": task, + "implementation_paths": implementation_paths, + "implementation_kind": implementation_kind, + "evaluation_status": evaluation_status, + "suitable_metrics": suitable_metrics, + "claim_boundary": ( + "A family mapping is not evidence that this capability has a " + "separate representative product benchmark." + ), + } + + +def task_inventory() -> list[dict[str, Any]]: + """Map concrete capabilities to evaluator families without overclaiming.""" + + synthetic = "synthetic_contract_case_only" + covered = "covered_by_family_not_separately_benchmarked" + separate = "not_separately_benchmarked" + return [ + _capability( + "model_object_detection", + "object_detection", + ["backend/app/services/detection_service.py"], + "model_inference_pipeline", + synthetic, + [ + "precision", + "recall", + "F1", + "AP50", + "AP50-95", + "IoU", + "ECE", + "Brier", + "coverage-risk", + ], + ), + _capability( + "building_proposal_filtering", + "object_detection", + [ + "scripts/train_building_proposal_classifier.py", + "scripts/evaluate_belgium_building_candidate.py", + ], + "supporting_candidate_classifier", + separate, + ["candidate precision", "candidate recall", "F1", "calibration"], + ), + _capability( + "footprint_segmentation", + "footprint_segmentation", + [ + "backend/app/services/segmentation_service.py", + "backend/app/services/segmentation_adapter.py", + ], + "model_inference_pipeline", + synthetic, + [ + "object precision", + "object recall", + "F1", + "IoU", + "Dice", + "boundary F1", + "centroid distance", + "area error", + "topology", + ], + ), + _capability( + "detection_and_vector_qa", + "vector_comparison", + [ + "backend/app/services/qa_service.py", + "backend/app/services/detection_qa_service.py", + "backend/app/services/quality_check_service.py", + ], + "deterministic_geospatial_comparison", + synthetic, + ["precision", "recall", "F1", "IoU", "topology", "coverage"], + ), + _capability( + "vector_clip_buffer_intersect", + "vector_comparison", + [ + "backend/app/services/vector_operations_service.py", + "backend/app/services/vector_feature_service.py", + ], + "deterministic_vector_processing", + covered, + [ + "geometry validity", + "CRS correctness", + "area conservation", + "feature counts", + "topology", + ], + ), + _capability( + "temporal_vector_change", + "change_detection", + ["backend/app/services/change_detection_service.py"], + "deterministic_change_detection", + synthetic, + ["event precision", "event recall", "event F1", "IoU"], + ), + _capability( + "thematic_raster_interpretation", + "raster_classification", + ["backend/app/services/thematic_raster_analysis_service.py"], + "deterministic_source_interpretation", + "synthetic_metric_contract_only_no_generic_learned_classifier_claim", + ["pixel accuracy", "per-class F1", "per-class IoU", "mean IoU"], + ), + _capability( + "raster_clip_reproject_indices", + "raster_classification", + [ + "backend/app/services/raster_service.py", + "backend/app/services/raster_operations_service.py", + ], + "deterministic_raster_processing", + covered, + [ + "CRS/transform preservation", + "pixel alignment", + "nodata", + "numeric tolerance", + ], + ), + _capability( + "raster_partition_mosaic", + "raster_classification", + ["backend/app/services/raster_partition_analysis_service.py"], + "deterministic_raster_partitioning", + separate, + ["seam equality", "coverage completeness", "resolution consistency"], + ), + _capability( + "terrain_height_interpretation", + "terrain_interpretation", + [ + "backend/app/services/terrain_analysis_service.py", + "backend/app/services/spw_terrain_service.py", + ], + "deterministic_continuous_raster_analysis", + synthetic, + ["MAE", "RMSE", "bias", "coverage", "unit integrity"], + ), + _capability( + "flood_hazard_interpretation", + "terrain_interpretation", + ["backend/app/services/flood_hazard_analysis_service.py"], + "deterministic_scenario_raster_analysis", + covered, + ["depth MAE/RMSE", "hazard-class IoU", "coverage", "scenario identity"], + ), + _capability( + "bathymetry_interpretation", + "terrain_interpretation", + [ + "backend/app/services/bathymetry_raster_analysis_service.py", + "backend/app/services/mdk_bathymetry_probe_service.py", + ], + "deterministic_vertical_reference_analysis", + covered, + ["MAE", "RMSE", "bias", "coverage", "vertical-datum integrity"], + ), + _capability( + "data_contract_validation_and_scan", + "geospatial_data_validation", + [ + "backend/app/services/data_contract_validation.py", + "scripts/run_accuracy_phase3_full_data_scan.py", + ], + "deterministic_validation", + synthetic, + ["anomaly precision", "anomaly recall", "anomaly F1", "critical misses"], + ), + _capability( + "aoi_partition_orchestration", + "geospatial_data_validation", + [ + "backend/app/services/aoi_operation_service.py", + "backend/app/services/aoi_operation_executor.py", + "backend/app/services/aoi_operation_worker.py", + ], + "deterministic_orchestration", + separate, + [ + "partition completeness", + "overlap/gap", + "idempotency", + "resume correctness", + ], + ), + _capability( + "geo_assistant_orchestration", + "geospatial_data_validation", + ["backend/app/services/geo_assistant_service.py"], + "tool_orchestration_interface", + "no_independent_accuracy_score_underlying_tool_results_are_authoritative", + ["tool-selection correctness", "grounding", "unsupported-claim rate"], + ), + ] + + +def _numeric_metric(metrics: dict[str, Any], key: str) -> float | None: + value = metrics.get(key) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + converted = float(value) + return converted if math.isfinite(converted) else None + + +def _macro_metrics( + values: list[dict[str, Any]], + keys: Iterable[str], +) -> dict[str, Any]: + output: dict[str, Any] = { + "status": "computed", + "method": "unweighted mean across cases with defined values", + } + computed = 0 + for key in keys: + items = [ + value + for metrics in values + if (value := _numeric_metric(metrics, key)) is not None + ] + output[key] = mean(items) if items else None + output[f"{key}_case_support"] = len(items) + computed += bool(items) + if not computed: + output["status"] = "not_evaluable" + return output + + +def _aggregate_count_family( + values: list[dict[str, Any]], + extra_macro_keys: Iterable[str] = (), +) -> dict[str, Any]: + metrics = [item["metrics"] for item in values] + micro = count_metrics( + sum(int(item["true_positive"]) for item in metrics), + sum(int(item["false_positive"]) for item in metrics), + sum(int(item["false_negative"]) for item in metrics), + ) + return { + "micro": micro, + "macro": _macro_metrics( + metrics, + ("precision", "recall", "f1", *extra_macro_keys), + ), + "observation_support": { + "references": sum(int(item["reference_count"]) for item in metrics), + "predictions": sum(int(item["prediction_count"]) for item in metrics), + }, + "primary_metric": { + "name": "micro.f1", + "value": micro["f1"], + "direction": "higher_is_better", + }, + } + + +def _aggregate_raster(values: list[dict[str, Any]]) -> dict[str, Any]: + metrics = [item["metrics"] for item in values] + class_counts: dict[str, dict[str, int]] = defaultdict( + lambda: {"tp": 0, "fp": 0, "fn": 0} + ) + for item in metrics: + for class_name, class_metrics in item["per_class"].items(): + class_counts[class_name]["tp"] += int(class_metrics["true_positive"]) + class_counts[class_name]["fp"] += int(class_metrics["false_positive"]) + class_counts[class_name]["fn"] += int(class_metrics["false_negative"]) + per_class = {} + for class_name, counts in sorted(class_counts.items()): + class_metrics = count_metrics(counts["tp"], counts["fp"], counts["fn"]) + class_metrics["iou"] = safe_rate( + counts["tp"], + counts["tp"] + counts["fp"] + counts["fn"], + ) + per_class[class_name] = class_metrics + evaluated = sum(int(item["evaluated_reference_pixel_count"]) for item in metrics) + present = sum(int(item["prediction_present_pixel_count"]) for item in metrics) + correct = sum(int(item["correct_pixel_count"]) for item in metrics) + micro = { + "evaluated_reference_pixel_count": evaluated, + "prediction_present_pixel_count": present, + "correct_pixel_count": correct, + "accuracy": safe_rate(correct, evaluated), + "accuracy_ci95_wilson": wilson_interval(correct, evaluated), + "prediction_coverage": safe_rate(present, evaluated), + "prediction_coverage_ci95_wilson": wilson_interval(present, evaluated), + "mean_iou": _mean_or_none([item["iou"] for item in per_class.values()]), + "macro_f1_across_classes": _mean_or_none( + [item["f1"] for item in per_class.values()] + ), + "per_class": per_class, + } + return { + "micro": micro, + "macro": _macro_metrics( + metrics, + ("accuracy", "mean_iou", "macro_f1"), + ), + "observation_support": {"evaluated_pixels": evaluated}, + "primary_metric": { + "name": "micro.mean_iou", + "value": micro["mean_iou"], + "direction": "higher_is_better", + }, + } + + +def _aggregate_terrain(values: list[dict[str, Any]]) -> dict[str, Any]: + metrics = [item["metrics"] for item in values] + units = sorted({str(item["units"]) for item in metrics}) + references = sum(int(item["reference_count"]) for item in metrics) + evaluated = sum(int(item["evaluated_count"]) for item in metrics) + if len(units) != 1: + return { + "micro": {"status": "not_evaluable", "reason": "mixed units"}, + "macro": {"status": "not_evaluable", "reason": "mixed units"}, + "observation_support": {"references": references}, + "primary_metric": { + "name": "micro.rmse", + "value": None, + "direction": "lower_is_better", + }, + } + absolute_error_sum = sum(float(item["absolute_error_sum"]) for item in metrics) + squared_error_sum = sum(float(item["squared_error_sum"]) for item in metrics) + error_sum = sum(float(item["error_sum"]) for item in metrics) + micro = { + "status": "computed" if evaluated else "not_evaluable", + "units": units[0], + "reference_count": references, + "evaluated_count": evaluated, + "coverage": safe_rate(evaluated, references), + "coverage_ci95_wilson": wilson_interval(evaluated, references), + "mae": safe_rate(absolute_error_sum, evaluated), + "rmse": (math.sqrt(squared_error_sum / evaluated) if evaluated else None), + "bias": safe_rate(error_sum, evaluated), + } + return { + "micro": micro, + "macro": _macro_metrics( + metrics, + ("coverage", "mae", "rmse", "bias"), + ), + "observation_support": { + "references": references, + "evaluated": evaluated, + }, + "primary_metric": { + "name": "micro.rmse", + "value": micro["rmse"], + "direction": "lower_is_better", + }, + } + + +def _aggregate_task(values: list[dict[str, Any]]) -> dict[str, Any]: + task = values[0]["task"] + if task == "raster_classification": + aggregation = _aggregate_raster(values) + elif task == "terrain_interpretation": + aggregation = _aggregate_terrain(values) + else: + extras = { + "object_detection": ("ap50", "ap50_95"), + "footprint_segmentation": ( + "mean_iou", + "mean_dice", + "mean_boundary_f1", + ), + "vector_comparison": ("mean_iou",), + "change_detection": (), + "geospatial_data_validation": ("blocker_or_critical_miss_count",), + }[task] + aggregation = _aggregate_count_family(values, extras) + if task == "geospatial_data_validation": + aggregation["micro"]["blocker_or_critical_miss_count"] = sum( + int(item["metrics"]["blocker_or_critical_miss_count"]) + for item in values + ) + case_support = len(values) + status = ( + "evaluable" + if case_support >= SUBGROUP_MIN_CASE_SUPPORT + and aggregation["primary_metric"]["value"] is not None + else "insufficient_support" + ) + return { + "case_support": case_support, + "minimum_case_support": SUBGROUP_MIN_CASE_SUPPORT, + "status": status, + "release_gate_status": "not_evaluable", + "release_gate_reason": ( + "insufficient_support" + if status == "insufficient_support" + else "no_frozen_release_target" + ), + **aggregation, + } + + +def _stratum_key(value: Any) -> str: + return "__not_applicable__" if value is None else str(value) + + +def _worst_stratum_by_task(strata: dict[str, Any]) -> dict[str, Any]: + tasks = sorted( + {task for stratum in strata.values() for task in stratum["task_metrics"]} + ) + output = {} + for task in tasks: + candidates = [] + for stratum_name, stratum in strata.items(): + task_metrics = stratum["task_metrics"].get(task) + if not task_metrics or task_metrics["status"] != "evaluable": + continue + primary = task_metrics["primary_metric"] + if primary["value"] is not None: + candidates.append((stratum_name, primary)) + if not candidates: + output[task] = { + "status": "not_evaluable", + "reason": ( + "no stratum meets minimum support with a defined primary metric" + ), + } + continue + direction = candidates[0][1]["direction"] + selected = ( + min(candidates, key=lambda item: (item[1]["value"], item[0])) + if direction == "higher_is_better" + else max( + candidates, + key=lambda item: (item[1]["value"], item[0]), + ) + ) + output[task] = { + "status": "computed", + "stratum": selected[0], + "primary_metric": selected[1], + } + return output + + +def subgroup_report(results: list[dict[str, Any]]) -> dict[str, Any]: + dimensions = ( + "region", + "municipality", + "urbanity", + "object_size", + "source", + "sensor", + "resolution_m", + "season", + "date", + "vegetation", + "occlusion", + "difficulty", + ) + dimension_reports = {} + any_insufficient = False + for dimension in dimensions: + groups: dict[str, list[dict[str, Any]]] = defaultdict(list) + for item in results: + groups[_stratum_key(item["metadata"].get(dimension))].append(item) + strata = {} + for key, values in sorted(groups.items()): + per_task: dict[str, list[dict[str, Any]]] = defaultdict(list) + for value in values: + per_task[value["task"]].append(value) + task_metrics = { + task: _aggregate_task(task_values) + for task, task_values in sorted(per_task.items()) + } + insufficient = any( + item["status"] == "insufficient_support" + for item in task_metrics.values() + ) + any_insufficient = any_insufficient or insufficient + strata[key] = { + "case_support": len(values), + "tasks": sorted(per_task), + "failure_count": sum(len(value["failures"]) for value in values), + "task_metrics": task_metrics, + "release_gate_status": "not_evaluable", + "release_gate_reason": "insufficient_support" + if insufficient + else "no_frozen_release_target", + } + dimension_reports[dimension] = { + "strata": strata, + "worst_stratum_by_task": _worst_stratum_by_task(strata), + } + return { + "schema_version": REPORT_SCHEMA_VERSION, + "minimum_case_support": SUBGROUP_MIN_CASE_SUPPORT, + "support_unit": ( + "independent benchmark cases; pixel/object counts are reported " + "separately and do not replace case support" + ), + "overall_status": ( + "not_evaluable" if any_insufficient else "evaluable_no_release_target" + ), + "dimensions": dimension_reports, + } + + +def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]: + file_hash = file_sha256(path) + portfolio = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(portfolio, dict): + raise ValueError("The evaluation portfolio must be a JSON object") + claim_boundary = portfolio.get("claim_boundary") + if not isinstance(claim_boundary, str) or ( + "synthetic" not in claim_boundary.lower() + ): + raise ValueError( + "The portfolio claim boundary must explicitly state that it is synthetic" + ) + split_roles = portfolio.get("split_roles") + if split_roles not in (["test"], ["test", "background-test"]): + raise ValueError( + "split_roles must be exactly ['test'] or ['test', 'background-test']" + ) + if "challenge_cases" in portfolio or "challenge_labels" in portfolio: + raise ValueError("Challenge cases and labels must remain sealed and absent") + selection_policy = portfolio.get("selection_policy") + if not isinstance(selection_policy, str) or not selection_policy.strip(): + raise ValueError("The portfolio requires an explicit selection policy") + declared_portfolio_lineage = _nonempty_mapping( + portfolio.get("portfolio_lineage"), + "portfolio_lineage", + ) + for field in ("origin", "source_path", "version"): + value = declared_portfolio_lineage.get(field) + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"portfolio_lineage.{field} must be a non-empty string") + cases = portfolio.get("cases") + if not isinstance(cases, list) or not all(isinstance(item, dict) for item in cases): + raise ValueError("The portfolio cases must be an object list") + for case in cases: + if case.get("split") not in split_roles: + raise ValueError( + f"{case['sample_id']}: split must be one of {split_roles}; " + "challenge remains sealed" + ) + _validate_case_common(case) + identifiers = [item["sample_id"] for item in cases] + duplicates = sorted( + {identifier for identifier in identifiers if identifiers.count(identifier) > 1} + ) + if duplicates: + raise ValueError(f"Duplicate protected case ids: {duplicates}") + case_ids = set(identifiers) + unexpected = sorted(case_ids - allowed_sample_ids) + missing = sorted(allowed_sample_ids - case_ids) + if unexpected or missing: + raise ValueError( + "Protected case identity mismatch: " + f"unexpected={unexpected}, missing={missing}" + ) + portfolio_canonical_hash = canonical_hash(portfolio) + results = [ + EVALUATORS[item["task"]](item) + for item in sorted(cases, key=lambda item: item["sample_id"]) + ] + portfolio_lineage = { + "declared": serializable(declared_portfolio_lineage), + "portfolio_id": portfolio.get("portfolio_id"), + "portfolio_schema_version": portfolio.get("schema_version"), + "portfolio_file_sha256": file_hash, + "portfolio_canonical_json_sha256": portfolio_canonical_hash, + "source_path": declared_portfolio_lineage["source_path"], + "split_roles": list(split_roles), + "claim_boundary": claim_boundary, + } + for item in results: + item["raw"]["portfolio_lineage"] = portfolio_lineage + failures = sorted( + (failure for item in results for failure in item["failures"]), + key=lambda item: item["failure_id"], + ) + results_hash = canonical_hash(results) + return { + "schema_version": REPORT_SCHEMA_VERSION, + "evaluator_version": EVALUATOR_VERSION, + "portfolio_id": portfolio["portfolio_id"], + "portfolio_file_sha256": file_hash, + "portfolio_canonical_json_sha256": portfolio_canonical_hash, + "claim_boundary": claim_boundary, + "selection_policy": selection_policy, + "split_roles": list(split_roles), + "protected_policy": serializable(portfolio.get("protected_policy")), + "hash_specification": { + "algorithm": "SHA-256", + "canonical_json": CANONICAL_JSON_SPEC, + "portfolio_file_sha256_input": "exact source-file bytes", + "portfolio_canonical_json_sha256_input": ("parsed complete portfolio"), + "results_canonical_json_sha256_input": ( + "complete sample-id-ordered results array" + ), + }, + "task_inventory": task_inventory(), + "evaluated_task_families": sorted({item["task"] for item in results}), + "task_count": len({item["task"] for item in results}), + "case_count": len(results), + "results": results, + "subgroups": subgroup_report(results), + "failures": failures, + "results_canonical_json_sha256": results_hash, + } diff --git a/scripts/build_building_proposal_classifier_dataset.py b/scripts/build_building_proposal_classifier_dataset.py index e2422a9d..f687331c 100644 --- a/scripts/build_building_proposal_classifier_dataset.py +++ b/scripts/build_building_proposal_classifier_dataset.py @@ -28,7 +28,7 @@ from training_release_manifest import ( # noqa: E402 ) -PROTECTED_SPLITS = {"calibration", "test", "background-test"} +PROTECTED_SPLITS = {"calibration", "test", "background-test", "challenge"} PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json" PROPOSAL_DATASET_EVIDENCE_NAME = "proposal-dataset-evidence.json" PROPOSAL_DATASET_SCHEMA_VERSION = 1 diff --git a/scripts/build_regional_yolo_dataset.py b/scripts/build_regional_yolo_dataset.py index bc194045..5d67aa8e 100644 --- a/scripts/build_regional_yolo_dataset.py +++ b/scripts/build_regional_yolo_dataset.py @@ -26,7 +26,7 @@ from training_release_manifest import ( # noqa: E402 ) -PROTECTED_SPLITS = {"calibration", "test", "background-test"} +PROTECTED_SPLITS = {"calibration", "test", "background-test", "challenge"} def sha256(path: Path) -> str: diff --git a/scripts/generate_accuracy_phase4_splits.py b/scripts/generate_accuracy_phase4_splits.py new file mode 100644 index 00000000..2348d388 --- /dev/null +++ b/scripts/generate_accuracy_phase4_splits.py @@ -0,0 +1,701 @@ +#!/usr/bin/env python3 +"""Generate immutable P4 split manifests and fail closed on leakage.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +from collections import Counter, defaultdict +from datetime import date +from pathlib import Path +from typing import Any, Iterable + +from pyproj import CRS + +SCHEMA_VERSION = 1 +GENERATOR_VERSION = "1.1.0" +TRAIN_SPLITS = {"train"} +SELECTION_SPLITS = {"val", "calibration"} +DEVELOPMENT_SPLITS = TRAIN_SPLITS | SELECTION_SPLITS +PROTECTED_SPLITS = {"test", "background-test", "challenge"} +ALL_SPLITS = DEVELOPMENT_SPLITS | PROTECTED_SPLITS +NORMATIVE_SPLITS = {"train", "val", "calibration", "test", "background-test"} +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +FINGERPRINT_PATTERN = re.compile(r"^[0-9a-f]{16}$") + + +class LeakageError(ValueError): + """Raised when split isolation is not demonstrably safe.""" + + +def canonical_bytes(value: Any) -> bytes: + return json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + + +def canonical_hash(value: Any) -> str: + return hashlib.sha256(canonical_bytes(value)).hexdigest() + + +def write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + content = json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text(content, encoding="utf-8") + temporary.replace(path) + + +def validate_metric_crs(value: Any) -> str: + try: + crs = CRS.from_user_input(value) + except Exception as exc: # noqa: BLE001 - pyproj exposes multiple parser errors + raise LeakageError(f"Invalid CRS: {value!r}") from exc + axes = crs.axis_info + if crs.is_geographic or not axes: + raise LeakageError(f"CRS must be projected in metres: {value!r}") + if any( + abs(float(axis.unit_conversion_factor or 0.0) - 1.0) > 1e-12 + for axis in axes[:2] + ): + raise LeakageError(f"CRS axes must use metres: {value!r}") + return crs.to_string() + + +def require_hex( + sample_id: str, field: str, value: Any, pattern: re.Pattern[str] +) -> str: + normalized = str(value or "").lower() + if not pattern.fullmatch(normalized): + raise LeakageError(f"{sample_id}: {field} has an invalid fingerprint") + return normalized + + +def fingerprint_distance(left: str, right: str) -> int: + return (int(left, 16) ^ int(right, 16)).bit_count() + + +def bbox_distance(left: list[float], right: list[float]) -> float: + dx = max(left[0] - right[2], right[0] - left[2], 0.0) + dy = max(left[1] - right[3], right[1] - left[3], 0.0) + return math.hypot(dx, dy) + + +def normalized_sample(sample: dict[str, Any]) -> dict[str, Any]: + required = { + "sample_id", + "task", + "split", + "group_id", + "source_family", + "temporal_family", + "object_ids", + "bbox", + "raw_image_sha256", + "processed_image_sha256", + "label_sha256", + "perceptual_image_hash", + "label_geometry_hash", + "label_geometry_fingerprint", + "native_feature_ids", + "parent_raster_id", + "acquisition_id", + "acquisition_date", + } + missing = sorted(required - set(sample)) + if missing: + raise LeakageError( + f"{sample.get('sample_id', '')}: missing fields {missing}" + ) + split = str(sample["split"]) + if split not in ALL_SPLITS: + raise LeakageError(f"{sample['sample_id']}: unsupported split {split!r}") + bbox = sample["bbox"] + if not isinstance(bbox, list) or len(bbox) != 4: + raise LeakageError(f"{sample['sample_id']}: bbox must contain four values") + values = [float(value) for value in bbox] + if ( + not all(math.isfinite(value) for value in values) + or values[0] >= values[2] + or values[1] >= values[3] + ): + raise LeakageError(f"{sample['sample_id']}: invalid bbox") + result = dict(sample) + result["bbox"] = values + for field in ( + "raw_image_sha256", + "processed_image_sha256", + "label_sha256", + "label_geometry_hash", + ): + result[field] = require_hex( + str(sample["sample_id"]), field, sample[field], SHA256_PATTERN + ) + for field in ("perceptual_image_hash", "label_geometry_fingerprint"): + result[field] = require_hex( + str(sample["sample_id"]), field, sample[field], FINGERPRINT_PATTERN + ) + for field in ("object_ids", "native_feature_ids"): + if not isinstance(sample[field], list) or any( + not str(value) for value in sample[field] + ): + raise LeakageError( + f"{sample['sample_id']}: {field} must be a list of non-empty identities" + ) + result[field] = sorted({str(value) for value in sample[field]}) + for field in ( + "group_id", + "source_family", + "temporal_family", + "parent_raster_id", + "acquisition_id", + ): + if not str(sample[field]).strip(): + raise LeakageError(f"{sample['sample_id']}: {field} must be non-empty") + result[field] = str(sample[field]) + try: + result["acquisition_date"] = date.fromisoformat( + str(sample["acquisition_date"]) + ).isoformat() + except ValueError as exc: + raise LeakageError( + f"{sample['sample_id']}: acquisition_date must be ISO-8601" + ) from exc + record_without_split = { + key: value for key, value in result.items() if key != "split" + } + result["record_sha256"] = canonical_hash(record_without_split) + return result + + +def assign_split_roles(source: dict[str, Any]) -> list[dict[str, Any]]: + """Assign complete leakage groups deterministically, or validate a frozen assignment.""" + raw_samples = [dict(item) for item in source.get("samples", [])] + mode = str(source.get("assignment_mode") or "preassigned") + split_presence = [bool(item.get("split")) for item in raw_samples] + if mode == "preassigned": + if raw_samples and not all(split_presence): + raise LeakageError("Preassigned mode requires a split on every sample") + return raw_samples + if mode != "deterministic_grouped": + raise LeakageError(f"Unsupported assignment_mode: {mode!r}") + if any(split_presence): + raise LeakageError( + "deterministic_grouped mode refuses partially or fully preassigned splits" + ) + config = source.get("split_assignment") or {} + roles = list( + config.get("roles") + or ["train", "val", "calibration", "test", "background-test"] + ) + if not roles or len(set(roles)) != len(roles) or set(roles) - ALL_SPLITS: + raise LeakageError("split_assignment.roles must be unique supported roles") + weights = { + role: float((config.get("weights") or {}).get(role, 1.0)) for role in roles + } + if any(not math.isfinite(value) or value <= 0 for value in weights.values()): + raise LeakageError("split_assignment weights must be positive finite values") + placeholder = [ + normalized_sample({**item, "split": "train"}) for item in raw_samples + ] + parent = list(range(len(placeholder))) + + def find(index: int) -> int: + while parent[index] != index: + parent[index] = parent[parent[index]] + index = parent[index] + return index + + def union(left: int, right: int) -> None: + left_root = find(left) + right_root = find(right) + if left_root != right_root: + parent[max(left_root, right_root)] = min(left_root, right_root) + + identity_fields = ( + "group_id", + "source_family", + "temporal_family", + "raw_image_sha256", + "processed_image_sha256", + "label_sha256", + "label_geometry_hash", + "parent_raster_id", + "acquisition_id", + ) + seen: dict[tuple[str, str], int] = {} + for index, sample in enumerate(placeholder): + identities = [(field, str(sample[field])) for field in identity_fields] + identities.extend(("object_id", str(value)) for value in sample["object_ids"]) + identities.extend( + ("native_feature_id", str(value)) for value in sample["native_feature_ids"] + ) + for identity in identities: + if identity in seen: + union(index, seen[identity]) + else: + seen[identity] = index + image_threshold = int(source.get("perceptual_hamming_threshold", 4)) + geometry_threshold = int(source.get("label_geometry_hamming_threshold", 2)) + buffer_m = float(source.get("independence_buffer_m") or 0) + for index, left in enumerate(placeholder): + for right_index, right in enumerate(placeholder[index + 1 :], start=index + 1): + if ( + fingerprint_distance( + left["perceptual_image_hash"], right["perceptual_image_hash"] + ) + <= image_threshold + or fingerprint_distance( + left["label_geometry_fingerprint"], + right["label_geometry_fingerprint"], + ) + <= geometry_threshold + or bbox_distance(left["bbox"], right["bbox"]) < buffer_m + ): + union(index, right_index) + components: dict[int, list[int]] = defaultdict(list) + for index in range(len(placeholder)): + components[find(index)].append(index) + if len(components) < len(roles): + raise LeakageError( + f"Not enough independent groups for required roles: {len(components)} < {len(roles)}" + ) + seed = str(config.get("seed") or "geointel-p4-group-split-v1") + groups = sorted( + components.values(), + key=lambda indices: canonical_hash( + { + "seed": seed, + "sample_ids": sorted( + placeholder[index]["sample_id"] for index in indices + ), + } + ), + ) + stratify_by = tuple(config.get("stratify_by") or ["task"]) + assigned_counts: Counter[str] = Counter() + stratum_counts: dict[str, Counter[str]] = defaultdict(Counter) + assignment: dict[int, str] = {} + for group_index, indices in enumerate(groups): + strata = { + f"{field}={placeholder[index].get(field) or (placeholder[index].get('metadata') or {}).get(field)}" + for index in indices + for field in stratify_by + } + candidates = ( + roles[group_index : group_index + 1] if group_index < len(roles) else roles + ) + role = min( + candidates, + key=lambda candidate: ( + assigned_counts[candidate] / weights[candidate] + + sum( + stratum_counts[stratum][candidate] / weights[candidate] + for stratum in strata + ), + candidate, + ), + ) + for index in indices: + assignment[index] = role + assigned_counts[role] += len(indices) + for stratum in strata: + stratum_counts[stratum][role] += len(indices) + return [ + {**sample, "split": assignment[index]} + for index, sample in enumerate(raw_samples) + ] + + +def leakage_findings( + samples: list[dict[str, Any]], + independence_buffer_m: float, + perceptual_hamming_threshold: int, + label_geometry_hamming_threshold: int, +) -> list[dict[str, Any]]: + findings: list[dict[str, Any]] = [] + identity_fields = ( + "sample_id", + "group_id", + "source_family", + "temporal_family", + "raw_image_sha256", + "processed_image_sha256", + "label_sha256", + "label_geometry_hash", + "parent_raster_id", + "acquisition_id", + "record_sha256", + ) + identities: dict[str, dict[str, set[str]]] = { + key: defaultdict(set) for key in identity_fields + } + object_splits: dict[str, set[str]] = defaultdict(set) + native_feature_splits: dict[str, set[str]] = defaultdict(set) + for sample in samples: + for key, index in identities.items(): + index[str(sample[key])].add(sample["split"]) + for object_id in sample["object_ids"]: + object_splits[str(object_id)].add(sample["split"]) + for feature_id in sample["native_feature_ids"]: + native_feature_splits[str(feature_id)].add(sample["split"]) + code_by_key = { + "sample_id": "S-SAMPLE-IDENTITY", + "group_id": "S-SPATIAL-GROUP", + "source_family": "S-SOURCE-FAMILY", + "temporal_family": "S-TEMPORAL-FAMILY", + "raw_image_sha256": "S-RAW-IMAGE-DUPLICATE", + "processed_image_sha256": "S-PROCESSED-IMAGE-DUPLICATE", + "label_sha256": "S-LABEL-DUPLICATE", + "label_geometry_hash": "S-LABEL-GEOMETRY-DUPLICATE", + "parent_raster_id": "S-PARENT-RASTER", + "acquisition_id": "S-ACQUISITION", + "record_sha256": "S-EXACT-DUPLICATE", + } + for key, index in identities.items(): + for value, splits in sorted(index.items()): + if len(splits) > 1: + findings.append( + { + "code": code_by_key[key], + "identity": value, + "splits": sorted(splits), + } + ) + for object_id, splits in sorted(object_splits.items()): + if len(splits) > 1: + findings.append( + { + "code": "S-OBJECT-INSTANCE", + "identity": object_id, + "splits": sorted(splits), + } + ) + for feature_id, splits in sorted(native_feature_splits.items()): + if len(splits) > 1: + findings.append( + { + "code": "S-NATIVE-FEATURE", + "identity": feature_id, + "splits": sorted(splits), + } + ) + for index, left in enumerate(samples): + for right in samples[index + 1 :]: + if left["split"] == right["split"]: + continue + image_distance = fingerprint_distance( + left["perceptual_image_hash"], right["perceptual_image_hash"] + ) + if image_distance <= perceptual_hamming_threshold: + findings.append( + { + "code": "S-PERCEPTUAL-IMAGE-NEAR-DUPLICATE", + "left": left["sample_id"], + "right": right["sample_id"], + "distance": image_distance, + "maximum_allowed_distance": perceptual_hamming_threshold, + } + ) + geometry_distance = fingerprint_distance( + left["label_geometry_fingerprint"], right["label_geometry_fingerprint"] + ) + if geometry_distance <= label_geometry_hamming_threshold: + findings.append( + { + "code": "S-LABEL-GEOMETRY-NEAR-DUPLICATE", + "left": left["sample_id"], + "right": right["sample_id"], + "distance": geometry_distance, + "maximum_allowed_distance": label_geometry_hamming_threshold, + } + ) + distance = bbox_distance(left["bbox"], right["bbox"]) + if distance < independence_buffer_m: + findings.append( + { + "code": "S-SPATIAL-OVERLAP", + "left": left["sample_id"], + "right": right["sample_id"], + "left_split": left["split"], + "right_split": right["split"], + "distance_m": distance, + "required_distance_m": independence_buffer_m, + } + ) + return findings + + +def build_manifests( + source: dict[str, Any], +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + if source.get("schema_version") != 1: + raise LeakageError("Unsupported source manifest schema_version") + metric_crs = validate_metric_crs(source.get("crs")) + assigned_source_samples = assign_split_roles(source) + samples = [normalized_sample(item) for item in assigned_source_samples] + if not samples: + raise LeakageError("Source manifest has no samples") + if len({item["sample_id"] for item in samples}) != len(samples): + raise LeakageError("Duplicate sample_id in source manifest") + buffer_m = float(source.get("independence_buffer_m") or 0) + if not math.isfinite(buffer_m) or buffer_m <= 0: + raise LeakageError("independence_buffer_m must be a positive finite number") + split_counts = Counter(item["split"] for item in samples) + required_splits = set(source.get("required_splits") or NORMATIVE_SPLITS) + unsupported_required = sorted(required_splits - ALL_SPLITS) + if unsupported_required: + raise LeakageError(f"Unsupported required splits: {unsupported_required}") + missing_splits = sorted(required_splits - set(split_counts)) + if missing_splits: + raise LeakageError(f"Required splits are absent: {missing_splits}") + perceptual_threshold = int(source.get("perceptual_hamming_threshold", 4)) + geometry_threshold = int(source.get("label_geometry_hamming_threshold", 2)) + if not 0 <= perceptual_threshold < 64 or not 0 <= geometry_threshold < 64: + raise LeakageError("Near-duplicate Hamming thresholds must be between 0 and 63") + findings = leakage_findings( + samples, buffer_m, perceptual_threshold, geometry_threshold + ) + canonical_source = dict(source) + canonical_source["samples"] = sorted( + source.get("samples", []), key=lambda item: str(item.get("sample_id", "")) + ) + source_hash = canonical_hash(canonical_source) + + development_samples = sorted( + (item for item in samples if item["split"] in DEVELOPMENT_SPLITS), + key=lambda item: item["sample_id"], + ) + protected_samples = sorted( + (item for item in samples if item["split"] in PROTECTED_SPLITS), + key=lambda item: item["sample_id"], + ) + common = { + "schema_version": SCHEMA_VERSION, + "generator_version": GENERATOR_VERSION, + "dataset_version": source["dataset_version"], + "source_manifest_sha256": source_hash, + "crs": metric_crs, + "independence_buffer_m": buffer_m, + "perceptual_hamming_threshold": perceptual_threshold, + "label_geometry_hamming_threshold": geometry_threshold, + "assignment_mode": str(source.get("assignment_mode") or "preassigned"), + "assignment_algorithm": "group-before-split-deficit-balancer-v1", + "assignment_seed": str( + (source.get("split_assignment") or {}).get("seed") + or "geointel-p4-group-split-v1" + ), + "claim_boundary": source.get("claim_boundary"), + } + development = { + **common, + "manifest_role": "development_and_calibration", + "allowed_splits": sorted(DEVELOPMENT_SPLITS), + "training_access_allowed_by_split": { + "train": True, + "val": False, + "calibration": False, + }, + "selection_access_allowed_by_split": { + "train": False, + "val": True, + "calibration": True, + }, + "samples": development_samples, + } + development["manifest_sha256"] = canonical_hash(development) + protected = { + **common, + "manifest_role": "protected_release_only", + "allowed_splits": sorted(PROTECTED_SPLITS), + "training_access_allowed": False, + "selection_use_allowed": False, + "labels_available_by_split": { + "test": "frozen_evaluator_only", + "background-test": "frozen_evaluator_only", + "challenge": "sealed_external", + }, + "access_policy": ( + "Only the frozen release evaluator may consume test/background-test labels; " + "challenge labels remain external/sealed." + ), + "samples": protected_samples, + } + protected["manifest_sha256"] = canonical_hash(protected) + leakage = { + "schema_version": SCHEMA_VERSION, + "generator_version": GENERATOR_VERSION, + "status": "pass" if not findings else "fail", + "source_manifest_sha256": source_hash, + "development_manifest_sha256": development["manifest_sha256"], + "protected_manifest_sha256": protected["manifest_sha256"], + "inventory_total": len(samples), + "split_counts": dict(sorted(split_counts.items())), + "checked_identity_fields": [ + "sample_id", + "group_id", + "source_family", + "temporal_family", + "object_ids", + "native_feature_ids", + "raw_image_sha256", + "processed_image_sha256", + "label_sha256", + "record_sha256", + "perceptual_image_hash", + "label_geometry_hash", + "label_geometry_fingerprint", + "parent_raster_id", + "acquisition_id", + "bbox_distance", + ], + "crs_validation": {"status": "pass", "crs": metric_crs, "distance_units": "m"}, + "independence_buffer_m": buffer_m, + "perceptual_hamming_threshold": perceptual_threshold, + "label_geometry_hamming_threshold": geometry_threshold, + "finding_count": len(findings), + "findings": findings, + } + return development, protected, leakage + + +def protected_identities(protected_manifest: dict[str, Any]) -> set[str]: + identities: set[str] = set() + scalar_fields = ( + "sample_id", + "group_id", + "record_sha256", + "source_family", + "temporal_family", + "raw_image_sha256", + "processed_image_sha256", + "label_sha256", + "label_geometry_hash", + "perceptual_image_hash", + "label_geometry_fingerprint", + "parent_raster_id", + "acquisition_id", + ) + for item in protected_manifest.get("samples", []): + identities.update(str(item[field]) for field in scalar_fields) + identities.update(str(value) for value in item.get("object_ids", [])) + identities.update(str(value) for value in item.get("native_feature_ids", [])) + return identities + + +def assert_training_inputs_safe( + input_paths: Iterable[Path], + input_records: Iterable[dict[str, Any]], + protected_manifest: dict[str, Any], +) -> None: + """Refuse non-train roles and protected identities at a fitting boundary.""" + forbidden = protected_identities(protected_manifest) + violations: list[str] = [] + for path in input_paths: + lowered = path.as_posix().lower() + if any( + token in lowered + for token in ("protected", "holdout", "challenge", "background-test") + ): + violations.append(f"protected_path:{path}") + scalar_fields = ( + "sample_id", + "group_id", + "record_sha256", + "source_family", + "temporal_family", + "raw_image_sha256", + "processed_image_sha256", + "label_sha256", + "label_geometry_hash", + "perceptual_image_hash", + "label_geometry_fingerprint", + "parent_raster_id", + "acquisition_id", + ) + for record in input_records: + if record.get("split") not in TRAIN_SPLITS: + violations.append( + f"non_train_role:{record.get('sample_id')}:{record.get('split')}" + ) + values = {str(record.get(field, "")) for field in scalar_fields} + values.update(str(value) for value in record.get("object_ids", [])) + values.update(str(value) for value in record.get("native_feature_ids", [])) + overlap = sorted((values - {""}) & forbidden) + if overlap: + violations.append(f"protected_identity:{','.join(overlap)}") + if violations: + raise LeakageError( + "Training input firewall blocked: " + "; ".join(sorted(violations)) + ) + + +def generate(source_path: Path, output_dir: Path) -> dict[str, Any]: + source = json.loads(source_path.read_text(encoding="utf-8")) + development, protected, leakage = build_manifests(source) + output_dir.mkdir(parents=True, exist_ok=True) + generation_status = { + "schema_version": SCHEMA_VERSION, + "generator_version": GENERATOR_VERSION, + "status": leakage["status"], + "source_manifest_sha256": leakage["source_manifest_sha256"], + "development_manifest_sha256": ( + development["manifest_sha256"] if leakage["status"] == "pass" else None + ), + "protected_manifest_sha256": ( + protected["manifest_sha256"] if leakage["status"] == "pass" else None + ), + } + write_json(output_dir / "generation-status.json", generation_status) + write_json(output_dir / "leakage-gate-report.json", leakage) + if leakage["status"] != "pass": + raise LeakageError( + f"Leakage gate failed with {leakage['finding_count']} findings" + ) + train_samples = [ + item for item in development["samples"] if item["split"] == "train" + ] + assert_training_inputs_safe([], train_samples, protected) + write_json(output_dir / "development-split-manifest.json", development) + write_json(output_dir / "protected-split-manifest.json", protected) + return { + "development": development, + "protected": protected, + "leakage": leakage, + "generation_status": generation_status, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + result = generate(args.source, args.output_dir) + except (OSError, json.JSONDecodeError, LeakageError) as exc: + print(json.dumps({"status": "fail", "error": str(exc)}, indent=2)) + return 2 + print( + json.dumps( + { + "status": "pass", + "development_manifest_sha256": result["development"]["manifest_sha256"], + "protected_manifest_sha256": result["protected"]["manifest_sha256"], + "split_counts": result["leakage"]["split_counts"], + }, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_accuracy_phase4_benchmark.py b/scripts/run_accuracy_phase4_benchmark.py new file mode 100644 index 00000000..41c3a7e5 --- /dev/null +++ b/scripts/run_accuracy_phase4_benchmark.py @@ -0,0 +1,996 @@ +#!/usr/bin/env python3 +"""Run the complete local Phase 4 evaluation workflow from frozen inputs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import subprocess +import sys +from importlib import metadata as importlib_metadata +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +BACKEND_ROOT = ROOT / "backend" +for entry in (str(ROOT), str(BACKEND_ROOT), str(ROOT / "scripts")): + if entry not in sys.path: + sys.path.insert(0, entry) + +from accuracy_phase4_evaluator import EVALUATOR_VERSION, canonical_hash, evaluate_cases # noqa: E402 +from generate_accuracy_phase4_splits import ( # noqa: E402 + GENERATOR_VERSION, + LeakageError, + assert_training_inputs_safe, + build_manifests, +) +from run_golden_qa_benchmark import run_benchmark # noqa: E402 + +WORKFLOW_VERSION = "2.0.0" +BENCHMARK_ID = "geointel-p4-reference-harness-v2" +GATE_STATES = {"pass", "fail", "not_evaluable"} + + +class EvidenceConflictError(RuntimeError): + """Raised when an immutable evidence path already contains different bytes.""" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def json_bytes(payload: Any) -> bytes: + return ( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ).encode("utf-8") + + +def write_json_immutable(path: Path, payload: Any) -> None: + content = json_bytes(payload) + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + if path.read_bytes() != content: + raise EvidenceConflictError( + f"Refusing to overwrite immutable evidence with different content: {path}" + ) + return + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_bytes(content) + temporary.replace(path) + + +def repository_commit(repo_root: Path) -> str | None: + try: + return subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return None + + +def dependency_version(distribution: str) -> str | None: + try: + return importlib_metadata.version(distribution) + except importlib_metadata.PackageNotFoundError: + return None + + +def repository_file(repo_root: Path, relative_path: str) -> dict[str, Any]: + path = repo_root / relative_path + return { + "path": relative_path, + "sha256": sha256(path), + "size_bytes": path.stat().st_size, + } + + +def canonical_golden_baseline() -> dict[str, Any]: + result = run_benchmark() + scenarios = [] + for item in result["scenarios"]: + normalized = dict(item) + normalized.pop("quality_check_id", None) + scenarios.append(normalized) + return { + "status": result["status"], + "version": result["version"], + "scenario_count": result["scenario_count"], + "scenarios": scenarios, + "persistence": result["persistence"], + "implementation": "backend/app/services/qa_service.py via scripts/run_golden_qa_benchmark.py", + "claim_boundary": "Reference implementation regression evidence; not production model accuracy.", + "content_sha256": canonical_hash(scenarios), + } + + +def product_baseline_manifest_gate( + repo_root: Path, + manifest_path: Path, + active_model: dict[str, Any], +) -> dict[str, Any]: + relative = manifest_path + try: + relative = manifest_path.resolve().relative_to(repo_root.resolve()) + except (OSError, ValueError): + return { + "status": "fail", + "reason": "Product baseline manifest must reside inside the governed repository evidence root.", + "path": str(manifest_path), + } + if not manifest_path.is_file(): + return { + "status": "not_evaluable", + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "expected_path": relative.as_posix(), + } + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return { + "status": "fail", + "reason": f"Unreadable product baseline manifest: {exc}", + } + required_keys = { + "schema_version", + "status", + "synthetic", + "active_model_sha256", + "evaluator_sha256", + "configuration_sha256", + "protected_split_manifest", + "authoritative_reference_manifest", + "raw_predictions", + "metric_report", + "inference", + } + missing = sorted(required_keys - set(manifest)) + violations: list[str] = [] + if missing: + violations.append(f"missing_fields:{','.join(missing)}") + if manifest.get("status") != "pass": + violations.append("manifest_status_not_pass") + if manifest.get("synthetic") is not False: + violations.append("synthetic_or_unspecified") + if manifest.get("active_model_sha256") != active_model.get("sha256"): + violations.append("active_model_hash_mismatch") + evaluator_path = repo_root / "scripts/accuracy_phase4_evaluator.py" + if manifest.get("evaluator_sha256") != sha256(evaluator_path): + violations.append("evaluator_hash_mismatch") + inference = manifest.get("inference") or {} + if inference.get("executed") is not True: + violations.append("inference_not_executed") + if inference.get("test_used_for_selection") is not False: + violations.append("protected_test_selection_policy_invalid") + if not str(inference.get("device") or "").lower().startswith("cuda"): + violations.append("governed_cuda_execution_not_proven") + checked_artifacts: list[dict[str, Any]] = [] + for key in ( + "protected_split_manifest", + "authoritative_reference_manifest", + "raw_predictions", + "metric_report", + ): + item = manifest.get(key) or {} + item_path = repo_root / str(item.get("path") or "") + try: + item_path.resolve().relative_to(repo_root.resolve()) + except (OSError, ValueError): + violations.append(f"{key}_outside_repository") + continue + if not item_path.is_file(): + violations.append(f"{key}_missing") + continue + observed_hash = sha256(item_path) + checked_artifacts.append( + { + "role": key, + "path": item_path.relative_to(repo_root).as_posix(), + "sha256": observed_hash, + } + ) + if observed_hash != item.get("sha256"): + violations.append(f"{key}_hash_mismatch") + return { + "status": "fail" if violations else "pass", + "path": relative.as_posix(), + "manifest_sha256": sha256(manifest_path), + "violations": sorted(violations), + "checked_artifacts": checked_artifacts, + "evidence": ( + "A non-synthetic active-model inference, protected split, authority reference, " + "raw predictions and metric report are all checksum-bound." + if not violations + else None + ), + } + + +def readiness_snapshot(repo_root: Path) -> dict[str, Any]: + status_path = repo_root / "docs/accuracy-program/status.json" + p3_path = repo_root / "artifacts/evidence/accuracy/P3/full-scan-manifest.json" + leakage_path = repo_root / "artifacts/evidence/accuracy/P3/leakage-report.json" + status = json.loads(status_path.read_text(encoding="utf-8")) + p3 = json.loads(p3_path.read_text(encoding="utf-8")) + leakage = json.loads(leakage_path.read_text(encoding="utf-8")) + ml_data = status.get("ml_data") or {} + return { + "schema_version": 1, + "source_paths": { + "phase3_full_scan": repository_file( + repo_root, "artifacts/evidence/accuracy/P3/full-scan-manifest.json" + ), + "phase3_leakage": repository_file( + repo_root, "artifacts/evidence/accuracy/P3/leakage-report.json" + ), + }, + "active_model": (status.get("runtime") or {}).get("active_model"), + "v56_review_and_split": (ml_data.get("v56") or {}), + "protected_test_isolation": ml_data.get("protected_test_isolation"), + "phase3_scan": { + "scan_id": p3.get("scan_id"), + "content_hash": p3.get("content_hash"), + "grb_consistency": p3.get("grb_consistency"), + }, + "phase3_leakage_status": leakage.get("status"), + "authority_requirements": [ + {"task": "building_validation", "zone": "flanders", "primary": "grb"}, + {"task": "building_validation", "zone": "wallonia", "primary": "picc"}, + {"task": "building_validation", "zone": "brussels", "primary": "urbis"}, + {"task": "terrain_height", "zone": "flanders", "primary": "dhmv"}, + {"task": "terrain_height", "zone": "wallonia", "primary": "spw_terrain"}, + { + "task": "north_sea_bathymetry", + "zone": "belgian_north_sea", + "primary": "mdk", + }, + { + "task": "imagery_corroboration", + "zone": "belgium", + "primary": "official_orthophoto", + "contextual": "sentinel-2", + }, + ], + } + + +def product_gate_evidence( + repo_root: Path, + snapshot: dict[str, Any], + product_baseline_manifest: Path, +) -> dict[str, Any]: + v56 = snapshot["v56_review_and_split"] + active_model = snapshot["active_model"] or {} + configured_path = Path(str(active_model.get("path") or "")) + p3_grb = snapshot["phase3_scan"].get("grb_consistency") or {} + baseline_gate = product_baseline_manifest_gate( + repo_root, + product_baseline_manifest, + active_model, + ) + return { + "active_model_available_and_hash_verified": { + "status": "pass" + if configured_path.is_file() + and sha256(configured_path) == active_model.get("sha256") + else "not_evaluable", + "configured_path": str(configured_path), + "configured_sha256": active_model.get("sha256"), + "reason": None + if configured_path.is_file() + else "Configured active model is not locally accessible.", + }, + "authoritative_reference_portfolio_available": { + "status": "not_evaluable", + "requirements": snapshot["authority_requirements"], + "observed_grb": p3_grb, + "reason": "A task- and zone-complete governed GRB/PICC/UrbIS/DHMV/SPW/MDK reference portfolio is not locally accessible.", + }, + "human_review_complete": { + "status": "pass" if bool(v56.get("review_complete")) else "fail", + "observed": v56.get("reviewed_sample_count"), + "required": v56.get("sample_count"), + }, + "split_independence": { + "status": "pass" if bool(v56.get("split_independence_proven")) else "fail", + "cross_split_pairs_below_2000_m": v56.get("cross_split_pairs_below_2000_m"), + }, + "phase3_leakage_resolved": { + "status": "pass" + if snapshot.get("phase3_leakage_status") == "pass" + else "not_evaluable", + "observed": snapshot.get("phase3_leakage_status"), + }, + "protected_storage_isolation": { + "status": "pass" + if snapshot.get("protected_test_isolation") is True + else "not_evaluable", + "reason": "A physically isolated vault, scoped credentials and immutable access log are not proven.", + }, + "executed_product_incumbent_baseline": baseline_gate, + "representative_product_subgroup_support": { + "status": "not_evaluable", + "reason": "No real protected raw-prediction portfolio is available for AOI/region/context subgroup support.", + }, + } + + +def build_release_gate_report( + split_result: dict[str, Any], + evaluation: dict[str, Any], + portfolio: dict[str, Any], + golden: dict[str, Any], + firewall_checks: dict[str, bool], + product_gates: dict[str, Any], +) -> dict[str, Any]: + declared_families = {item["task"] for item in evaluation["task_inventory"]} + observed_families = {item["task"] for item in evaluation["results"]} + required_split_roles = {"train", "val", "calibration", "test", "background-test"} + observed_split_roles = set(split_result["leakage"]["split_counts"]) + required_raw_fields = { + "references", + "predictions_pre_filter", + "predictions_post_filter", + "config", + "input_lineage", + "portfolio_lineage", + } + raw_violations = [ + item["sample_id"] + for item in evaluation["results"] + if not required_raw_fields <= set(item.get("raw") or {}) + ] + protected_policy = portfolio.get("protected_policy") or {} + selection_contract_valid = ( + protected_policy.get("operating_point_selection_allowed") is False + and protected_policy.get("diagnostic_curves_select_operating_point") is False + and protected_policy.get("test_feedback_allowed") is False + and protected_policy.get("threshold_selection_source") + == "pre_registered_configuration_only" + and all( + isinstance((item.get("raw") or {}).get("config"), dict) + for item in evaluation["results"] + ) + ) + empty_case = next( + ( + item + for item in evaluation["results"] + if item["sample_id"] == "background-test-pure-empty" + ), + None, + ) + empty_metrics = (empty_case or {}).get("metrics") or {} + null_semantics_valid = ( + empty_case is not None + and empty_metrics.get("reference_count") == 0 + and empty_metrics.get("prediction_count") == 0 + and empty_metrics.get("precision") is None + and empty_metrics.get("recall") is None + and empty_metrics.get("f1") is None + ) + subgroup_report = evaluation.get("subgroups") or {} + subgroup_contract_valid = ( + subgroup_report.get("overall_status") + in {"not_evaluable", "evaluable_no_release_target"} + and isinstance(subgroup_report.get("dimensions"), dict) + and bool(subgroup_report.get("dimensions")) + and all( + isinstance(dimension.get("strata"), dict) + and isinstance(dimension.get("worst_stratum_by_task"), dict) + for dimension in subgroup_report["dimensions"].values() + ) + ) + capability_inventory = evaluation.get("task_inventory") or [] + capability_contract_valid = bool(capability_inventory) and all( + item.get("capability_id") + and item.get("implementation_paths") + and item.get("suitable_metrics") + and item.get("evaluation_status") + in { + "synthetic_contract_case_only", + "covered_by_family_not_separately_benchmarked", + "not_separately_benchmarked", + "no_independent_accuracy_score_underlying_tool_results_are_authoritative", + "synthetic_metric_contract_only_no_generic_learned_classifier_claim", + } + for item in capability_inventory + ) + local_gates = { + "all_declared_evaluator_families_exercised": { + "status": "pass" if declared_families == observed_families else "fail", + "declared": sorted(declared_families), + "observed": sorted(observed_families), + }, + "implemented_capability_inventory": { + "status": "pass" if capability_contract_valid else "fail", + "capability_count": len(capability_inventory), + }, + "normative_split_roles_and_leakage": { + "status": ( + "pass" + if required_split_roles <= observed_split_roles + and split_result["leakage"]["status"] == "pass" + else "fail" + ), + "required_roles": sorted(required_split_roles), + "observed_roles": sorted(observed_split_roles), + "leakage_status": split_result["leakage"]["status"], + }, + "manifest_training_firewall_contract": { + "status": "pass" + if firewall_checks and all(firewall_checks.values()) + else "fail", + "checks": firewall_checks, + }, + "protected_operating_point_contract": { + "status": "pass" if selection_contract_valid else "fail", + "evidence": ( + "Protected cases carry pre-registered configurations. Fixed AP/risk-coverage " + "diagnostics cannot select an operating point or feed back into training." + ), + }, + "complete_raw_predictions_retained": { + "status": "pass" if not raw_violations else "fail", + "violating_samples": raw_violations, + }, + "reference_implementation_baseline": { + "status": "pass" if golden.get("status") == "passed" else "fail", + }, + "stratified_metric_contract": { + "status": "pass" if subgroup_contract_valid else "fail", + "observed_overall_status": subgroup_report.get("overall_status"), + }, + "undefined_metric_truth_table": { + "status": "pass" if null_semantics_valid else "fail", + "sample_id": "background-test-pure-empty", + }, + } + invalid_gate_states = { + f"{family}.{name}": item.get("status") + for family, gates in (("local", local_gates), ("product", product_gates)) + for name, item in gates.items() + if item.get("status") not in GATE_STATES + } + local_green = not invalid_gate_states and all( + item["status"] == "pass" for item in local_gates.values() + ) + product_green = not invalid_gate_states and all( + item["status"] == "pass" for item in product_gates.values() + ) + if not local_green: + overall_status = "fail" + elif not product_green: + overall_status = ( + "not_evaluable" + if any(item["status"] == "not_evaluable" for item in product_gates.values()) + else "fail" + ) + else: + overall_status = "pass" + return { + "schema_version": 2, + "gate_policy": "geointel-p4-evaluation-harness-v2", + "status": overall_status, + "phase_decision": "ready_for_phase5" if overall_status == "pass" else "blocked", + "local_harness_status": "pass" if local_green else "fail", + "product_benchmark_status": "pass" if product_green else "not_evaluable", + "promotion_allowed": False, + "numeric_model_release_targets": "not_frozen_without_reviewed_representative_incumbent_baseline", + "local_gates": local_gates, + "product_gates": product_gates, + "invalid_gate_states": invalid_gate_states, + "critical_subgroup_policy": ( + "Any required subgroup with insufficient support, missing metrics, a failed " + "non-inferiority comparison or regression blocks promotion; averages cannot override it." + ), + "decision": ( + "The local contract harness passes, but Phase 4 remains in progress and Phase 5 " + "is not ready until a real protected product incumbent baseline is evaluable." + if local_green and not product_green + else "All Phase 4 completion gates pass." + if local_green and product_green + else "The local Phase 4 harness has failing contract gates." + ), + } + + +def build_evidence_manifest( + artifacts: dict[str, Any], + benchmark_manifest: dict[str, Any], +) -> dict[str, Any]: + retained = [] + for name, payload in sorted(artifacts.items()): + content = json_bytes(payload) + retained.append( + { + "path": name, + "sha256": hashlib.sha256(content).hexdigest(), + "size_bytes": len(content), + } + ) + return { + "schema_version": 2, + "phase": "P4", + "benchmark_id": benchmark_manifest["benchmark_id"], + "artifacts": retained, + "artifact_count": len(retained), + "claim_boundary": ( + "Immutable local reference-harness evidence only; product accuracy and release " + "remain blocked while product gates are not evaluable or fail." + ), + } + + +def firewall_contract_checks( + split_result: dict[str, Any], + protected_cases_path: Path, +) -> dict[str, bool]: + development = split_result["development"]["samples"] + protected = split_result["protected"] + train = [item for item in development if item["split"] == "train"] + validation = next(item for item in development if item["split"] == "val") + protected_item = protected["samples"][0] + checks: dict[str, bool] = {} + try: + assert_training_inputs_safe([], train, protected) + except LeakageError: + checks["clean_train_allowed"] = False + else: + checks["clean_train_allowed"] = True + for name, paths, records in ( + ("non_train_role_blocked", [], [validation]), + ("protected_path_blocked", [protected_cases_path], []), + ( + "renamed_protected_lineage_blocked", + [], + [{**train[0], "source_family": protected_item["source_family"]}], + ), + ): + try: + assert_training_inputs_safe(paths, records, protected) + except LeakageError: + checks[name] = True + else: + checks[name] = False + return checks + + +def runtime_identity() -> dict[str, Any]: + return { + "python": platform.python_version(), + "python_implementation": platform.python_implementation(), + "platform": platform.platform(), + "dependencies": { + "numpy": dependency_version("numpy"), + "pyproj": dependency_version("pyproj"), + "shapely": dependency_version("shapely"), + }, + "execution_device": "CPU deterministic evaluator arithmetic; no production model inference", + "cuda_used_for_reference_harness": False, + } + + +def metric_results_without_raw(evaluation: dict[str, Any]) -> list[dict[str, Any]]: + return [ + {key: value for key, value in item.items() if key not in {"raw", "failures"}} + for item in evaluation["results"] + ] + + +def build_input_manifest( + repo_root: Path, + snapshot: dict[str, Any], +) -> dict[str, Any]: + code_paths = [ + "scripts/accuracy_phase4_evaluator.py", + "scripts/generate_accuracy_phase4_splits.py", + "scripts/run_accuracy_phase4_benchmark.py", + "scripts/run_golden_qa_benchmark.py", + "backend/app/services/qa_service.py", + ] + input_paths = [ + "fixtures/accuracy/p4/split-source-manifest.json", + "fixtures/accuracy/p4/protected-baseline-cases.json", + "fixtures/golden/golden_qa_benchmarks.json", + "docs/accuracy-program/05-metric-framework.md", + "docs/accuracy-program/07-source-authority-matrix.md", + "artifacts/evidence/accuracy/P3/full-scan-manifest.json", + "artifacts/evidence/accuracy/P3/leakage-report.json", + ] + return { + "schema_version": 2, + "repository_commit": repository_commit(repo_root), + "code": [repository_file(repo_root, path) for path in code_paths], + "inputs": [repository_file(repo_root, path) for path in input_paths], + "readiness_snapshot": snapshot, + "readiness_snapshot_sha256": canonical_hash(snapshot), + "runtime": runtime_identity(), + "model_execution": { + "status": "not_evaluable", + "reason": "The configured active model and governed protected product inputs are not locally accessible.", + "configured_active_model": snapshot.get("active_model"), + }, + } + + +def run_workflow( + repo_root: Path, + output_dir: Path, + product_baseline_manifest: Path | None = None, +) -> dict[str, Any]: + source_path = repo_root / "fixtures/accuracy/p4/split-source-manifest.json" + cases_path = repo_root / "fixtures/accuracy/p4/protected-baseline-cases.json" + source = json.loads(source_path.read_text(encoding="utf-8")) + development, protected, leakage = build_manifests(source) + if leakage["status"] != "pass": + raise LeakageError( + f"Leakage gate failed with {leakage['finding_count']} findings" + ) + split_result = { + "development": development, + "protected": protected, + "leakage": leakage, + "generation_status": { + "schema_version": 1, + "generator_version": GENERATOR_VERSION, + "status": "pass", + "source_manifest_sha256": leakage["source_manifest_sha256"], + "development_manifest_sha256": development["manifest_sha256"], + "protected_manifest_sha256": protected["manifest_sha256"], + }, + } + protected_evaluation_ids = { + item["sample_id"] + for item in protected["samples"] + if item["split"] in {"test", "background-test"} + } + portfolio = json.loads(cases_path.read_text(encoding="utf-8")) + evaluation = evaluate_cases(cases_path, protected_evaluation_ids) + firewall_checks = firewall_contract_checks(split_result, cases_path) + golden = canonical_golden_baseline() + snapshot = readiness_snapshot(repo_root) + baseline_path = ( + product_baseline_manifest + if product_baseline_manifest is not None + else repo_root / "artifacts/evidence/accuracy/P4/product-baseline-manifest.json" + ) + product_gates = product_gate_evidence(repo_root, snapshot, baseline_path) + gate_report = build_release_gate_report( + split_result, + evaluation, + portfolio, + golden, + firewall_checks, + product_gates, + ) + input_manifest = build_input_manifest(repo_root, snapshot) + evaluation_contract = { + "schema_version": 2, + "benchmark_id": BENCHMARK_ID, + "workflow_version": WORKFLOW_VERSION, + "evaluator_version": EVALUATOR_VERSION, + "split_generator_version": GENERATOR_VERSION, + "evaluator_families": sorted(evaluation["evaluated_task_families"]), + "implemented_capabilities": evaluation["task_inventory"], + "metric_contract": repository_file( + repo_root, "docs/accuracy-program/05-metric-framework.md" + ), + "gate_states": sorted(GATE_STATES), + "protected_policy": evaluation["protected_policy"], + "undefined_value_policy": ( + "Undefined denominators are null with numerator, denominator and support; " + "they are never coerced to a perfect score." + ), + "claim_boundary": evaluation["claim_boundary"], + } + raw_items = [item["raw"] for item in evaluation["results"]] + raw_predictions = { + "schema_version": 2, + "evaluator_version": EVALUATOR_VERSION, + "portfolio_file_sha256": evaluation["portfolio_file_sha256"], + "items": raw_items, + "items_canonical_json_sha256": canonical_hash(raw_items), + "hash_specification": evaluation["hash_specification"], + } + metric_results = metric_results_without_raw(evaluation) + metric_report = { + key: value + for key, value in evaluation.items() + if key not in {"results", "failures"} + } + metric_report["results"] = metric_results + metric_report["metric_results_canonical_json_sha256"] = canonical_hash( + metric_results + ) + metric_report["full_results_canonical_json_sha256"] = evaluation[ + "results_canonical_json_sha256" + ] + failure_gallery = { + "schema_version": 2, + "taxonomy": "docs/accuracy-program/05-metric-framework.md section 4", + "failure_count": len(evaluation["failures"]), + "items": evaluation["failures"], + "items_canonical_json_sha256": canonical_hash(evaluation["failures"]), + "rendering_status": ( + "machine_readable_examples_retained; a visual production gallery requires " + "controlled access to protected imagery" + ), + } + taxonomy_entries = sorted( + {(item["error_code"], item["kind"]) for item in evaluation["failures"]} + ) + error_taxonomy = { + "schema_version": 2, + "source": "docs/accuracy-program/05-metric-framework.md section 4", + "observed_codes": [ + {"error_code": code, "kind": kind} for code, kind in taxonomy_entries + ], + "observed_failure_count": len(evaluation["failures"]), + "claim_boundary": evaluation["claim_boundary"], + } + object_task_names = { + "object_detection", + "footprint_segmentation", + "vector_comparison", + "change_detection", + "geospatial_data_validation", + } + object_metrics = { + "schema_version": 2, + "status": "fixture_contract_only", + "results": [ + item for item in metric_results if item["task"] in object_task_names + ], + } + tile_metrics = { + "schema_version": 2, + "status": "fixture_contract_only", + "results": [ + item + for item in metric_results + if item["task"] in {"raster_classification", "terrain_interpretation"} + ], + } + aoi_metrics = { + "schema_version": 2, + "status": "not_evaluable", + "reason": ( + "Synthetic single-case fixtures do not provide independent product AOI clusters. " + "AOI micro/macro and cluster-bootstrap evidence requires the protected product corpus." + ), + "required_future_outputs": [ + "per-AOI primary metrics", + "micro and macro aggregation", + "paired candidate-minus-incumbent deltas", + "cluster-bootstrap confidence intervals", + ], + } + stratified_metrics = evaluation["subgroups"] + calibration_items = [] + for item in metric_results: + calibration = item["metrics"].get("calibration") + coverage_risk = item["metrics"].get("coverage_risk") + if calibration is not None or coverage_risk is not None: + calibration_items.append( + { + "sample_id": item["sample_id"], + "task": item["task"], + "calibration": calibration, + "coverage_risk": coverage_risk, + } + ) + calibration_metrics = { + "schema_version": 2, + "status": "fixture_diagnostic_only", + "selection_allowed": False, + "items": calibration_items, + "note": ( + "Fixed diagnostic bins and risk thresholds test metric arithmetic; they do not " + "select or change any operating point." + ), + } + latency_reliability = { + "schema_version": 2, + "status": "not_evaluable", + "model_inference_executed": False, + "reason": ( + "The reference harness performs deterministic evaluator arithmetic only. " + "GPU latency, VRAM, throughput and failure-rate gates require the real active model." + ), + } + human_review_summary = { + "schema_version": 2, + "status": product_gates["human_review_complete"]["status"], + "reviewed": product_gates["human_review_complete"].get("observed"), + "required": product_gates["human_review_complete"].get("required"), + "source": "readiness-snapshot.json bound to Phase-1/3 evidence", + "ai_review_is_human_signoff": False, + } + candidate_vs_incumbent = { + "schema_version": 2, + "status": "not_evaluable", + "reason": ( + "Phase 4 has no valid real incumbent product baseline and no pre-registered " + "candidate; synthetic fixture values cannot define non-inferiority." + ), + "future_gate_contract": { + "unit": "paired independent AOI", + "global_and_critical_subgroups_required": True, + "missing_or_insufficient_support": "not_evaluable", + "aggregate_improvement_may_mask_subgroup_regression": False, + "numeric_margin": "to_be_frozen_before_protected_access", + }, + } + input_manifest_file_sha256 = hashlib.sha256(json_bytes(input_manifest)).hexdigest() + benchmark_manifest = { + "schema_version": 2, + "benchmark_id": BENCHMARK_ID, + "workflow_version": WORKFLOW_VERSION, + "evaluator_version": EVALUATOR_VERSION, + "split_generator_version": GENERATOR_VERSION, + "repository_commit": input_manifest["repository_commit"], + "input_manifest": { + "path": "input-manifest.json", + "sha256": input_manifest_file_sha256, + }, + "inputs": { + "split_source": repository_file( + repo_root, "fixtures/accuracy/p4/split-source-manifest.json" + ), + "protected_cases": repository_file( + repo_root, "fixtures/accuracy/p4/protected-baseline-cases.json" + ), + "golden_qa_manifest": repository_file( + repo_root, "fixtures/golden/golden_qa_benchmarks.json" + ), + "phase3_full_scan": repository_file( + repo_root, "artifacts/evidence/accuracy/P3/full-scan-manifest.json" + ), + "phase3_leakage": repository_file( + repo_root, "artifacts/evidence/accuracy/P3/leakage-report.json" + ), + }, + "code": input_manifest["code"], + "runtime": input_manifest["runtime"], + "split_manifests": { + "development_sha256": development["manifest_sha256"], + "protected_sha256": protected["manifest_sha256"], + "leakage_status": leakage["status"], + }, + "inference_and_selection": { + "synthetic_reference_harness": True, + "production_model_inference_executed": False, + "test_used_for_selection": False, + "background_test_used_for_selection": False, + "challenge_labels_available": False, + "raw_predictions_retained": True, + "threshold_source": "pre_registered_configuration_only", + }, + "evaluation_results_canonical_json_sha256": evaluation[ + "results_canonical_json_sha256" + ], + "reference_baseline_sha256": golden["content_sha256"], + "claim_boundary": evaluation["claim_boundary"], + } + benchmark_manifest["manifest_sha256"] = canonical_hash(benchmark_manifest) + gate_report["benchmark_manifest_sha256"] = benchmark_manifest["manifest_sha256"] + workflow_summary = { + "schema_version": 2, + "status": gate_report["status"], + "phase_decision": gate_report["phase_decision"], + "local_harness_status": gate_report["local_harness_status"], + "product_benchmark_status": gate_report["product_benchmark_status"], + "benchmark_manifest_sha256": benchmark_manifest["manifest_sha256"], + "split_counts": leakage["split_counts"], + "task_family_count": evaluation["task_count"], + "implemented_capability_count": len(evaluation["task_inventory"]), + "case_count": evaluation["case_count"], + "failure_example_count": len(evaluation["failures"]), + "evaluation_results_canonical_json_sha256": evaluation[ + "results_canonical_json_sha256" + ], + "reference_baseline_sha256": golden["content_sha256"], + "promotion_allowed": False, + "phase4_done": gate_report["status"] == "pass", + "phase5_ready": gate_report["status"] == "pass", + } + artifacts: dict[str, Any] = { + "acceptance-gates.json": gate_report, + "aoi-metrics.json": aoi_metrics, + "baseline-raw-predictions.json": raw_predictions, + "benchmark-manifest.json": benchmark_manifest, + "calibration-metrics.json": calibration_metrics, + "candidate-vs-incumbent.json": candidate_vs_incumbent, + "development-split-manifest.json": development, + "error-taxonomy.json": error_taxonomy, + "evaluation-contract.json": evaluation_contract, + "failure-gallery.json": failure_gallery, + "generation-status.json": split_result["generation_status"], + "human-review-summary.json": human_review_summary, + "input-manifest.json": input_manifest, + "latency-and-reliability.json": latency_reliability, + "leakage-gate-report.json": leakage, + "metric-report.json": metric_report, + "object-metrics.json": object_metrics, + "protected-split-manifest.json": protected, + "reference-implementation-baseline.json": golden, + "release-gate-report.json": gate_report, + "split-and-leakage-audit.json": leakage, + "stratified-metrics.json": stratified_metrics, + "tile-metrics.json": tile_metrics, + "workflow-summary.json": workflow_summary, + } + evidence = build_evidence_manifest(artifacts, benchmark_manifest) + for name, payload in sorted(artifacts.items()): + write_json_immutable(output_dir / name, payload) + write_json_immutable(output_dir / "evidence-manifest.json", evidence) + return workflow_summary + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-root", type=Path, default=ROOT) + parser.add_argument( + "--output-dir", + type=Path, + default=ROOT / "artifacts/evidence/accuracy/P4/reference-harness-v2", + ) + parser.add_argument( + "--product-baseline-manifest", + type=Path, + help=( + "Optional governed product incumbent manifest. It can pass only when real " + "active-model inference and all referenced artifacts validate." + ), + ) + parser.add_argument( + "--allow-product-blocked", + action="store_true", + help=( + "Return zero when the local harness passes while product evidence remains " + "fail/not_evaluable. This never changes a gate or phase decision." + ), + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + summary = run_workflow( + args.repo_root.resolve(), + args.output_dir.resolve(), + args.product_baseline_manifest.resolve() + if args.product_baseline_manifest + else None, + ) + except Exception as exc: # noqa: BLE001 - workflow evidence must fail closed + print( + json.dumps( + {"status": "fail", "error": f"{type(exc).__name__}: {exc}"}, + indent=2, + ) + ) + return 2 + print(json.dumps(summary, indent=2, sort_keys=True)) + if summary["status"] == "pass": + return 0 + return ( + 0 + if args.allow_product_blocked and summary["local_harness_status"] == "pass" + else 2 + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_build_regional_yolo_dataset.py b/tests/test_build_regional_yolo_dataset.py index 82d03ca5..b1b61518 100644 --- a/tests/test_build_regional_yolo_dataset.py +++ b/tests/test_build_regional_yolo_dataset.py @@ -35,11 +35,12 @@ def test_regional_dataset_balances_contexts_and_keeps_validation_closed() -> Non assert evidence["protected_samples_in_training"] == [] -def test_regional_dataset_rejects_protected_tiles() -> None: +@pytest.mark.parametrize("protected_split", ["calibration", "test", "background-test", "challenge"]) +def test_regional_dataset_rejects_protected_tiles(protected_split: str) -> None: manifest = {"samples": [ - {"sample_slug": "f-cal", "region": "flanders", "split": "calibration", "context": "ribbon"}, + {"sample_slug": "f-cal", "region": "flanders", "split": protected_split, "context": "ribbon"}, ]} - summary = {"tiles": [{"sample_slug": "f-cal", "split": "calibration", "image_path": "/cal.png"}]} + summary = {"tiles": [{"sample_slug": "f-cal", "split": protected_split, "image_path": "/cal.png"}]} with pytest.raises(ValueError, match="protected"): module.build( summary=summary, manifest=manifest, region="flanders", diff --git a/tests/test_building_proposal_classifier.py b/tests/test_building_proposal_classifier.py index d58c8987..562267d5 100644 --- a/tests/test_building_proposal_classifier.py +++ b/tests/test_building_proposal_classifier.py @@ -114,8 +114,9 @@ def test_classify_proposals_consumes_reference_once() -> None: assert [item[0] for item in miner.classify_proposals(proposals, reference, 0.25)] == ["positive", "negative"] -def test_eligible_tiles_rejects_protected_manifest_split() -> None: - manifest = {"samples": [{"sample_slug": "x", "region": "flanders", "split": "test"}]} +@pytest.mark.parametrize("protected_split", ["calibration", "test", "background-test", "challenge"]) +def test_eligible_tiles_rejects_protected_manifest_split(protected_split: str) -> None: + manifest = {"samples": [{"sample_slug": "x", "region": "flanders", "split": protected_split}]} summary = {"tiles": [{"sample_slug": "x", "split": "train", "image_path": "x.png"}]} with pytest.raises(ValueError, match="protected"): miner.eligible_tiles(summary, manifest, "flanders")