301 lines
11 KiB
JSON
301 lines
11 KiB
JSON
{
|
|
"benchmark_id": "geointel-p4-reference-harness-v2",
|
|
"claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.",
|
|
"evaluator_families": [
|
|
"change_detection",
|
|
"footprint_segmentation",
|
|
"geospatial_data_validation",
|
|
"object_detection",
|
|
"raster_classification",
|
|
"terrain_interpretation",
|
|
"vector_comparison"
|
|
],
|
|
"evaluator_version": "2.1.0",
|
|
"gate_states": [
|
|
"fail",
|
|
"not_evaluable",
|
|
"pass"
|
|
],
|
|
"implemented_capabilities": [
|
|
{
|
|
"capability_id": "model_object_detection",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "synthetic_contract_case_only",
|
|
"implementation_kind": "model_inference_pipeline",
|
|
"implementation_paths": [
|
|
"backend/app/services/detection_service.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"precision",
|
|
"recall",
|
|
"F1",
|
|
"AP50",
|
|
"AP50-95",
|
|
"IoU",
|
|
"ECE",
|
|
"Brier",
|
|
"coverage-risk"
|
|
],
|
|
"task": "object_detection"
|
|
},
|
|
{
|
|
"capability_id": "building_proposal_filtering",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "not_separately_benchmarked",
|
|
"implementation_kind": "supporting_candidate_classifier",
|
|
"implementation_paths": [
|
|
"scripts/train_building_proposal_classifier.py",
|
|
"scripts/evaluate_belgium_building_candidate.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"candidate precision",
|
|
"candidate recall",
|
|
"F1",
|
|
"calibration"
|
|
],
|
|
"task": "object_detection"
|
|
},
|
|
{
|
|
"capability_id": "footprint_segmentation",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "synthetic_contract_case_only",
|
|
"implementation_kind": "model_inference_pipeline",
|
|
"implementation_paths": [
|
|
"backend/app/services/segmentation_service.py",
|
|
"backend/app/services/segmentation_adapter.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"object precision",
|
|
"object recall",
|
|
"F1",
|
|
"IoU",
|
|
"Dice",
|
|
"boundary F1",
|
|
"centroid distance",
|
|
"area error",
|
|
"topology"
|
|
],
|
|
"task": "footprint_segmentation"
|
|
},
|
|
{
|
|
"capability_id": "detection_and_vector_qa",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "synthetic_contract_case_only",
|
|
"implementation_kind": "deterministic_geospatial_comparison",
|
|
"implementation_paths": [
|
|
"backend/app/services/qa_service.py",
|
|
"backend/app/services/detection_qa_service.py",
|
|
"backend/app/services/quality_check_service.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"precision",
|
|
"recall",
|
|
"F1",
|
|
"IoU",
|
|
"topology",
|
|
"coverage"
|
|
],
|
|
"task": "vector_comparison"
|
|
},
|
|
{
|
|
"capability_id": "vector_clip_buffer_intersect",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
|
"implementation_kind": "deterministic_vector_processing",
|
|
"implementation_paths": [
|
|
"backend/app/services/vector_operations_service.py",
|
|
"backend/app/services/vector_feature_service.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"geometry validity",
|
|
"CRS correctness",
|
|
"area conservation",
|
|
"feature counts",
|
|
"topology"
|
|
],
|
|
"task": "vector_comparison"
|
|
},
|
|
{
|
|
"capability_id": "temporal_vector_change",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "synthetic_contract_case_only",
|
|
"implementation_kind": "deterministic_change_detection",
|
|
"implementation_paths": [
|
|
"backend/app/services/change_detection_service.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"event precision",
|
|
"event recall",
|
|
"event F1",
|
|
"IoU"
|
|
],
|
|
"task": "change_detection"
|
|
},
|
|
{
|
|
"capability_id": "thematic_raster_interpretation",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "synthetic_metric_contract_only_no_generic_learned_classifier_claim",
|
|
"implementation_kind": "deterministic_source_interpretation",
|
|
"implementation_paths": [
|
|
"backend/app/services/thematic_raster_analysis_service.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"pixel accuracy",
|
|
"per-class F1",
|
|
"per-class IoU",
|
|
"mean IoU"
|
|
],
|
|
"task": "raster_classification"
|
|
},
|
|
{
|
|
"capability_id": "raster_clip_reproject_indices",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
|
"implementation_kind": "deterministic_raster_processing",
|
|
"implementation_paths": [
|
|
"backend/app/services/raster_service.py",
|
|
"backend/app/services/raster_operations_service.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"CRS/transform preservation",
|
|
"pixel alignment",
|
|
"nodata",
|
|
"numeric tolerance"
|
|
],
|
|
"task": "raster_classification"
|
|
},
|
|
{
|
|
"capability_id": "raster_partition_mosaic",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "not_separately_benchmarked",
|
|
"implementation_kind": "deterministic_raster_partitioning",
|
|
"implementation_paths": [
|
|
"backend/app/services/raster_partition_analysis_service.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"seam equality",
|
|
"coverage completeness",
|
|
"resolution consistency"
|
|
],
|
|
"task": "raster_classification"
|
|
},
|
|
{
|
|
"capability_id": "terrain_height_interpretation",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "synthetic_contract_case_only",
|
|
"implementation_kind": "deterministic_continuous_raster_analysis",
|
|
"implementation_paths": [
|
|
"backend/app/services/terrain_analysis_service.py",
|
|
"backend/app/services/spw_terrain_service.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"MAE",
|
|
"RMSE",
|
|
"bias",
|
|
"coverage",
|
|
"unit integrity"
|
|
],
|
|
"task": "terrain_interpretation"
|
|
},
|
|
{
|
|
"capability_id": "flood_hazard_interpretation",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
|
"implementation_kind": "deterministic_scenario_raster_analysis",
|
|
"implementation_paths": [
|
|
"backend/app/services/flood_hazard_analysis_service.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"depth MAE/RMSE",
|
|
"hazard-class IoU",
|
|
"coverage",
|
|
"scenario identity"
|
|
],
|
|
"task": "terrain_interpretation"
|
|
},
|
|
{
|
|
"capability_id": "bathymetry_interpretation",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
|
"implementation_kind": "deterministic_vertical_reference_analysis",
|
|
"implementation_paths": [
|
|
"backend/app/services/bathymetry_raster_analysis_service.py",
|
|
"backend/app/services/mdk_bathymetry_probe_service.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"MAE",
|
|
"RMSE",
|
|
"bias",
|
|
"coverage",
|
|
"vertical-datum integrity"
|
|
],
|
|
"task": "terrain_interpretation"
|
|
},
|
|
{
|
|
"capability_id": "data_contract_validation_and_scan",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "synthetic_contract_case_only",
|
|
"implementation_kind": "deterministic_validation",
|
|
"implementation_paths": [
|
|
"backend/app/services/data_contract_validation.py",
|
|
"scripts/run_accuracy_phase3_full_data_scan.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"anomaly precision",
|
|
"anomaly recall",
|
|
"anomaly F1",
|
|
"critical misses"
|
|
],
|
|
"task": "geospatial_data_validation"
|
|
},
|
|
{
|
|
"capability_id": "aoi_partition_orchestration",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "not_separately_benchmarked",
|
|
"implementation_kind": "deterministic_orchestration",
|
|
"implementation_paths": [
|
|
"backend/app/services/aoi_operation_service.py",
|
|
"backend/app/services/aoi_operation_executor.py",
|
|
"backend/app/services/aoi_operation_worker.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"partition completeness",
|
|
"overlap/gap",
|
|
"idempotency",
|
|
"resume correctness"
|
|
],
|
|
"task": "geospatial_data_validation"
|
|
},
|
|
{
|
|
"capability_id": "geo_assistant_orchestration",
|
|
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
|
"evaluation_status": "no_independent_accuracy_score_underlying_tool_results_are_authoritative",
|
|
"implementation_kind": "tool_orchestration_interface",
|
|
"implementation_paths": [
|
|
"backend/app/services/geo_assistant_service.py"
|
|
],
|
|
"suitable_metrics": [
|
|
"tool-selection correctness",
|
|
"grounding",
|
|
"unsupported-claim rate"
|
|
],
|
|
"task": "geospatial_data_validation"
|
|
}
|
|
],
|
|
"metric_contract": {
|
|
"path": "docs/accuracy-program/05-metric-framework.md",
|
|
"sha256": "18abfc21964b7d98848d8ae513afce242184980c9517c9184d40efd9165ea00d",
|
|
"size_bytes": 28053
|
|
},
|
|
"protected_policy": {
|
|
"diagnostic_curves_select_operating_point": false,
|
|
"operating_point_selection_allowed": false,
|
|
"test_feedback_allowed": false,
|
|
"threshold_selection_source": "pre_registered_configuration_only"
|
|
},
|
|
"schema_version": 2,
|
|
"split_generator_version": "1.3.0",
|
|
"undefined_value_policy": "Undefined denominators are null with numerator, denominator and support; they are never coerced to a perfect score.",
|
|
"workflow_version": "2.0.1"
|
|
}
|