docs(accuracy): publish phase 4 benchmark evidence
This commit is contained in:
Jens
+300
@@ -0,0 +1,300 @@
|
||||
{
|
||||
"benchmark_id": "geointel-p4-reference-harness-v2",
|
||||
"claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.",
|
||||
"evaluator_families": [
|
||||
"change_detection",
|
||||
"footprint_segmentation",
|
||||
"geospatial_data_validation",
|
||||
"object_detection",
|
||||
"raster_classification",
|
||||
"terrain_interpretation",
|
||||
"vector_comparison"
|
||||
],
|
||||
"evaluator_version": "2.1.0",
|
||||
"gate_states": [
|
||||
"fail",
|
||||
"not_evaluable",
|
||||
"pass"
|
||||
],
|
||||
"implemented_capabilities": [
|
||||
{
|
||||
"capability_id": "model_object_detection",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "model_inference_pipeline",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/detection_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"precision",
|
||||
"recall",
|
||||
"F1",
|
||||
"AP50",
|
||||
"AP50-95",
|
||||
"IoU",
|
||||
"ECE",
|
||||
"Brier",
|
||||
"coverage-risk"
|
||||
],
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"capability_id": "building_proposal_filtering",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "not_separately_benchmarked",
|
||||
"implementation_kind": "supporting_candidate_classifier",
|
||||
"implementation_paths": [
|
||||
"scripts/train_building_proposal_classifier.py",
|
||||
"scripts/evaluate_belgium_building_candidate.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"candidate precision",
|
||||
"candidate recall",
|
||||
"F1",
|
||||
"calibration"
|
||||
],
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"capability_id": "footprint_segmentation",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "model_inference_pipeline",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/segmentation_service.py",
|
||||
"backend/app/services/segmentation_adapter.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"object precision",
|
||||
"object recall",
|
||||
"F1",
|
||||
"IoU",
|
||||
"Dice",
|
||||
"boundary F1",
|
||||
"centroid distance",
|
||||
"area error",
|
||||
"topology"
|
||||
],
|
||||
"task": "footprint_segmentation"
|
||||
},
|
||||
{
|
||||
"capability_id": "detection_and_vector_qa",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "deterministic_geospatial_comparison",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/qa_service.py",
|
||||
"backend/app/services/detection_qa_service.py",
|
||||
"backend/app/services/quality_check_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"precision",
|
||||
"recall",
|
||||
"F1",
|
||||
"IoU",
|
||||
"topology",
|
||||
"coverage"
|
||||
],
|
||||
"task": "vector_comparison"
|
||||
},
|
||||
{
|
||||
"capability_id": "vector_clip_buffer_intersect",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_vector_processing",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/vector_operations_service.py",
|
||||
"backend/app/services/vector_feature_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"geometry validity",
|
||||
"CRS correctness",
|
||||
"area conservation",
|
||||
"feature counts",
|
||||
"topology"
|
||||
],
|
||||
"task": "vector_comparison"
|
||||
},
|
||||
{
|
||||
"capability_id": "temporal_vector_change",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "deterministic_change_detection",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/change_detection_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"event precision",
|
||||
"event recall",
|
||||
"event F1",
|
||||
"IoU"
|
||||
],
|
||||
"task": "change_detection"
|
||||
},
|
||||
{
|
||||
"capability_id": "thematic_raster_interpretation",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_metric_contract_only_no_generic_learned_classifier_claim",
|
||||
"implementation_kind": "deterministic_source_interpretation",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/thematic_raster_analysis_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"pixel accuracy",
|
||||
"per-class F1",
|
||||
"per-class IoU",
|
||||
"mean IoU"
|
||||
],
|
||||
"task": "raster_classification"
|
||||
},
|
||||
{
|
||||
"capability_id": "raster_clip_reproject_indices",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_raster_processing",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/raster_service.py",
|
||||
"backend/app/services/raster_operations_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"CRS/transform preservation",
|
||||
"pixel alignment",
|
||||
"nodata",
|
||||
"numeric tolerance"
|
||||
],
|
||||
"task": "raster_classification"
|
||||
},
|
||||
{
|
||||
"capability_id": "raster_partition_mosaic",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_raster_partitioning",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/raster_partition_analysis_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"seam equality",
|
||||
"coverage completeness",
|
||||
"resolution consistency"
|
||||
],
|
||||
"task": "raster_classification"
|
||||
},
|
||||
{
|
||||
"capability_id": "terrain_height_interpretation",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "deterministic_continuous_raster_analysis",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/terrain_analysis_service.py",
|
||||
"backend/app/services/spw_terrain_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"MAE",
|
||||
"RMSE",
|
||||
"bias",
|
||||
"coverage",
|
||||
"unit integrity"
|
||||
],
|
||||
"task": "terrain_interpretation"
|
||||
},
|
||||
{
|
||||
"capability_id": "flood_hazard_interpretation",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_scenario_raster_analysis",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/flood_hazard_analysis_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"depth MAE/RMSE",
|
||||
"hazard-class IoU",
|
||||
"coverage",
|
||||
"scenario identity"
|
||||
],
|
||||
"task": "terrain_interpretation"
|
||||
},
|
||||
{
|
||||
"capability_id": "bathymetry_interpretation",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_vertical_reference_analysis",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/bathymetry_raster_analysis_service.py",
|
||||
"backend/app/services/mdk_bathymetry_probe_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"MAE",
|
||||
"RMSE",
|
||||
"bias",
|
||||
"coverage",
|
||||
"vertical-datum integrity"
|
||||
],
|
||||
"task": "terrain_interpretation"
|
||||
},
|
||||
{
|
||||
"capability_id": "data_contract_validation_and_scan",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "deterministic_validation",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/data_contract_validation.py",
|
||||
"scripts/run_accuracy_phase3_full_data_scan.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"anomaly precision",
|
||||
"anomaly recall",
|
||||
"anomaly F1",
|
||||
"critical misses"
|
||||
],
|
||||
"task": "geospatial_data_validation"
|
||||
},
|
||||
{
|
||||
"capability_id": "aoi_partition_orchestration",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_orchestration",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/aoi_operation_service.py",
|
||||
"backend/app/services/aoi_operation_executor.py",
|
||||
"backend/app/services/aoi_operation_worker.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"partition completeness",
|
||||
"overlap/gap",
|
||||
"idempotency",
|
||||
"resume correctness"
|
||||
],
|
||||
"task": "geospatial_data_validation"
|
||||
},
|
||||
{
|
||||
"capability_id": "geo_assistant_orchestration",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "no_independent_accuracy_score_underlying_tool_results_are_authoritative",
|
||||
"implementation_kind": "tool_orchestration_interface",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/geo_assistant_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"tool-selection correctness",
|
||||
"grounding",
|
||||
"unsupported-claim rate"
|
||||
],
|
||||
"task": "geospatial_data_validation"
|
||||
}
|
||||
],
|
||||
"metric_contract": {
|
||||
"path": "docs/accuracy-program/05-metric-framework.md",
|
||||
"sha256": "18abfc21964b7d98848d8ae513afce242184980c9517c9184d40efd9165ea00d",
|
||||
"size_bytes": 28053
|
||||
},
|
||||
"protected_policy": {
|
||||
"diagnostic_curves_select_operating_point": false,
|
||||
"operating_point_selection_allowed": false,
|
||||
"test_feedback_allowed": false,
|
||||
"threshold_selection_source": "pre_registered_configuration_only"
|
||||
},
|
||||
"schema_version": 2,
|
||||
"split_generator_version": "1.3.0",
|
||||
"undefined_value_policy": "Undefined denominators are null with numerator, denominator and support; they are never coerced to a perfect score.",
|
||||
"workflow_version": "2.0.1"
|
||||
}
|
||||
Reference in New Issue
Block a user