{
  "benchmark_id": "geointel-p4-reference-harness-v2",
  "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.",
  "evaluator_families": [
    "change_detection",
    "footprint_segmentation",
    "geospatial_data_validation",
    "object_detection",
    "raster_classification",
    "terrain_interpretation",
    "vector_comparison"
  ],
  "evaluator_version": "2.1.0",
  "gate_states": [
    "fail",
    "not_evaluable",
    "pass"
  ],
  "implemented_capabilities": [
    {
      "capability_id": "model_object_detection",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "synthetic_contract_case_only",
      "implementation_kind": "model_inference_pipeline",
      "implementation_paths": [
        "backend/app/services/detection_service.py"
      ],
      "suitable_metrics": [
        "precision",
        "recall",
        "F1",
        "AP50",
        "AP50-95",
        "IoU",
        "ECE",
        "Brier",
        "coverage-risk"
      ],
      "task": "object_detection"
    },
    {
      "capability_id": "building_proposal_filtering",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "not_separately_benchmarked",
      "implementation_kind": "supporting_candidate_classifier",
      "implementation_paths": [
        "scripts/train_building_proposal_classifier.py",
        "scripts/evaluate_belgium_building_candidate.py"
      ],
      "suitable_metrics": [
        "candidate precision",
        "candidate recall",
        "F1",
        "calibration"
      ],
      "task": "object_detection"
    },
    {
      "capability_id": "footprint_segmentation",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "synthetic_contract_case_only",
      "implementation_kind": "model_inference_pipeline",
      "implementation_paths": [
        "backend/app/services/segmentation_service.py",
        "backend/app/services/segmentation_adapter.py"
      ],
      "suitable_metrics": [
        "object precision",
        "object recall",
        "F1",
        "IoU",
        "Dice",
        "boundary F1",
        "centroid distance",
        "area error",
        "topology"
      ],
      "task": "footprint_segmentation"
    },
    {
      "capability_id": "detection_and_vector_qa",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "synthetic_contract_case_only",
      "implementation_kind": "deterministic_geospatial_comparison",
      "implementation_paths": [
        "backend/app/services/qa_service.py",
        "backend/app/services/detection_qa_service.py",
        "backend/app/services/quality_check_service.py"
      ],
      "suitable_metrics": [
        "precision",
        "recall",
        "F1",
        "IoU",
        "topology",
        "coverage"
      ],
      "task": "vector_comparison"
    },
    {
      "capability_id": "vector_clip_buffer_intersect",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "covered_by_family_not_separately_benchmarked",
      "implementation_kind": "deterministic_vector_processing",
      "implementation_paths": [
        "backend/app/services/vector_operations_service.py",
        "backend/app/services/vector_feature_service.py"
      ],
      "suitable_metrics": [
        "geometry validity",
        "CRS correctness",
        "area conservation",
        "feature counts",
        "topology"
      ],
      "task": "vector_comparison"
    },
    {
      "capability_id": "temporal_vector_change",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "synthetic_contract_case_only",
      "implementation_kind": "deterministic_change_detection",
      "implementation_paths": [
        "backend/app/services/change_detection_service.py"
      ],
      "suitable_metrics": [
        "event precision",
        "event recall",
        "event F1",
        "IoU"
      ],
      "task": "change_detection"
    },
    {
      "capability_id": "thematic_raster_interpretation",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "synthetic_metric_contract_only_no_generic_learned_classifier_claim",
      "implementation_kind": "deterministic_source_interpretation",
      "implementation_paths": [
        "backend/app/services/thematic_raster_analysis_service.py"
      ],
      "suitable_metrics": [
        "pixel accuracy",
        "per-class F1",
        "per-class IoU",
        "mean IoU"
      ],
      "task": "raster_classification"
    },
    {
      "capability_id": "raster_clip_reproject_indices",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "covered_by_family_not_separately_benchmarked",
      "implementation_kind": "deterministic_raster_processing",
      "implementation_paths": [
        "backend/app/services/raster_service.py",
        "backend/app/services/raster_operations_service.py"
      ],
      "suitable_metrics": [
        "CRS/transform preservation",
        "pixel alignment",
        "nodata",
        "numeric tolerance"
      ],
      "task": "raster_classification"
    },
    {
      "capability_id": "raster_partition_mosaic",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "not_separately_benchmarked",
      "implementation_kind": "deterministic_raster_partitioning",
      "implementation_paths": [
        "backend/app/services/raster_partition_analysis_service.py"
      ],
      "suitable_metrics": [
        "seam equality",
        "coverage completeness",
        "resolution consistency"
      ],
      "task": "raster_classification"
    },
    {
      "capability_id": "terrain_height_interpretation",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "synthetic_contract_case_only",
      "implementation_kind": "deterministic_continuous_raster_analysis",
      "implementation_paths": [
        "backend/app/services/terrain_analysis_service.py",
        "backend/app/services/spw_terrain_service.py"
      ],
      "suitable_metrics": [
        "MAE",
        "RMSE",
        "bias",
        "coverage",
        "unit integrity"
      ],
      "task": "terrain_interpretation"
    },
    {
      "capability_id": "flood_hazard_interpretation",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "covered_by_family_not_separately_benchmarked",
      "implementation_kind": "deterministic_scenario_raster_analysis",
      "implementation_paths": [
        "backend/app/services/flood_hazard_analysis_service.py"
      ],
      "suitable_metrics": [
        "depth MAE/RMSE",
        "hazard-class IoU",
        "coverage",
        "scenario identity"
      ],
      "task": "terrain_interpretation"
    },
    {
      "capability_id": "bathymetry_interpretation",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "covered_by_family_not_separately_benchmarked",
      "implementation_kind": "deterministic_vertical_reference_analysis",
      "implementation_paths": [
        "backend/app/services/bathymetry_raster_analysis_service.py",
        "backend/app/services/mdk_bathymetry_probe_service.py"
      ],
      "suitable_metrics": [
        "MAE",
        "RMSE",
        "bias",
        "coverage",
        "vertical-datum integrity"
      ],
      "task": "terrain_interpretation"
    },
    {
      "capability_id": "data_contract_validation_and_scan",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "synthetic_contract_case_only",
      "implementation_kind": "deterministic_validation",
      "implementation_paths": [
        "backend/app/services/data_contract_validation.py",
        "scripts/run_accuracy_phase3_full_data_scan.py"
      ],
      "suitable_metrics": [
        "anomaly precision",
        "anomaly recall",
        "anomaly F1",
        "critical misses"
      ],
      "task": "geospatial_data_validation"
    },
    {
      "capability_id": "aoi_partition_orchestration",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "not_separately_benchmarked",
      "implementation_kind": "deterministic_orchestration",
      "implementation_paths": [
        "backend/app/services/aoi_operation_service.py",
        "backend/app/services/aoi_operation_executor.py",
        "backend/app/services/aoi_operation_worker.py"
      ],
      "suitable_metrics": [
        "partition completeness",
        "overlap/gap",
        "idempotency",
        "resume correctness"
      ],
      "task": "geospatial_data_validation"
    },
    {
      "capability_id": "geo_assistant_orchestration",
      "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
      "evaluation_status": "no_independent_accuracy_score_underlying_tool_results_are_authoritative",
      "implementation_kind": "tool_orchestration_interface",
      "implementation_paths": [
        "backend/app/services/geo_assistant_service.py"
      ],
      "suitable_metrics": [
        "tool-selection correctness",
        "grounding",
        "unsupported-claim rate"
      ],
      "task": "geospatial_data_validation"
    }
  ],
  "metric_contract": {
    "path": "docs/accuracy-program/05-metric-framework.md",
    "sha256": "18abfc21964b7d98848d8ae513afce242184980c9517c9184d40efd9165ea00d",
    "size_bytes": 28053
  },
  "protected_policy": {
    "diagnostic_curves_select_operating_point": false,
    "operating_point_selection_allowed": false,
    "test_feedback_allowed": false,
    "threshold_selection_source": "pre_registered_configuration_only"
  },
  "schema_version": 2,
  "split_generator_version": "1.3.0",
  "undefined_value_policy": "Undefined denominators are null with numerator, denominator and support; they are never coerced to a perfect score.",
  "workflow_version": "2.0.1"
}