Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
0e3c1b20e931941c346da08f6fa8847e674bdadb
geointel/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/stratified-metrics.json
T
Jens f41392a415
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
Details
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
Details
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s
Details
docs(accuracy): publish phase 4 benchmark evidence
2026-08-02 05:10:05 +02:00

8799 lines
319 KiB
JSON
Raw Blame History

{
"dimensions": {
"context": {
"strata": {
"coastal_hard_negative": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": null,
"f1_case_support": 0,
"method": "unweighted mean across cases with defined values",
"precision": 0.0,
"precision_case_support": 1,
"recall": null,
"recall_case_support": 0,
"status": "computed"
},
"micro": {
"ap50": null,
"ap50_95": null,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": null,
"count": 0,
"lower": 0.6,
"mean_confidence": null,
"upper": 0.8
},
{
"accuracy": 0,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.92,
"upper": 1.0
}
],
"brier": 0.8464,
"ece": 0.92,
"status": "computed"
},
"f1": null,
"false_discovery_rate": 1.0,
"false_negative": 0,
"false_positive": 1,
"map50": null,
"map50_95": null,
"miss_rate": null,
"per_class_ap": {
"building": {
"ap50": null,
"ap50_95": null,
"prediction_count": 1,
"reference_count": 0
}
},
"precision": 0.0,
"precision_ci95_wilson": {
"lower": 0.0,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 0.7934506856227626
},
"prediction_count": 1,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": null,
"recall_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"reference_count": 0,
"true_positive": 0
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 1,
"predictions": 1,
"ranking_predictions": 1,
"references": 0
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": null
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"data_contract": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation"
]
},
"dense_urban": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 1.0,
"ap50_95": 0.900990099009901,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 1,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.9,
"upper": 1.0
}
],
"brier": 0.15333333333333332,
"ece": 0.13333333333333325,
"status": "computed"
},
"f1": 0.8,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 0,
"false_positive": 1,
"map50": 1.0,
"map50_95": 0.900990099009901,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 1.0,
"ap50_95": 0.900990099009901,
"prediction_count": 4,
"reference_count": 2
}
},
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 3,
"predictions": 3,
"ranking_predictions": 4,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.8
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"dense_urban_raster": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"raster_classification"
]
},
"industrial_change": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection"
]
},
"pure_background": {
"case_support": 1,
"failure_count": 0,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": null,
"f1_case_support": 0,
"method": "unweighted mean across cases with defined values",
"precision": null,
"precision_case_support": 0,
"recall": null,
"recall_case_support": 0,
"status": "not_evaluable"
},
"micro": {
"ap50": null,
"ap50_95": null,
"calibration": {
"bins": [],
"brier": null,
"ece": null,
"status": "undefined"
},
"f1": null,
"false_discovery_rate": null,
"false_negative": 0,
"false_positive": 0,
"map50": null,
"map50_95": null,
"miss_rate": null,
"per_class_ap": {
"building": {
"ap50": null,
"ap50_95": null,
"prediction_count": 0,
"reference_count": 0
}
},
"precision": null,
"precision_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"prediction_count": 0,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": null,
"recall_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"reference_count": 0,
"true_positive": 0
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 0,
"predictions": 0,
"ranking_predictions": 0,
"references": 0
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": null
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"rural_buildings": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"footprint_segmentation"
]
},
"suburban_buildings": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"vector_comparison"
]
},
"terrain_profile": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"terrain_interpretation"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"date": {
"strata": {
"2023-01-01": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"terrain_interpretation"
]
},
"2024-05-01/2025-05-01": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection"
]
},
"2025-01-01": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation"
]
},
"2025-01-15": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"vector_comparison"
]
},
"2025-02-01": {
"case_support": 1,
"failure_count": 0,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": null,
"f1_case_support": 0,
"method": "unweighted mean across cases with defined values",
"precision": null,
"precision_case_support": 0,
"recall": null,
"recall_case_support": 0,
"status": "not_evaluable"
},
"micro": {
"ap50": null,
"ap50_95": null,
"calibration": {
"bins": [],
"brier": null,
"ece": null,
"status": "undefined"
},
"f1": null,
"false_discovery_rate": null,
"false_negative": 0,
"false_positive": 0,
"map50": null,
"map50_95": null,
"miss_rate": null,
"per_class_ap": {
"building": {
"ap50": null,
"ap50_95": null,
"prediction_count": 0,
"reference_count": 0
}
},
"precision": null,
"precision_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"prediction_count": 0,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": null,
"recall_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"reference_count": 0,
"true_positive": 0
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 0,
"predictions": 0,
"ranking_predictions": 0,
"references": 0
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": null
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"2025-04-20": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"footprint_segmentation"
]
},
"2025-06-15": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 1.0,
"ap50_95": 0.900990099009901,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 1,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.9,
"upper": 1.0
}
],
"brier": 0.15333333333333332,
"ece": 0.13333333333333325,
"status": "computed"
},
"f1": 0.8,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 0,
"false_positive": 1,
"map50": 1.0,
"map50_95": 0.900990099009901,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 1.0,
"ap50_95": 0.900990099009901,
"prediction_count": 4,
"reference_count": 2
}
},
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 3,
"predictions": 3,
"ranking_predictions": 4,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.8
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"2025-07-01": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": null,
"f1_case_support": 0,
"method": "unweighted mean across cases with defined values",
"precision": 0.0,
"precision_case_support": 1,
"recall": null,
"recall_case_support": 0,
"status": "computed"
},
"micro": {
"ap50": null,
"ap50_95": null,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": null,
"count": 0,
"lower": 0.6,
"mean_confidence": null,
"upper": 0.8
},
{
"accuracy": 0,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.92,
"upper": 1.0
}
],
"brier": 0.8464,
"ece": 0.92,
"status": "computed"
},
"f1": null,
"false_discovery_rate": 1.0,
"false_negative": 0,
"false_positive": 1,
"map50": null,
"map50_95": null,
"miss_rate": null,
"per_class_ap": {
"building": {
"ap50": null,
"ap50_95": null,
"prediction_count": 1,
"reference_count": 0
}
},
"precision": 0.0,
"precision_ci95_wilson": {
"lower": 0.0,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 0.7934506856227626
},
"prediction_count": 1,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": null,
"recall_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"reference_count": 0,
"true_positive": 0
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 1,
"predictions": 1,
"ranking_predictions": 1,
"references": 0
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": null
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"2025-10-05": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"raster_classification"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"difficulty": {
"strata": {
"hard": {
"case_support": 6,
"failure_count": 10,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"object_detection": {
"case_support": 3,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.3333333333333333,
"precision_case_support": 2,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.8,
"mean_confidence": 0.91,
"upper": 1.0
}
],
"brier": 0.3266,
"ece": 0.27999999999999997,
"status": "computed"
},
"f1": 0.6666666666666666,
"false_discovery_rate": 0.5,
"false_negative": 0,
"false_positive": 2,
"map50": 0.6666666666666666,
"map50_95": 0.5838283828382838,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"prediction_count": 5,
"reference_count": 2
}
},
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.15003898915214947,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 0.8499610108478506
},
"prediction_count": 4,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 4,
"predictions": 4,
"ranking_predictions": 5,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection",
"geospatial_data_validation",
"object_detection",
"raster_classification"
]
},
"normal": {
"case_support": 3,
"failure_count": 4,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"footprint_segmentation",
"terrain_interpretation",
"vector_comparison"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"municipality": {
"strata": {
"Antwerpen": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 1.0,
"ap50_95": 0.900990099009901,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 1,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.9,
"upper": 1.0
}
],
"brier": 0.15333333333333332,
"ece": 0.13333333333333325,
"status": "computed"
},
"f1": 0.8,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 0,
"false_positive": 1,
"map50": 1.0,
"map50_95": 0.900990099009901,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 1.0,
"ap50_95": 0.900990099009901,
"prediction_count": 4,
"reference_count": 2
}
},
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 3,
"predictions": 3,
"ranking_predictions": 4,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.8
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"Brussel": {
"case_support": 2,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": null,
"f1_case_support": 0,
"method": "unweighted mean across cases with defined values",
"precision": null,
"precision_case_support": 0,
"recall": null,
"recall_case_support": 0,
"status": "not_evaluable"
},
"micro": {
"ap50": null,
"ap50_95": null,
"calibration": {
"bins": [],
"brier": null,
"ece": null,
"status": "undefined"
},
"f1": null,
"false_discovery_rate": null,
"false_negative": 0,
"false_positive": 0,
"map50": null,
"map50_95": null,
"miss_rate": null,
"per_class_ap": {
"building": {
"ap50": null,
"ap50_95": null,
"prediction_count": 0,
"reference_count": 0
}
},
"precision": null,
"precision_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"prediction_count": 0,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": null,
"recall_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"reference_count": 0,
"true_positive": 0
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 0,
"predictions": 0,
"ranking_predictions": 0,
"references": 0
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": null
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection",
"raster_classification"
]
},
"Hasselt": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"terrain_interpretation"
]
},
"Liège": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection"
]
},
"Mol": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"vector_comparison"
]
},
"Namur": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"footprint_segmentation"
]
},
"Oostende": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": null,
"f1_case_support": 0,
"method": "unweighted mean across cases with defined values",
"precision": 0.0,
"precision_case_support": 1,
"recall": null,
"recall_case_support": 0,
"status": "computed"
},
"micro": {
"ap50": null,
"ap50_95": null,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": null,
"count": 0,
"lower": 0.6,
"mean_confidence": null,
"upper": 0.8
},
{
"accuracy": 0,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.92,
"upper": 1.0
}
],
"brier": 0.8464,
"ece": 0.92,
"status": "computed"
},
"f1": null,
"false_discovery_rate": 1.0,
"false_negative": 0,
"false_positive": 1,
"map50": null,
"map50_95": null,
"miss_rate": null,
"per_class_ap": {
"building": {
"ap50": null,
"ap50_95": null,
"prediction_count": 1,
"reference_count": 0
}
},
"precision": 0.0,
"precision_ci95_wilson": {
"lower": 0.0,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 0.7934506856227626
},
"prediction_count": 1,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": null,
"recall_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"reference_count": 0,
"true_positive": 0
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 1,
"predictions": 1,
"ranking_predictions": 1,
"references": 0
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": null
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"not_applicable": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"object_size": {
"strata": {
"empty": {
"case_support": 2,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 2,
"macro": {
"f1": null,
"f1_case_support": 0,
"method": "unweighted mean across cases with defined values",
"precision": 0.0,
"precision_case_support": 1,
"recall": null,
"recall_case_support": 0,
"status": "computed"
},
"micro": {
"ap50": null,
"ap50_95": null,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": null,
"count": 0,
"lower": 0.6,
"mean_confidence": null,
"upper": 0.8
},
{
"accuracy": 0,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.92,
"upper": 1.0
}
],
"brier": 0.8464,
"ece": 0.92,
"status": "computed"
},
"f1": null,
"false_discovery_rate": 1.0,
"false_negative": 0,
"false_positive": 1,
"map50": null,
"map50_95": null,
"miss_rate": null,
"per_class_ap": {
"building": {
"ap50": null,
"ap50_95": null,
"prediction_count": 1,
"reference_count": 0
}
},
"precision": 0.0,
"precision_ci95_wilson": {
"lower": 0.0,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 0.7934506856227626
},
"prediction_count": 1,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": null,
"recall_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"reference_count": 0,
"true_positive": 0
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 1,
"predictions": 1,
"ranking_predictions": 1,
"references": 0
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": null
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"large": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"footprint_segmentation"
]
},
"mixed": {
"case_support": 3,
"failure_count": 6,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"object_detection": {
"case_support": 1,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 1.0,
"ap50_95": 0.900990099009901,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 1,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.9,
"upper": 1.0
}
],
"brier": 0.15333333333333332,
"ece": 0.13333333333333325,
"status": "computed"
},
"f1": 0.8,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 0,
"false_positive": 1,
"map50": 1.0,
"map50_95": 0.900990099009901,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 1.0,
"ap50_95": 0.900990099009901,
"prediction_count": 4,
"reference_count": 2
}
},
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 3,
"predictions": 3,
"ranking_predictions": 4,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.8
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection",
"object_detection",
"vector_comparison"
]
},
"not_applicable": {
"case_support": 3,
"failure_count": 5,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation",
"raster_classification",
"terrain_interpretation"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"occlusion": {
"strata": {
"none": {
"case_support": 4,
"failure_count": 5,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"object_detection": {
"case_support": 2,
"macro": {
"f1": null,
"f1_case_support": 0,
"method": "unweighted mean across cases with defined values",
"precision": 0.0,
"precision_case_support": 1,
"recall": null,
"recall_case_support": 0,
"status": "computed"
},
"micro": {
"ap50": null,
"ap50_95": null,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": null,
"count": 0,
"lower": 0.6,
"mean_confidence": null,
"upper": 0.8
},
{
"accuracy": 0,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.92,
"upper": 1.0
}
],
"brier": 0.8464,
"ece": 0.92,
"status": "computed"
},
"f1": null,
"false_discovery_rate": 1.0,
"false_negative": 0,
"false_positive": 1,
"map50": null,
"map50_95": null,
"miss_rate": null,
"per_class_ap": {
"building": {
"ap50": null,
"ap50_95": null,
"prediction_count": 1,
"reference_count": 0
}
},
"precision": 0.0,
"precision_ci95_wilson": {
"lower": 0.0,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 0.7934506856227626
},
"prediction_count": 1,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": null,
"recall_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"reference_count": 0,
"true_positive": 0
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 1,
"predictions": 1,
"ranking_predictions": 1,
"references": 0
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": null
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection",
"footprint_segmentation",
"object_detection"
]
},
"not_applicable": {
"case_support": 4,
"failure_count": 7,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation",
"raster_classification",
"terrain_interpretation",
"vector_comparison"
]
},
"partial": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 1.0,
"ap50_95": 0.900990099009901,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 1,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.9,
"upper": 1.0
}
],
"brier": 0.15333333333333332,
"ece": 0.13333333333333325,
"status": "computed"
},
"f1": 0.8,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 0,
"false_positive": 1,
"map50": 1.0,
"map50_95": 0.900990099009901,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 1.0,
"ap50_95": 0.900990099009901,
"prediction_count": 4,
"reference_count": 2
}
},
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 3,
"predictions": 3,
"ranking_predictions": 4,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.8
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"region": {
"strata": {
"brussels": {
"case_support": 2,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": null,
"f1_case_support": 0,
"method": "unweighted mean across cases with defined values",
"precision": null,
"precision_case_support": 0,
"recall": null,
"recall_case_support": 0,
"status": "not_evaluable"
},
"micro": {
"ap50": null,
"ap50_95": null,
"calibration": {
"bins": [],
"brier": null,
"ece": null,
"status": "undefined"
},
"f1": null,
"false_discovery_rate": null,
"false_negative": 0,
"false_positive": 0,
"map50": null,
"map50_95": null,
"miss_rate": null,
"per_class_ap": {
"building": {
"ap50": null,
"ap50_95": null,
"prediction_count": 0,
"reference_count": 0
}
},
"precision": null,
"precision_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"prediction_count": 0,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": null,
"recall_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"reference_count": 0,
"true_positive": 0
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 0,
"predictions": 0,
"ranking_predictions": 0,
"references": 0
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": null
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection",
"raster_classification"
]
},
"flanders": {
"case_support": 4,
"failure_count": 7,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 2,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.3333333333333333,
"precision_case_support": 2,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.8,
"mean_confidence": 0.91,
"upper": 1.0
}
],
"brier": 0.3266,
"ece": 0.27999999999999997,
"status": "computed"
},
"f1": 0.6666666666666666,
"false_discovery_rate": 0.5,
"false_negative": 0,
"false_positive": 2,
"map50": 0.6666666666666666,
"map50_95": 0.5838283828382838,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"prediction_count": 5,
"reference_count": 2
}
},
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.15003898915214947,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 0.8499610108478506
},
"prediction_count": 4,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 4,
"predictions": 4,
"ranking_predictions": 5,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection",
"terrain_interpretation",
"vector_comparison"
]
},
"national": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation"
]
},
"wallonia": {
"case_support": 2,
"failure_count": 3,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection",
"footprint_segmentation"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"resolution_m": {
"strata": {
"0.1": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"vector_comparison"
]
},
"0.25": {
"case_support": 3,
"failure_count": 4,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 3,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.3333333333333333,
"precision_case_support": 2,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.8,
"mean_confidence": 0.91,
"upper": 1.0
}
],
"brier": 0.3266,
"ece": 0.27999999999999997,
"status": "computed"
},
"f1": 0.6666666666666666,
"false_discovery_rate": 0.5,
"false_negative": 0,
"false_positive": 2,
"map50": 0.6666666666666666,
"map50_95": 0.5838283828382838,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"prediction_count": 5,
"reference_count": 2
}
},
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.15003898915214947,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 0.8499610108478506
},
"prediction_count": 4,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 4,
"predictions": 4,
"ranking_predictions": 5,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"0.5": {
"case_support": 2,
"failure_count": 3,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection",
"footprint_segmentation"
]
},
"1": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"raster_classification"
]
},
"5": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"terrain_interpretation"
]
},
"__not_applicable__": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"season": {
"strata": {
"autumn": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"raster_classification"
]
},
"multi-date": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection"
]
},
"not_applicable": {
"case_support": 3,
"failure_count": 5,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation",
"terrain_interpretation",
"vector_comparison"
]
},
"spring": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"footprint_segmentation"
]
},
"summer": {
"case_support": 2,
"failure_count": 4,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 2,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.3333333333333333,
"precision_case_support": 2,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.8,
"mean_confidence": 0.91,
"upper": 1.0
}
],
"brier": 0.3266,
"ece": 0.27999999999999997,
"status": "computed"
},
"f1": 0.6666666666666666,
"false_discovery_rate": 0.5,
"false_negative": 0,
"false_positive": 2,
"map50": 0.6666666666666666,
"map50_95": 0.5838283828382838,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"prediction_count": 5,
"reference_count": 2
}
},
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.15003898915214947,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 0.8499610108478506
},
"prediction_count": 4,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 4,
"predictions": 4,
"ranking_predictions": 5,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"winter": {
"case_support": 1,
"failure_count": 0,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 1,
"macro": {
"f1": null,
"f1_case_support": 0,
"method": "unweighted mean across cases with defined values",
"precision": null,
"precision_case_support": 0,
"recall": null,
"recall_case_support": 0,
"status": "not_evaluable"
},
"micro": {
"ap50": null,
"ap50_95": null,
"calibration": {
"bins": [],
"brier": null,
"ece": null,
"status": "undefined"
},
"f1": null,
"false_discovery_rate": null,
"false_negative": 0,
"false_positive": 0,
"map50": null,
"map50_95": null,
"miss_rate": null,
"per_class_ap": {
"building": {
"ap50": null,
"ap50_95": null,
"prediction_count": 0,
"reference_count": 0
}
},
"precision": null,
"precision_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"prediction_count": 0,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": null,
"recall_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"reference_count": 0,
"true_positive": 0
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 0,
"predictions": 0,
"ranking_predictions": 0,
"references": 0
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": null
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"sensor": {
"strata": {
"elevation-raster": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"terrain_interpretation"
]
},
"mixed": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation"
]
},
"synthetic-multispectral": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"raster_classification"
]
},
"synthetic-rgb": {
"case_support": 4,
"failure_count": 5,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"object_detection": {
"case_support": 3,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.3333333333333333,
"precision_case_support": 2,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.8,
"mean_confidence": 0.91,
"upper": 1.0
}
],
"brier": 0.3266,
"ece": 0.27999999999999997,
"status": "computed"
},
"f1": 0.6666666666666666,
"false_discovery_rate": 0.5,
"false_negative": 0,
"false_positive": 2,
"map50": 0.6666666666666666,
"map50_95": 0.5838283828382838,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"prediction_count": 5,
"reference_count": 2
}
},
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.15003898915214947,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 0.8499610108478506
},
"prediction_count": 4,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 4,
"predictions": 4,
"ranking_predictions": 5,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"footprint_segmentation",
"object_detection"
]
},
"vector": {
"case_support": 2,
"failure_count": 4,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection",
"vector_comparison"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"source": {
"strata": {
"fixture-contracts": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation"
]
},
"fixture-dhmv": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"terrain_interpretation"
]
},
"fixture-grb": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"vector_comparison"
]
},
"fixture-orthophoto": {
"case_support": 3,
"failure_count": 4,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 3,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.3333333333333333,
"precision_case_support": 2,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.8,
"mean_confidence": 0.91,
"upper": 1.0
}
],
"brier": 0.3266,
"ece": 0.27999999999999997,
"status": "computed"
},
"f1": 0.6666666666666666,
"false_discovery_rate": 0.5,
"false_negative": 0,
"false_positive": 2,
"map50": 0.6666666666666666,
"map50_95": 0.5838283828382838,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"prediction_count": 5,
"reference_count": 2
}
},
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.15003898915214947,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 0.8499610108478506
},
"prediction_count": 4,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 4,
"predictions": 4,
"ranking_predictions": 5,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection"
]
},
"fixture-picc": {
"case_support": 1,
"failure_count": 1,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"footprint_segmentation"
]
},
"fixture-picc-temporal": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection"
]
},
"fixture-thematic-raster": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"raster_classification"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"urbanity": {
"strata": {
"industrial": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection"
]
},
"mixed": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation"
]
},
"rural": {
"case_support": 2,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"footprint_segmentation",
"terrain_interpretation"
]
},
"suburban": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"vector_comparison"
]
},
"urban": {
"case_support": 4,
"failure_count": 6,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 3,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.3333333333333333,
"precision_case_support": 2,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.8,
"mean_confidence": 0.91,
"upper": 1.0
}
],
"brier": 0.3266,
"ece": 0.27999999999999997,
"status": "computed"
},
"f1": 0.6666666666666666,
"false_discovery_rate": 0.5,
"false_negative": 0,
"false_positive": 2,
"map50": 0.6666666666666666,
"map50_95": 0.5838283828382838,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"prediction_count": 5,
"reference_count": 2
}
},
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.15003898915214947,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 0.8499610108478506
},
"prediction_count": 4,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 4,
"predictions": 4,
"ranking_predictions": 5,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection",
"raster_classification"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
},
"vegetation": {
"strata": {
"low": {
"case_support": 4,
"failure_count": 5,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"change_detection": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"footprint_segmentation": {
"case_support": 1,
"macro": {
"f1": 1.0,
"f1_case_support": 1,
"mean_boundary_f1": 1.0,
"mean_boundary_f1_case_support": 1,
"mean_dice": 0.8881578947368421,
"mean_dice_case_support": 1,
"mean_iou": 0.7988165680473372,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 1.0,
"precision_case_support": 1,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"prediction_count": 1,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.20654931437723745,
"method": "Wilson score interval",
"status": "computed",
"support": 1,
"upper": 1.0
},
"reference_count": 1,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 1,
"references": 1
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 1.0
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"object_detection": {
"case_support": 1,
"macro": {
"f1": null,
"f1_case_support": 0,
"method": "unweighted mean across cases with defined values",
"precision": null,
"precision_case_support": 0,
"recall": null,
"recall_case_support": 0,
"status": "not_evaluable"
},
"micro": {
"ap50": null,
"ap50_95": null,
"calibration": {
"bins": [],
"brier": null,
"ece": null,
"status": "undefined"
},
"f1": null,
"false_discovery_rate": null,
"false_negative": 0,
"false_positive": 0,
"map50": null,
"map50_95": null,
"miss_rate": null,
"per_class_ap": {
"building": {
"ap50": null,
"ap50_95": null,
"prediction_count": 0,
"reference_count": 0
}
},
"precision": null,
"precision_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"prediction_count": 0,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": null,
"recall_ci95_wilson": {
"lower": null,
"status": "undefined",
"support": 0,
"upper": null
},
"reference_count": 0,
"true_positive": 0
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 0,
"predictions": 0,
"ranking_predictions": 0,
"references": 0
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": null
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"vector_comparison": {
"case_support": 1,
"macro": {
"f1": 0.5,
"f1_case_support": 1,
"mean_iou": 1.0,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.5,
"precision_case_support": 1,
"recall": 0.5,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"f1": 0.5,
"false_discovery_rate": 0.5,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.5,
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"prediction_count": 2,
"recall": 0.5,
"recall_ci95_wilson": {
"lower": 0.09453120573423074,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 0.9054687942657693
},
"reference_count": 2,
"true_positive": 1
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 2,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.5
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"change_detection",
"footprint_segmentation",
"object_detection",
"vector_comparison"
]
},
"mixed": {
"case_support": 1,
"failure_count": 2,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"geospatial_data_validation": {
"case_support": 1,
"macro": {
"blocker_or_critical_miss_count": 1.0,
"blocker_or_critical_miss_count_case_support": 1,
"f1": 0.6666666666666666,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.6666666666666666,
"precision_case_support": 1,
"recall": 0.6666666666666666,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"blocker_or_critical_miss_count": 1,
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"predictions": 3,
"references": 3
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"geospatial_data_validation"
]
},
"moderate": {
"case_support": 4,
"failure_count": 7,
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"task_metrics": {
"object_detection": {
"case_support": 2,
"macro": {
"f1": 0.8,
"f1_case_support": 1,
"method": "unweighted mean across cases with defined values",
"precision": 0.3333333333333333,
"precision_case_support": 2,
"recall": 1.0,
"recall_case_support": 1,
"status": "computed"
},
"micro": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.8,
"mean_confidence": 0.91,
"upper": 1.0
}
],
"brier": 0.3266,
"ece": 0.27999999999999997,
"status": "computed"
},
"f1": 0.6666666666666666,
"false_discovery_rate": 0.5,
"false_negative": 0,
"false_positive": 2,
"map50": 0.6666666666666666,
"map50_95": 0.5838283828382838,
"miss_rate": 0.0,
"per_class_ap": {
"building": {
"ap50": 0.6666666666666666,
"ap50_95": 0.5838283828382838,
"prediction_count": 5,
"reference_count": 2
}
},
"precision": 0.5,
"precision_ci95_wilson": {
"lower": 0.15003898915214947,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 0.8499610108478506
},
"prediction_count": 4,
"ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP",
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.34238022750665303,
"method": "Wilson score interval",
"status": "computed",
"support": 2,
"upper": 1.0
},
"reference_count": 2,
"true_positive": 2
},
"minimum_case_support": 5,
"observation_support": {
"calibration_predictions": 4,
"predictions": 4,
"ranking_predictions": 5,
"references": 2
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.f1",
"value": 0.6666666666666666
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"raster_classification": {
"case_support": 1,
"macro": {
"accuracy": 0.8333333333333334,
"accuracy_case_support": 1,
"macro_f1": 0.8222222222222223,
"macro_f1_case_support": 1,
"mean_iou": 0.7222222222222222,
"mean_iou_case_support": 1,
"method": "unweighted mean across cases with defined values",
"status": "computed"
},
"micro": {
"accuracy": 0.8333333333333334,
"accuracy_ci95_wilson": {
"lower": 0.5519691377470266,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 0.9530348578161463
},
"correct_pixel_count": 10,
"evaluated_reference_pixel_count": 12,
"macro_f1_across_classes": 0.8222222222222223,
"mean_iou": 0.7222222222222222,
"per_class": {
"0": {
"f1": 1.0,
"false_discovery_rate": 0.0,
"false_negative": 0,
"false_positive": 0,
"iou": 1.0,
"miss_rate": 0.0,
"precision": 1.0,
"precision_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"prediction_count": 4,
"recall": 1.0,
"recall_ci95_wilson": {
"lower": 0.5101091635454027,
"method": "Wilson score interval",
"status": "computed",
"support": 4,
"upper": 1.0
},
"reference_count": 4,
"true_positive": 4
},
"1": {
"f1": 0.8000000000000002,
"false_discovery_rate": 0.2,
"false_negative": 1,
"false_positive": 1,
"iou": 0.6666666666666666,
"miss_rate": 0.2,
"precision": 0.8,
"precision_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"prediction_count": 5,
"recall": 0.8,
"recall_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"reference_count": 5,
"true_positive": 4
},
"2": {
"f1": 0.6666666666666666,
"false_discovery_rate": 0.3333333333333333,
"false_negative": 1,
"false_positive": 1,
"iou": 0.5,
"miss_rate": 0.3333333333333333,
"precision": 0.6666666666666666,
"precision_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"prediction_count": 3,
"recall": 0.6666666666666666,
"recall_ci95_wilson": {
"lower": 0.2076596008020477,
"method": "Wilson score interval",
"status": "computed",
"support": 3,
"upper": 0.9385080552796037
},
"reference_count": 3,
"true_positive": 2
}
},
"prediction_coverage": 1.0,
"prediction_coverage_ci95_wilson": {
"lower": 0.7575059933447592,
"method": "Wilson score interval",
"status": "computed",
"support": 12,
"upper": 1.0
},
"prediction_present_pixel_count": 12
},
"minimum_case_support": 5,
"observation_support": {
"evaluated_pixels": 12
},
"primary_metric": {
"direction": "higher_is_better",
"name": "micro.mean_iou",
"value": 0.7222222222222222
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
},
"terrain_interpretation": {
"case_support": 1,
"macro": {
"bias": 0.02499999999999991,
"bias_case_support": 1,
"coverage": 0.8,
"coverage_case_support": 1,
"mae": 0.2749999999999999,
"mae_case_support": 1,
"method": "unweighted mean across cases with defined values",
"rmse": 0.2872281323269015,
"rmse_case_support": 1,
"status": "computed"
},
"micro": {
"bias": 0.02499999999999991,
"coverage": 0.8,
"coverage_ci95_wilson": {
"lower": 0.37553462976252533,
"method": "Wilson score interval",
"status": "computed",
"support": 5,
"upper": 0.9637758913675698
},
"evaluated_count": 4,
"mae": 0.2749999999999999,
"reference_count": 5,
"rmse": 0.2872281323269015,
"status": "computed",
"units": "m_TAW"
},
"minimum_case_support": 5,
"observation_support": {
"evaluated": 4,
"references": 5
},
"primary_metric": {
"direction": "lower_is_better",
"name": "micro.rmse",
"value": 0.2872281323269015
},
"release_gate_reason": "insufficient_support",
"release_gate_status": "not_evaluable",
"status": "insufficient_support"
}
},
"tasks": [
"object_detection",
"raster_classification",
"terrain_interpretation"
]
}
},
"worst_stratum_by_task": {
"change_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"footprint_segmentation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"geospatial_data_validation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"object_detection": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"raster_classification": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"terrain_interpretation": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
},
"vector_comparison": {
"reason": "no stratum meets minimum support with a defined primary metric",
"status": "not_evaluable"
}
}
}
},
"minimum_case_support": 5,
"overall_status": "not_evaluable",
"schema_version": 2,
"support_unit": "independent benchmark cases; pixel/object counts are reported separately and do not replace case support"
}
Reference in New Issue View Git Blame Copy Permalink