docs(accuracy): publish phase 4 benchmark evidence
This commit is contained in:
Jens
+806
@@ -0,0 +1,806 @@
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"metadata": {
|
||||
"context": "coastal_hard_negative",
|
||||
"date": "2025-07-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Oostende",
|
||||
"object_size": "empty",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-orthophoto",
|
||||
"tile_edge": true,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"metrics": {
|
||||
"ap50": null,
|
||||
"ap50_95": null,
|
||||
"calibration": {
|
||||
"binning": "five fixed equal-width bins",
|
||||
"bins": [
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.0,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.2
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.2,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.4
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.4,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.6
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.6,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.8
|
||||
},
|
||||
{
|
||||
"accuracy": 0,
|
||||
"count": 1,
|
||||
"lower": 0.8,
|
||||
"mean_confidence": 0.92,
|
||||
"upper": 1.0
|
||||
}
|
||||
],
|
||||
"brier": 0.8464,
|
||||
"ece": 0.92,
|
||||
"status": "computed"
|
||||
},
|
||||
"coverage_risk": [
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.0,
|
||||
"total_prediction_count": 1
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.5,
|
||||
"total_prediction_count": 1
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.7,
|
||||
"total_prediction_count": 1
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.9,
|
||||
"total_prediction_count": 1
|
||||
}
|
||||
],
|
||||
"f1": null,
|
||||
"false_discovery_rate": 1.0,
|
||||
"false_negative": 0,
|
||||
"false_positive": 1,
|
||||
"map50": null,
|
||||
"map50_95": null,
|
||||
"match_iou": 0.5,
|
||||
"matched_iou": {
|
||||
"count": 0,
|
||||
"max": null,
|
||||
"mean": null,
|
||||
"median": null,
|
||||
"min": null
|
||||
},
|
||||
"miss_rate": null,
|
||||
"operating_confidence": 0.5,
|
||||
"per_class": {
|
||||
"building": {
|
||||
"ap50": null,
|
||||
"ap50_95": null,
|
||||
"f1": null,
|
||||
"false_discovery_rate": 1.0,
|
||||
"false_negative": 0,
|
||||
"false_positive": 1,
|
||||
"miss_rate": null,
|
||||
"precision": 0.0,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.0,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 0.7934506856227626
|
||||
},
|
||||
"prediction_count": 1,
|
||||
"recall": null,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"reference_count": 0,
|
||||
"true_positive": 0
|
||||
}
|
||||
},
|
||||
"precision": 0.0,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.0,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 0.7934506856227626
|
||||
},
|
||||
"prediction_count": 1,
|
||||
"recall": null,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"reference_count": 0,
|
||||
"true_positive": 0
|
||||
},
|
||||
"sample_id": "background-test-hard-negative",
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "pure_background",
|
||||
"date": "2025-02-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Brussel",
|
||||
"object_size": "empty",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "brussels",
|
||||
"resolution_m": 0.25,
|
||||
"season": "winter",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-orthophoto",
|
||||
"tile_edge": false,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"metrics": {
|
||||
"ap50": null,
|
||||
"ap50_95": null,
|
||||
"calibration": {
|
||||
"bins": [],
|
||||
"brier": null,
|
||||
"ece": null,
|
||||
"status": "undefined"
|
||||
},
|
||||
"coverage_risk": [
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.0,
|
||||
"total_prediction_count": 0
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.5,
|
||||
"total_prediction_count": 0
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.7,
|
||||
"total_prediction_count": 0
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.9,
|
||||
"total_prediction_count": 0
|
||||
}
|
||||
],
|
||||
"f1": null,
|
||||
"false_discovery_rate": null,
|
||||
"false_negative": 0,
|
||||
"false_positive": 0,
|
||||
"map50": null,
|
||||
"map50_95": null,
|
||||
"match_iou": 0.5,
|
||||
"matched_iou": {
|
||||
"count": 0,
|
||||
"max": null,
|
||||
"mean": null,
|
||||
"median": null,
|
||||
"min": null
|
||||
},
|
||||
"miss_rate": null,
|
||||
"operating_confidence": 0.5,
|
||||
"per_class": {
|
||||
"building": {
|
||||
"ap50": null,
|
||||
"ap50_95": null,
|
||||
"f1": null,
|
||||
"false_discovery_rate": null,
|
||||
"false_negative": 0,
|
||||
"false_positive": 0,
|
||||
"miss_rate": null,
|
||||
"precision": null,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"prediction_count": 0,
|
||||
"recall": null,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"reference_count": 0,
|
||||
"true_positive": 0
|
||||
}
|
||||
},
|
||||
"precision": null,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"prediction_count": 0,
|
||||
"recall": null,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"reference_count": 0,
|
||||
"true_positive": 0
|
||||
},
|
||||
"sample_id": "background-test-pure-empty",
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "industrial_change",
|
||||
"date": "2024-05-01/2025-05-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Liège",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "wallonia",
|
||||
"resolution_m": 0.5,
|
||||
"season": "multi-date",
|
||||
"sensor": "vector",
|
||||
"source": "fixture-picc-temporal",
|
||||
"tile_edge": false,
|
||||
"urbanity": "industrial",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"metrics": {
|
||||
"event_metrics": {
|
||||
"added": {
|
||||
"f1": 1.0,
|
||||
"false_discovery_rate": 0.0,
|
||||
"false_negative": 0,
|
||||
"false_positive": 0,
|
||||
"miss_rate": 0.0,
|
||||
"precision": 1.0,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.20654931437723745,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 1.0
|
||||
},
|
||||
"prediction_count": 1,
|
||||
"recall": 1.0,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.20654931437723745,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 1.0
|
||||
},
|
||||
"reference_count": 1,
|
||||
"true_positive": 1
|
||||
},
|
||||
"removed": {
|
||||
"f1": null,
|
||||
"false_discovery_rate": 1.0,
|
||||
"false_negative": 1,
|
||||
"false_positive": 1,
|
||||
"miss_rate": 1.0,
|
||||
"precision": 0.0,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.0,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 0.7934506856227626
|
||||
},
|
||||
"prediction_count": 1,
|
||||
"recall": 0.0,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.0,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 0.7934506856227626
|
||||
},
|
||||
"reference_count": 1,
|
||||
"true_positive": 0
|
||||
}
|
||||
},
|
||||
"f1": 0.5,
|
||||
"false_discovery_rate": 0.5,
|
||||
"false_negative": 1,
|
||||
"false_positive": 1,
|
||||
"miss_rate": 0.5,
|
||||
"precision": 0.5,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.09453120573423074,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 0.9054687942657693
|
||||
},
|
||||
"prediction_count": 2,
|
||||
"recall": 0.5,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.09453120573423074,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 0.9054687942657693
|
||||
},
|
||||
"reference_count": 2,
|
||||
"true_positive": 1
|
||||
},
|
||||
"sample_id": "change-test-wallonia-industrial",
|
||||
"task": "change_detection"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "dense_urban",
|
||||
"date": "2025-06-15",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Antwerpen",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "partial",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-orthophoto",
|
||||
"tile_edge": false,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"metrics": {
|
||||
"ap50": 1.0,
|
||||
"ap50_95": 0.900990099009901,
|
||||
"calibration": {
|
||||
"binning": "five fixed equal-width bins",
|
||||
"bins": [
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.0,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.2
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.2,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.4
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.4,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.6
|
||||
},
|
||||
{
|
||||
"accuracy": 0.5,
|
||||
"count": 2,
|
||||
"lower": 0.6,
|
||||
"mean_confidence": 0.6499999999999999,
|
||||
"upper": 0.8
|
||||
},
|
||||
{
|
||||
"accuracy": 1,
|
||||
"count": 1,
|
||||
"lower": 0.8,
|
||||
"mean_confidence": 0.9,
|
||||
"upper": 1.0
|
||||
}
|
||||
],
|
||||
"brier": 0.15333333333333332,
|
||||
"ece": 0.13333333333333325,
|
||||
"status": "computed"
|
||||
},
|
||||
"coverage_risk": [
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 2,
|
||||
"matched_reference_count": 2,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 1.0,
|
||||
"retained_prediction_count": 4,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 0.5,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.0,
|
||||
"total_prediction_count": 4
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 2,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 1.0,
|
||||
"retained_prediction_count": 3,
|
||||
"retained_prediction_coverage": 0.75,
|
||||
"risk": 0.3333333333333333,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.5,
|
||||
"total_prediction_count": 4
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 2,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 1.0,
|
||||
"retained_prediction_count": 2,
|
||||
"retained_prediction_coverage": 0.5,
|
||||
"risk": 0.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.7,
|
||||
"total_prediction_count": 4
|
||||
},
|
||||
{
|
||||
"false_negative_count": 1,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 1,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 0.5,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 0.25,
|
||||
"risk": 0.5,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.9,
|
||||
"total_prediction_count": 4
|
||||
}
|
||||
],
|
||||
"f1": 0.8,
|
||||
"false_discovery_rate": 0.3333333333333333,
|
||||
"false_negative": 0,
|
||||
"false_positive": 1,
|
||||
"map50": 1.0,
|
||||
"map50_95": 0.900990099009901,
|
||||
"match_iou": 0.5,
|
||||
"matched_iou": {
|
||||
"count": 2,
|
||||
"max": 1.0,
|
||||
"mean": 0.9355555555555555,
|
||||
"median": 0.9355555555555555,
|
||||
"min": 0.8711111111111111
|
||||
},
|
||||
"miss_rate": 0.0,
|
||||
"operating_confidence": 0.5,
|
||||
"per_class": {
|
||||
"building": {
|
||||
"ap50": 1.0,
|
||||
"ap50_95": 0.900990099009901,
|
||||
"f1": 0.8,
|
||||
"false_discovery_rate": 0.3333333333333333,
|
||||
"false_negative": 0,
|
||||
"false_positive": 1,
|
||||
"miss_rate": 0.0,
|
||||
"precision": 0.6666666666666666,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.2076596008020477,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 3,
|
||||
"upper": 0.9385080552796037
|
||||
},
|
||||
"prediction_count": 3,
|
||||
"recall": 1.0,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.34238022750665303,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 1.0
|
||||
},
|
||||
"reference_count": 2,
|
||||
"true_positive": 2
|
||||
}
|
||||
},
|
||||
"precision": 0.6666666666666666,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.2076596008020477,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 3,
|
||||
"upper": 0.9385080552796037
|
||||
},
|
||||
"prediction_count": 3,
|
||||
"recall": 1.0,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.34238022750665303,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 1.0
|
||||
},
|
||||
"reference_count": 2,
|
||||
"true_positive": 2
|
||||
},
|
||||
"sample_id": "det-test-flanders-urban",
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "rural_buildings",
|
||||
"date": "2025-04-20",
|
||||
"difficulty": "normal",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Namur",
|
||||
"object_size": "large",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "wallonia",
|
||||
"resolution_m": 0.5,
|
||||
"season": "spring",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-picc",
|
||||
"tile_edge": false,
|
||||
"urbanity": "rural",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"metrics": {
|
||||
"centroid_distance_m": {
|
||||
"count": 1,
|
||||
"max": 0.5,
|
||||
"mean": 0.5,
|
||||
"median": 0.5,
|
||||
"min": 0.5
|
||||
},
|
||||
"f1": 1.0,
|
||||
"false_discovery_rate": 0.0,
|
||||
"false_negative": 0,
|
||||
"false_positive": 0,
|
||||
"mean_boundary_f1": 1.0,
|
||||
"mean_dice": 0.8881578947368421,
|
||||
"mean_iou": 0.7988165680473372,
|
||||
"miss_rate": 0.0,
|
||||
"precision": 1.0,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.20654931437723745,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 1.0
|
||||
},
|
||||
"prediction_count": 1,
|
||||
"recall": 1.0,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.20654931437723745,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 1.0
|
||||
},
|
||||
"reference_count": 1,
|
||||
"relative_area_error": {
|
||||
"count": 1,
|
||||
"max": -0.1,
|
||||
"mean": -0.1,
|
||||
"median": -0.1,
|
||||
"min": -0.1
|
||||
},
|
||||
"spatial_context_validated": true,
|
||||
"topologically_invalid_predictions": 0,
|
||||
"topologically_valid_predictions": 1,
|
||||
"true_positive": 1
|
||||
},
|
||||
"sample_id": "seg-test-wallonia-rural",
|
||||
"task": "footprint_segmentation"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "data_contract",
|
||||
"date": "2025-01-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "not_applicable",
|
||||
"object_size": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "national",
|
||||
"resolution_m": null,
|
||||
"season": "not_applicable",
|
||||
"sensor": "mixed",
|
||||
"source": "fixture-contracts",
|
||||
"tile_edge": false,
|
||||
"urbanity": "mixed",
|
||||
"vegetation": "mixed"
|
||||
},
|
||||
"metrics": {
|
||||
"blocker_or_critical_miss_count": 1,
|
||||
"f1": 0.6666666666666666,
|
||||
"false_discovery_rate": 0.3333333333333333,
|
||||
"false_negative": 1,
|
||||
"false_positive": 1,
|
||||
"miss_rate": 0.3333333333333333,
|
||||
"misses_by_severity": {
|
||||
"blocker": 0,
|
||||
"critical": 1,
|
||||
"informational": 0,
|
||||
"major": 0,
|
||||
"minor": 0
|
||||
},
|
||||
"precision": 0.6666666666666666,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.2076596008020477,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 3,
|
||||
"upper": 0.9385080552796037
|
||||
},
|
||||
"prediction_count": 3,
|
||||
"recall": 0.6666666666666666,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.2076596008020477,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 3,
|
||||
"upper": 0.9385080552796037
|
||||
},
|
||||
"reference_count": 3,
|
||||
"severity_mismatch_count": 0,
|
||||
"severity_mismatches": [],
|
||||
"true_positive": 2
|
||||
},
|
||||
"sample_id": "validation-test-national",
|
||||
"task": "geospatial_data_validation"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "suburban_buildings",
|
||||
"date": "2025-01-15",
|
||||
"difficulty": "normal",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Mol",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.1,
|
||||
"season": "not_applicable",
|
||||
"sensor": "vector",
|
||||
"source": "fixture-grb",
|
||||
"tile_edge": false,
|
||||
"urbanity": "suburban",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.5,
|
||||
"false_discovery_rate": 0.5,
|
||||
"false_negative": 1,
|
||||
"false_positive": 1,
|
||||
"mean_iou": 1.0,
|
||||
"miss_rate": 0.5,
|
||||
"precision": 0.5,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.09453120573423074,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 0.9054687942657693
|
||||
},
|
||||
"prediction_count": 2,
|
||||
"recall": 0.5,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.09453120573423074,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 0.9054687942657693
|
||||
},
|
||||
"reference_count": 2,
|
||||
"spatial_context_validated": true,
|
||||
"topologically_valid": true,
|
||||
"true_positive": 1
|
||||
},
|
||||
"sample_id": "vector-test-flanders-suburban",
|
||||
"task": "vector_comparison"
|
||||
}
|
||||
],
|
||||
"schema_version": 2,
|
||||
"status": "fixture_contract_only"
|
||||
}
|
||||
Reference in New Issue
Block a user