275 lines
8.7 KiB
JSON
275 lines
8.7 KiB
JSON
{
|
|
"items": [
|
|
{
|
|
"calibration": {
|
|
"binning": "five fixed equal-width bins",
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"count": 0,
|
|
"lower": 0.0,
|
|
"mean_confidence": null,
|
|
"upper": 0.2
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"count": 0,
|
|
"lower": 0.2,
|
|
"mean_confidence": null,
|
|
"upper": 0.4
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"count": 0,
|
|
"lower": 0.4,
|
|
"mean_confidence": null,
|
|
"upper": 0.6
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"count": 0,
|
|
"lower": 0.6,
|
|
"mean_confidence": null,
|
|
"upper": 0.8
|
|
},
|
|
{
|
|
"accuracy": 0,
|
|
"count": 1,
|
|
"lower": 0.8,
|
|
"mean_confidence": 0.92,
|
|
"upper": 1.0
|
|
}
|
|
],
|
|
"brier": 0.8464,
|
|
"ece": 0.92,
|
|
"status": "computed"
|
|
},
|
|
"coverage_risk": [
|
|
{
|
|
"false_negative_count": 0,
|
|
"false_positive_count": 1,
|
|
"matched_reference_count": 0,
|
|
"reference_count": 0,
|
|
"reference_coverage": null,
|
|
"retained_prediction_count": 1,
|
|
"retained_prediction_coverage": 1.0,
|
|
"risk": 1.0,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.0,
|
|
"total_prediction_count": 1
|
|
},
|
|
{
|
|
"false_negative_count": 0,
|
|
"false_positive_count": 1,
|
|
"matched_reference_count": 0,
|
|
"reference_count": 0,
|
|
"reference_coverage": null,
|
|
"retained_prediction_count": 1,
|
|
"retained_prediction_coverage": 1.0,
|
|
"risk": 1.0,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.5,
|
|
"total_prediction_count": 1
|
|
},
|
|
{
|
|
"false_negative_count": 0,
|
|
"false_positive_count": 1,
|
|
"matched_reference_count": 0,
|
|
"reference_count": 0,
|
|
"reference_coverage": null,
|
|
"retained_prediction_count": 1,
|
|
"retained_prediction_coverage": 1.0,
|
|
"risk": 1.0,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.7,
|
|
"total_prediction_count": 1
|
|
},
|
|
{
|
|
"false_negative_count": 0,
|
|
"false_positive_count": 1,
|
|
"matched_reference_count": 0,
|
|
"reference_count": 0,
|
|
"reference_coverage": null,
|
|
"retained_prediction_count": 1,
|
|
"retained_prediction_coverage": 1.0,
|
|
"risk": 1.0,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.9,
|
|
"total_prediction_count": 1
|
|
}
|
|
],
|
|
"sample_id": "background-test-hard-negative",
|
|
"task": "object_detection"
|
|
},
|
|
{
|
|
"calibration": {
|
|
"bins": [],
|
|
"brier": null,
|
|
"ece": null,
|
|
"status": "undefined"
|
|
},
|
|
"coverage_risk": [
|
|
{
|
|
"false_negative_count": 0,
|
|
"false_positive_count": 0,
|
|
"matched_reference_count": 0,
|
|
"reference_count": 0,
|
|
"reference_coverage": null,
|
|
"retained_prediction_count": 0,
|
|
"retained_prediction_coverage": null,
|
|
"risk": null,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.0,
|
|
"total_prediction_count": 0
|
|
},
|
|
{
|
|
"false_negative_count": 0,
|
|
"false_positive_count": 0,
|
|
"matched_reference_count": 0,
|
|
"reference_count": 0,
|
|
"reference_coverage": null,
|
|
"retained_prediction_count": 0,
|
|
"retained_prediction_coverage": null,
|
|
"risk": null,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.5,
|
|
"total_prediction_count": 0
|
|
},
|
|
{
|
|
"false_negative_count": 0,
|
|
"false_positive_count": 0,
|
|
"matched_reference_count": 0,
|
|
"reference_count": 0,
|
|
"reference_coverage": null,
|
|
"retained_prediction_count": 0,
|
|
"retained_prediction_coverage": null,
|
|
"risk": null,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.7,
|
|
"total_prediction_count": 0
|
|
},
|
|
{
|
|
"false_negative_count": 0,
|
|
"false_positive_count": 0,
|
|
"matched_reference_count": 0,
|
|
"reference_count": 0,
|
|
"reference_coverage": null,
|
|
"retained_prediction_count": 0,
|
|
"retained_prediction_coverage": null,
|
|
"risk": null,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.9,
|
|
"total_prediction_count": 0
|
|
}
|
|
],
|
|
"sample_id": "background-test-pure-empty",
|
|
"task": "object_detection"
|
|
},
|
|
{
|
|
"calibration": {
|
|
"binning": "five fixed equal-width bins",
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"count": 0,
|
|
"lower": 0.0,
|
|
"mean_confidence": null,
|
|
"upper": 0.2
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"count": 0,
|
|
"lower": 0.2,
|
|
"mean_confidence": null,
|
|
"upper": 0.4
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"count": 0,
|
|
"lower": 0.4,
|
|
"mean_confidence": null,
|
|
"upper": 0.6
|
|
},
|
|
{
|
|
"accuracy": 0.5,
|
|
"count": 2,
|
|
"lower": 0.6,
|
|
"mean_confidence": 0.6499999999999999,
|
|
"upper": 0.8
|
|
},
|
|
{
|
|
"accuracy": 1,
|
|
"count": 1,
|
|
"lower": 0.8,
|
|
"mean_confidence": 0.9,
|
|
"upper": 1.0
|
|
}
|
|
],
|
|
"brier": 0.15333333333333332,
|
|
"ece": 0.13333333333333325,
|
|
"status": "computed"
|
|
},
|
|
"coverage_risk": [
|
|
{
|
|
"false_negative_count": 0,
|
|
"false_positive_count": 2,
|
|
"matched_reference_count": 2,
|
|
"reference_count": 2,
|
|
"reference_coverage": 1.0,
|
|
"retained_prediction_count": 4,
|
|
"retained_prediction_coverage": 1.0,
|
|
"risk": 0.5,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.0,
|
|
"total_prediction_count": 4
|
|
},
|
|
{
|
|
"false_negative_count": 0,
|
|
"false_positive_count": 1,
|
|
"matched_reference_count": 2,
|
|
"reference_count": 2,
|
|
"reference_coverage": 1.0,
|
|
"retained_prediction_count": 3,
|
|
"retained_prediction_coverage": 0.75,
|
|
"risk": 0.3333333333333333,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.5,
|
|
"total_prediction_count": 4
|
|
},
|
|
{
|
|
"false_negative_count": 0,
|
|
"false_positive_count": 0,
|
|
"matched_reference_count": 2,
|
|
"reference_count": 2,
|
|
"reference_coverage": 1.0,
|
|
"retained_prediction_count": 2,
|
|
"retained_prediction_coverage": 0.5,
|
|
"risk": 0.0,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.7,
|
|
"total_prediction_count": 4
|
|
},
|
|
{
|
|
"false_negative_count": 1,
|
|
"false_positive_count": 0,
|
|
"matched_reference_count": 1,
|
|
"reference_count": 2,
|
|
"reference_coverage": 0.5,
|
|
"retained_prediction_count": 1,
|
|
"retained_prediction_coverage": 0.25,
|
|
"risk": 0.5,
|
|
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
|
"threshold": 0.9,
|
|
"total_prediction_count": 4
|
|
}
|
|
],
|
|
"sample_id": "det-test-flanders-urban",
|
|
"task": "object_detection"
|
|
}
|
|
],
|
|
"note": "Fixed diagnostic bins and risk thresholds test metric arithmetic; they do not select or change any operating point.",
|
|
"schema_version": 2,
|
|
"selection_allowed": false,
|
|
"status": "fixture_diagnostic_only"
|
|
}
|