Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs(accuracy): publish phase 4 benchmark evidence
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
Details
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
Details
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
Jens
2026-08-02 05:10:05 +02:00
parent 70fb4b94e5
commit f41392a415
34 changed files with 25672 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/calibration-metrics.json
+274
View File
@@ -0,0 +1,274 @@
{
"items": [
{
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": null,
"count": 0,
"lower": 0.6,
"mean_confidence": null,
"upper": 0.8
},
{
"accuracy": 0,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.92,
"upper": 1.0
}
],
"brier": 0.8464,
"ece": 0.92,
"status": "computed"
},
"coverage_risk": [
{
"false_negative_count": 0,
"false_positive_count": 1,
"matched_reference_count": 0,
"reference_count": 0,
"reference_coverage": null,
"retained_prediction_count": 1,
"retained_prediction_coverage": 1.0,
"risk": 1.0,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.0,
"total_prediction_count": 1
},
{
"false_negative_count": 0,
"false_positive_count": 1,
"matched_reference_count": 0,
"reference_count": 0,
"reference_coverage": null,
"retained_prediction_count": 1,
"retained_prediction_coverage": 1.0,
"risk": 1.0,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.5,
"total_prediction_count": 1
},
{
"false_negative_count": 0,
"false_positive_count": 1,
"matched_reference_count": 0,
"reference_count": 0,
"reference_coverage": null,
"retained_prediction_count": 1,
"retained_prediction_coverage": 1.0,
"risk": 1.0,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.7,
"total_prediction_count": 1
},
{
"false_negative_count": 0,
"false_positive_count": 1,
"matched_reference_count": 0,
"reference_count": 0,
"reference_coverage": null,
"retained_prediction_count": 1,
"retained_prediction_coverage": 1.0,
"risk": 1.0,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.9,
"total_prediction_count": 1
}
],
"sample_id": "background-test-hard-negative",
"task": "object_detection"
},
{
"calibration": {
"bins": [],
"brier": null,
"ece": null,
"status": "undefined"
},
"coverage_risk": [
{
"false_negative_count": 0,
"false_positive_count": 0,
"matched_reference_count": 0,
"reference_count": 0,
"reference_coverage": null,
"retained_prediction_count": 0,
"retained_prediction_coverage": null,
"risk": null,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.0,
"total_prediction_count": 0
},
{
"false_negative_count": 0,
"false_positive_count": 0,
"matched_reference_count": 0,
"reference_count": 0,
"reference_coverage": null,
"retained_prediction_count": 0,
"retained_prediction_coverage": null,
"risk": null,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.5,
"total_prediction_count": 0
},
{
"false_negative_count": 0,
"false_positive_count": 0,
"matched_reference_count": 0,
"reference_count": 0,
"reference_coverage": null,
"retained_prediction_count": 0,
"retained_prediction_coverage": null,
"risk": null,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.7,
"total_prediction_count": 0
},
{
"false_negative_count": 0,
"false_positive_count": 0,
"matched_reference_count": 0,
"reference_count": 0,
"reference_coverage": null,
"retained_prediction_count": 0,
"retained_prediction_coverage": null,
"risk": null,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.9,
"total_prediction_count": 0
}
],
"sample_id": "background-test-pure-empty",
"task": "object_detection"
},
{
"calibration": {
"binning": "five fixed equal-width bins",
"bins": [
{
"accuracy": null,
"count": 0,
"lower": 0.0,
"mean_confidence": null,
"upper": 0.2
},
{
"accuracy": null,
"count": 0,
"lower": 0.2,
"mean_confidence": null,
"upper": 0.4
},
{
"accuracy": null,
"count": 0,
"lower": 0.4,
"mean_confidence": null,
"upper": 0.6
},
{
"accuracy": 0.5,
"count": 2,
"lower": 0.6,
"mean_confidence": 0.6499999999999999,
"upper": 0.8
},
{
"accuracy": 1,
"count": 1,
"lower": 0.8,
"mean_confidence": 0.9,
"upper": 1.0
}
],
"brier": 0.15333333333333332,
"ece": 0.13333333333333325,
"status": "computed"
},
"coverage_risk": [
{
"false_negative_count": 0,
"false_positive_count": 2,
"matched_reference_count": 2,
"reference_count": 2,
"reference_coverage": 1.0,
"retained_prediction_count": 4,
"retained_prediction_coverage": 1.0,
"risk": 0.5,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.0,
"total_prediction_count": 4
},
{
"false_negative_count": 0,
"false_positive_count": 1,
"matched_reference_count": 2,
"reference_count": 2,
"reference_coverage": 1.0,
"retained_prediction_count": 3,
"retained_prediction_coverage": 0.75,
"risk": 0.3333333333333333,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.5,
"total_prediction_count": 4
},
{
"false_negative_count": 0,
"false_positive_count": 0,
"matched_reference_count": 2,
"reference_count": 2,
"reference_coverage": 1.0,
"retained_prediction_count": 2,
"retained_prediction_coverage": 0.5,
"risk": 0.0,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.7,
"total_prediction_count": 4
},
{
"false_negative_count": 1,
"false_positive_count": 0,
"matched_reference_count": 1,
"reference_count": 2,
"reference_coverage": 0.5,
"retained_prediction_count": 1,
"retained_prediction_coverage": 0.25,
"risk": 0.5,
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
"threshold": 0.9,
"total_prediction_count": 4
}
],
"sample_id": "det-test-flanders-urban",
"task": "object_detection"
}
],
"note": "Fixed diagnostic bins and risk thresholds test metric arithmetic; they do not select or change any operating point.",
"schema_version": 2,
"selection_allowed": false,
"status": "fixture_diagnostic_only"
}