Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
8fa8e84ef6aa0dd8804b2db1f5f4f280842bceef
geointel/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/failure-gallery.json
T
Jens f41392a415
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
Details
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
Details
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s
Details
docs(accuracy): publish phase 4 benchmark evidence
2026-08-02 05:10:05 +02:00

538 lines
14 KiB
JSON
Raw Blame History

{
"failure_count": 14,
"items": [
{
"contexts": [],
"error_code": "P-PARTIAL",
"evidence": {
"missing_index": 3
},
"failure_id": "1397377fd37cd400f01a",
"kind": "terrain_missing",
"metadata": {
"context": "terrain_profile",
"date": "2023-01-01",
"difficulty": "normal",
"label_review_state": "fixture_reviewed",
"municipality": "Hasselt",
"object_size": "not_applicable",
"occlusion": "not_applicable",
"ood": false,
"region": "flanders",
"resolution_m": 5,
"season": "not_applicable",
"sensor": "elevation-raster",
"source": "fixture-dhmv",
"tile_edge": false,
"urbanity": "rural",
"vegetation": "moderate"
},
"sample_id": "terrain-test-flanders-rural",
"secondary_error_codes": [],
"task": "terrain_interpretation"
},
{
"contexts": [],
"error_code": "M-FN-MISSED",
"evidence": {
"bbox": [
10,
0,
15,
5
],
"class": "removed",
"id": "r2"
},
"failure_id": "28a0be1f9d4960036f3a",
"kind": "event_false_negative",
"metadata": {
"context": "industrial_change",
"date": "2024-05-01/2025-05-01",
"difficulty": "hard",
"label_review_state": "fixture_reviewed",
"municipality": "Liège",
"object_size": "mixed",
"occlusion": "none",
"ood": false,
"region": "wallonia",
"resolution_m": 0.5,
"season": "multi-date",
"sensor": "vector",
"source": "fixture-picc-temporal",
"tile_edge": false,
"urbanity": "industrial",
"vegetation": "low"
},
"sample_id": "change-test-wallonia-industrial",
"secondary_error_codes": [],
"task": "change_detection"
},
{
"contexts": [],
"error_code": "M-AREA-BIAS",
"evidence": {
"prediction_id": "p1",
"reference_id": "r1",
"relative_area_error": -0.1
},
"failure_id": "2d53b69d95b17bf49580",
"kind": "area_bias",
"metadata": {
"context": "rural_buildings",
"date": "2025-04-20",
"difficulty": "normal",
"label_review_state": "fixture_reviewed",
"municipality": "Namur",
"object_size": "large",
"occlusion": "none",
"ood": false,
"region": "wallonia",
"resolution_m": 0.5,
"season": "spring",
"sensor": "synthetic-rgb",
"source": "fixture-picc",
"tile_edge": false,
"urbanity": "rural",
"vegetation": "low"
},
"sample_id": "seg-test-wallonia-rural",
"secondary_error_codes": [],
"task": "footprint_segmentation"
},
{
"contexts": [
"tile_edge"
],
"error_code": "M-CLASS",
"evidence": {
"column": 1,
"pixel_index": 9,
"prediction": 1,
"reason": "class_mismatch",
"reference": 2,
"row": 2
},
"failure_id": "2fabc83865f9e1f49efc",
"kind": "raster_misclassification",
"metadata": {
"context": "dense_urban_raster",
"date": "2025-10-05",
"difficulty": "hard",
"label_review_state": "fixture_reviewed",
"municipality": "Brussel",
"object_size": "not_applicable",
"occlusion": "not_applicable",
"ood": false,
"region": "brussels",
"resolution_m": 1,
"season": "autumn",
"sensor": "synthetic-multispectral",
"source": "fixture-thematic-raster",
"tile_edge": true,
"urbanity": "urban",
"vegetation": "moderate"
},
"sample_id": "raster-test-brussels-urban",
"secondary_error_codes": [],
"task": "raster_classification"
},
{
"contexts": [
"high_confidence",
"tile_edge"
],
"error_code": "M-FP-CONFUSER",
"evidence": {
"bbox": [
20,
20,
35,
35
],
"class": "building",
"confidence": 0.92,
"id": "p-hard-fp"
},
"failure_id": "39eae35c6660b1ac1bcd",
"kind": "false_positive",
"metadata": {
"context": "coastal_hard_negative",
"date": "2025-07-01",
"difficulty": "hard",
"label_review_state": "fixture_reviewed",
"municipality": "Oostende",
"object_size": "empty",
"occlusion": "none",
"ood": false,
"region": "flanders",
"resolution_m": 0.25,
"season": "summer",
"sensor": "synthetic-rgb",
"source": "fixture-orthophoto",
"tile_edge": true,
"urbanity": "urban",
"vegetation": "moderate"
},
"sample_id": "background-test-hard-negative",
"secondary_error_codes": [
"M-MISCALIBRATED"
],
"task": "object_detection"
},
{
"contexts": [],
"error_code": "D-VALIDATION-FN",
"evidence": {
"anomaly": "D-MISSING-PROVENANCE",
"severity": "critical"
},
"failure_id": "40e33e21c399cc657f26",
"kind": "validation_false_negative",
"metadata": {
"context": "data_contract",
"date": "2025-01-01",
"difficulty": "hard",
"label_review_state": "fixture_reviewed",
"municipality": "not_applicable",
"object_size": "not_applicable",
"occlusion": "not_applicable",
"ood": false,
"region": "national",
"resolution_m": null,
"season": "not_applicable",
"sensor": "mixed",
"source": "fixture-contracts",
"tile_edge": false,
"urbanity": "mixed",
"vegetation": "mixed"
},
"sample_id": "validation-test-national",
"secondary_error_codes": [],
"task": "geospatial_data_validation"
},
{
"contexts": [],
"error_code": "M-FP-CONFUSER",
"evidence": {
"bbox": [
20,
0,
25,
5
],
"class": "removed",
"confidence": 0.65,
"id": "p2"
},
"failure_id": "49560fb2dcb347c4cff5",
"kind": "event_false_positive",
"metadata": {
"context": "industrial_change",
"date": "2024-05-01/2025-05-01",
"difficulty": "hard",
"label_review_state": "fixture_reviewed",
"municipality": "Liège",
"object_size": "mixed",
"occlusion": "none",
"ood": false,
"region": "wallonia",
"resolution_m": 0.5,
"season": "multi-date",
"sensor": "vector",
"source": "fixture-picc-temporal",
"tile_edge": false,
"urbanity": "industrial",
"vegetation": "low"
},
"sample_id": "change-test-wallonia-industrial",
"secondary_error_codes": [],
"task": "change_detection"
},
{
"contexts": [],
"error_code": "M-MISCALIBRATED",
"evidence": {
"brier": 0.15333333333333332,
"claim_boundary": "diagnostic deviation only; not a representative population-calibration claim",
"ece": 0.13333333333333325,
"prediction_support": 3
},
"failure_id": "8d4113e9688207b68f98",
"kind": "miscalibrated",
"metadata": {
"context": "dense_urban",
"date": "2025-06-15",
"difficulty": "hard",
"label_review_state": "fixture_reviewed",
"municipality": "Antwerpen",
"object_size": "mixed",
"occlusion": "partial",
"ood": false,
"region": "flanders",
"resolution_m": 0.25,
"season": "summer",
"sensor": "synthetic-rgb",
"source": "fixture-orthophoto",
"tile_edge": false,
"urbanity": "urban",
"vegetation": "moderate"
},
"sample_id": "det-test-flanders-urban",
"secondary_error_codes": [],
"task": "object_detection"
},
{
"contexts": [
"tile_edge"
],
"error_code": "M-CLASS",
"evidence": {
"column": 3,
"pixel_index": 3,
"prediction": 2,
"reason": "class_mismatch",
"reference": 1,
"row": 0
},
"failure_id": "a7c91430a7766b02755e",
"kind": "raster_misclassification",
"metadata": {
"context": "dense_urban_raster",
"date": "2025-10-05",
"difficulty": "hard",
"label_review_state": "fixture_reviewed",
"municipality": "Brussel",
"object_size": "not_applicable",
"occlusion": "not_applicable",
"ood": false,
"region": "brussels",
"resolution_m": 1,
"season": "autumn",
"sensor": "synthetic-multispectral",
"source": "fixture-thematic-raster",
"tile_edge": true,
"urbanity": "urban",
"vegetation": "moderate"
},
"sample_id": "raster-test-brussels-urban",
"secondary_error_codes": [],
"task": "raster_classification"
},
{
"contexts": [],
"error_code": "M-FP-CONFUSER",
"evidence": {
"bbox": [
85,
85,
95,
95
],
"class": "building",
"confidence": 0.6,
"id": "p3"
},
"failure_id": "b7e154a0f6ddb9d8653f",
"kind": "false_positive",
"metadata": {
"context": "dense_urban",
"date": "2025-06-15",
"difficulty": "hard",
"label_review_state": "fixture_reviewed",
"municipality": "Antwerpen",
"object_size": "mixed",
"occlusion": "partial",
"ood": false,
"region": "flanders",
"resolution_m": 0.25,
"season": "summer",
"sensor": "synthetic-rgb",
"source": "fixture-orthophoto",
"tile_edge": false,
"urbanity": "urban",
"vegetation": "moderate"
},
"sample_id": "det-test-flanders-urban",
"secondary_error_codes": [],
"task": "object_detection"
},
{
"contexts": [],
"error_code": "M-FP-CONFUSER",
"evidence": {
"class": "building",
"id": "p2",
"polygon": [
[
20,
0
],
[
24,
0
],
[
24,
4
],
[
20,
4
],
[
20,
0
]
]
},
"failure_id": "bf43633497be5101cddb",
"kind": "false_positive",
"metadata": {
"context": "suburban_buildings",
"date": "2025-01-15",
"difficulty": "normal",
"label_review_state": "fixture_reviewed",
"municipality": "Mol",
"object_size": "mixed",
"occlusion": "not_applicable",
"ood": false,
"region": "flanders",
"resolution_m": 0.1,
"season": "not_applicable",
"sensor": "vector",
"source": "fixture-grb",
"tile_edge": false,
"urbanity": "suburban",
"vegetation": "low"
},
"sample_id": "vector-test-flanders-suburban",
"secondary_error_codes": [],
"task": "vector_comparison"
},
{
"contexts": [],
"error_code": "M-FN-MISSED",
"evidence": {
"class": "building",
"id": "r2",
"polygon": [
[
10,
0
],
[
14,
0
],
[
14,
4
],
[
10,
4
],
[
10,
0
]
]
},
"failure_id": "c9a80d5bb196dc7113e2",
"kind": "false_negative",
"metadata": {
"context": "suburban_buildings",
"date": "2025-01-15",
"difficulty": "normal",
"label_review_state": "fixture_reviewed",
"municipality": "Mol",
"object_size": "mixed",
"occlusion": "not_applicable",
"ood": false,
"region": "flanders",
"resolution_m": 0.1,
"season": "not_applicable",
"sensor": "vector",
"source": "fixture-grb",
"tile_edge": false,
"urbanity": "suburban",
"vegetation": "low"
},
"sample_id": "vector-test-flanders-suburban",
"secondary_error_codes": [],
"task": "vector_comparison"
},
{
"contexts": [],
"error_code": "D-VALIDATION-FP",
"evidence": {
"anomaly": "D-RESOLUTION",
"severity": "major"
},
"failure_id": "e00a10875bea399c62c3",
"kind": "validation_false_positive",
"metadata": {
"context": "data_contract",
"date": "2025-01-01",
"difficulty": "hard",
"label_review_state": "fixture_reviewed",
"municipality": "not_applicable",
"object_size": "not_applicable",
"occlusion": "not_applicable",
"ood": false,
"region": "national",
"resolution_m": null,
"season": "not_applicable",
"sensor": "mixed",
"source": "fixture-contracts",
"tile_edge": false,
"urbanity": "mixed",
"vegetation": "mixed"
},
"sample_id": "validation-test-national",
"secondary_error_codes": [],
"task": "geospatial_data_validation"
},
{
"contexts": [
"tile_edge"
],
"error_code": "M-MISCALIBRATED",
"evidence": {
"brier": 0.8464,
"claim_boundary": "diagnostic deviation only; not a representative population-calibration claim",
"ece": 0.92,
"prediction_support": 1
},
"failure_id": "edae8689bdec601bca2b",
"kind": "miscalibrated",
"metadata": {
"context": "coastal_hard_negative",
"date": "2025-07-01",
"difficulty": "hard",
"label_review_state": "fixture_reviewed",
"municipality": "Oostende",
"object_size": "empty",
"occlusion": "none",
"ood": false,
"region": "flanders",
"resolution_m": 0.25,
"season": "summer",
"sensor": "synthetic-rgb",
"source": "fixture-orthophoto",
"tile_edge": true,
"urbanity": "urban",
"vegetation": "moderate"
},
"sample_id": "background-test-hard-negative",
"secondary_error_codes": [],
"task": "object_detection"
}
],
"items_canonical_json_sha256": "2002025055588195db73e20017882e412e988f3c55dedc89a1fd0c1dcd7a7e3e",
"rendering_status": "machine_readable_examples_retained; a visual production gallery requires controlled access to protected imagery",
"schema_version": 2,
"taxonomy": "docs/accuracy-program/05-metric-framework.md section 4"
}
Reference in New Issue View Git Blame Copy Permalink