feat(accuracy): build hardened phase 4 evaluation harness
This commit is contained in:
Jens
@@ -0,0 +1,881 @@
|
||||
{
|
||||
"schema_version": 2,
|
||||
"portfolio_id": "geointel-p4-reference-harness-v2",
|
||||
"split_roles": [
|
||||
"test",
|
||||
"background-test"
|
||||
],
|
||||
"selection_policy": "No data-dependent threshold, model or gate selection is performed from protected cases.",
|
||||
"protected_policy": {
|
||||
"operating_point_selection_allowed": false,
|
||||
"diagnostic_curves_select_operating_point": false,
|
||||
"test_feedback_allowed": false,
|
||||
"threshold_selection_source": "pre_registered_configuration_only"
|
||||
},
|
||||
"portfolio_lineage": {
|
||||
"origin": "repository_fixture",
|
||||
"source_path": "fixtures/accuracy/p4/protected-baseline-cases.json",
|
||||
"version": "1"
|
||||
},
|
||||
"claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.",
|
||||
"cases": [
|
||||
{
|
||||
"sample_id": "det-test-flanders-urban",
|
||||
"split": "test",
|
||||
"task": "object_detection",
|
||||
"metadata": {
|
||||
"region": "flanders",
|
||||
"municipality": "Antwerpen",
|
||||
"urbanity": "urban",
|
||||
"object_size": "mixed",
|
||||
"source": "fixture-orthophoto",
|
||||
"sensor": "synthetic-rgb",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"occlusion": "partial",
|
||||
"vegetation": "moderate",
|
||||
"difficulty": "hard",
|
||||
"context": "dense_urban",
|
||||
"tile_edge": false,
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"ood": false,
|
||||
"date": "2025-06-15"
|
||||
},
|
||||
"config": {
|
||||
"confidence_threshold": 0.5,
|
||||
"match_iou": 0.5,
|
||||
"fixed_diagnostic_risk_thresholds": [
|
||||
0,
|
||||
0.5,
|
||||
0.7,
|
||||
0.9
|
||||
]
|
||||
},
|
||||
"lineage": {
|
||||
"reference": {
|
||||
"source_id": "synthetic:det-test-flanders-urban:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture"
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": "synthetic:det-test-flanders-urban:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output"
|
||||
}
|
||||
},
|
||||
"classes": [
|
||||
"building"
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"id": "r1",
|
||||
"class": "building",
|
||||
"bbox": [
|
||||
10,
|
||||
10,
|
||||
30,
|
||||
30
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "r2",
|
||||
"class": "building",
|
||||
"bbox": [
|
||||
50,
|
||||
50,
|
||||
80,
|
||||
80
|
||||
]
|
||||
}
|
||||
],
|
||||
"predictions": [
|
||||
{
|
||||
"id": "p1",
|
||||
"class": "building",
|
||||
"bbox": [
|
||||
10,
|
||||
10,
|
||||
30,
|
||||
30
|
||||
],
|
||||
"confidence": 0.9
|
||||
},
|
||||
{
|
||||
"id": "p2",
|
||||
"class": "building",
|
||||
"bbox": [
|
||||
52,
|
||||
52,
|
||||
80,
|
||||
80
|
||||
],
|
||||
"confidence": 0.7
|
||||
},
|
||||
{
|
||||
"id": "p3",
|
||||
"class": "building",
|
||||
"bbox": [
|
||||
85,
|
||||
85,
|
||||
95,
|
||||
95
|
||||
],
|
||||
"confidence": 0.6
|
||||
},
|
||||
{
|
||||
"id": "p4-filtered",
|
||||
"class": "building",
|
||||
"bbox": [
|
||||
2,
|
||||
80,
|
||||
7,
|
||||
86
|
||||
],
|
||||
"confidence": 0.2
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"sample_id": "seg-test-wallonia-rural",
|
||||
"split": "test",
|
||||
"task": "footprint_segmentation",
|
||||
"metadata": {
|
||||
"region": "wallonia",
|
||||
"municipality": "Namur",
|
||||
"urbanity": "rural",
|
||||
"object_size": "large",
|
||||
"source": "fixture-picc",
|
||||
"sensor": "synthetic-rgb",
|
||||
"resolution_m": 0.5,
|
||||
"season": "spring",
|
||||
"occlusion": "none",
|
||||
"vegetation": "low",
|
||||
"difficulty": "normal",
|
||||
"context": "rural_buildings",
|
||||
"tile_edge": false,
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"ood": false,
|
||||
"date": "2025-04-20"
|
||||
},
|
||||
"config": {
|
||||
"match_iou": 0.5,
|
||||
"boundary_tolerance_m": 1
|
||||
},
|
||||
"lineage": {
|
||||
"reference": {
|
||||
"source_id": "synthetic:seg-test-wallonia-rural:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture"
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": "synthetic:seg-test-wallonia-rural:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output"
|
||||
}
|
||||
},
|
||||
"spatial_context": {
|
||||
"crs": "EPSG:31370",
|
||||
"coordinate_units": "m",
|
||||
"metric": true
|
||||
},
|
||||
"classes": [
|
||||
"building"
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"id": "r1",
|
||||
"class": "building",
|
||||
"polygon": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
10,
|
||||
0
|
||||
],
|
||||
[
|
||||
10,
|
||||
8
|
||||
],
|
||||
[
|
||||
0,
|
||||
8
|
||||
],
|
||||
[
|
||||
0,
|
||||
0
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"predictions": [
|
||||
{
|
||||
"id": "p1",
|
||||
"class": "building",
|
||||
"polygon": [
|
||||
[
|
||||
0.5,
|
||||
0.5
|
||||
],
|
||||
[
|
||||
9.5,
|
||||
0.5
|
||||
],
|
||||
[
|
||||
9.5,
|
||||
8.5
|
||||
],
|
||||
[
|
||||
0.5,
|
||||
8.5
|
||||
],
|
||||
[
|
||||
0.5,
|
||||
0.5
|
||||
]
|
||||
],
|
||||
"confidence": 0.8
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"sample_id": "raster-test-brussels-urban",
|
||||
"split": "test",
|
||||
"task": "raster_classification",
|
||||
"metadata": {
|
||||
"region": "brussels",
|
||||
"municipality": "Brussel",
|
||||
"urbanity": "urban",
|
||||
"object_size": "not_applicable",
|
||||
"source": "fixture-thematic-raster",
|
||||
"sensor": "synthetic-multispectral",
|
||||
"resolution_m": 1,
|
||||
"season": "autumn",
|
||||
"occlusion": "not_applicable",
|
||||
"vegetation": "moderate",
|
||||
"difficulty": "hard",
|
||||
"context": "dense_urban_raster",
|
||||
"tile_edge": true,
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"ood": false,
|
||||
"date": "2025-10-05"
|
||||
},
|
||||
"config": {
|
||||
"class_order": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"masked_pixels_excluded": true
|
||||
},
|
||||
"lineage": {
|
||||
"reference": {
|
||||
"source_id": "synthetic:raster-test-brussels-urban:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture"
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": "synthetic:raster-test-brussels-urban:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output"
|
||||
}
|
||||
},
|
||||
"classes": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"raster_context": {
|
||||
"reference": {
|
||||
"crs": "EPSG:31370",
|
||||
"transform": [
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
-1,
|
||||
3
|
||||
],
|
||||
"shape": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"nodata": null,
|
||||
"mask": [
|
||||
[
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
[
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
[
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
]
|
||||
]
|
||||
},
|
||||
"prediction": {
|
||||
"crs": "EPSG:31370",
|
||||
"transform": [
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
-1,
|
||||
3
|
||||
],
|
||||
"shape": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"nodata": null,
|
||||
"mask": [
|
||||
[
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
[
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
[
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"references": [
|
||||
[
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1
|
||||
],
|
||||
[
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
2
|
||||
],
|
||||
[
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
0
|
||||
]
|
||||
],
|
||||
"predictions": [
|
||||
[
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
[
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
2
|
||||
],
|
||||
[
|
||||
2,
|
||||
1,
|
||||
1,
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"sample_id": "vector-test-flanders-suburban",
|
||||
"split": "test",
|
||||
"task": "vector_comparison",
|
||||
"metadata": {
|
||||
"region": "flanders",
|
||||
"municipality": "Mol",
|
||||
"urbanity": "suburban",
|
||||
"object_size": "mixed",
|
||||
"source": "fixture-grb",
|
||||
"sensor": "vector",
|
||||
"resolution_m": 0.1,
|
||||
"season": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"vegetation": "low",
|
||||
"difficulty": "normal",
|
||||
"context": "suburban_buildings",
|
||||
"tile_edge": false,
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"ood": false,
|
||||
"date": "2025-01-15"
|
||||
},
|
||||
"config": {
|
||||
"match_iou": 0.5
|
||||
},
|
||||
"lineage": {
|
||||
"reference": {
|
||||
"source_id": "synthetic:vector-test-flanders-suburban:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture"
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": "synthetic:vector-test-flanders-suburban:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output"
|
||||
}
|
||||
},
|
||||
"spatial_context": {
|
||||
"crs": "EPSG:31370",
|
||||
"coordinate_units": "m",
|
||||
"metric": true
|
||||
},
|
||||
"classes": [
|
||||
"building"
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"id": "r1",
|
||||
"class": "building",
|
||||
"polygon": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
4,
|
||||
0
|
||||
],
|
||||
[
|
||||
4,
|
||||
4
|
||||
],
|
||||
[
|
||||
0,
|
||||
4
|
||||
],
|
||||
[
|
||||
0,
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "r2",
|
||||
"class": "building",
|
||||
"polygon": [
|
||||
[
|
||||
10,
|
||||
0
|
||||
],
|
||||
[
|
||||
14,
|
||||
0
|
||||
],
|
||||
[
|
||||
14,
|
||||
4
|
||||
],
|
||||
[
|
||||
10,
|
||||
4
|
||||
],
|
||||
[
|
||||
10,
|
||||
0
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"predictions": [
|
||||
{
|
||||
"id": "p1",
|
||||
"class": "building",
|
||||
"polygon": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
4,
|
||||
0
|
||||
],
|
||||
[
|
||||
4,
|
||||
4
|
||||
],
|
||||
[
|
||||
0,
|
||||
4
|
||||
],
|
||||
[
|
||||
0,
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p2",
|
||||
"class": "building",
|
||||
"polygon": [
|
||||
[
|
||||
20,
|
||||
0
|
||||
],
|
||||
[
|
||||
24,
|
||||
0
|
||||
],
|
||||
[
|
||||
24,
|
||||
4
|
||||
],
|
||||
[
|
||||
20,
|
||||
4
|
||||
],
|
||||
[
|
||||
20,
|
||||
0
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"sample_id": "change-test-wallonia-industrial",
|
||||
"split": "test",
|
||||
"task": "change_detection",
|
||||
"metadata": {
|
||||
"region": "wallonia",
|
||||
"municipality": "Liège",
|
||||
"urbanity": "industrial",
|
||||
"object_size": "mixed",
|
||||
"source": "fixture-picc-temporal",
|
||||
"sensor": "vector",
|
||||
"resolution_m": 0.5,
|
||||
"season": "multi-date",
|
||||
"occlusion": "none",
|
||||
"vegetation": "low",
|
||||
"difficulty": "hard",
|
||||
"context": "industrial_change",
|
||||
"tile_edge": false,
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"ood": false,
|
||||
"date": "2024-05-01/2025-05-01"
|
||||
},
|
||||
"config": {
|
||||
"match_iou": 0.5
|
||||
},
|
||||
"lineage": {
|
||||
"reference": {
|
||||
"source_id": "synthetic:change-test-wallonia-industrial:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture"
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": "synthetic:change-test-wallonia-industrial:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output"
|
||||
}
|
||||
},
|
||||
"classes": [
|
||||
"added",
|
||||
"removed"
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"id": "r1",
|
||||
"class": "added",
|
||||
"bbox": [
|
||||
0,
|
||||
0,
|
||||
5,
|
||||
5
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "r2",
|
||||
"class": "removed",
|
||||
"bbox": [
|
||||
10,
|
||||
0,
|
||||
15,
|
||||
5
|
||||
]
|
||||
}
|
||||
],
|
||||
"predictions": [
|
||||
{
|
||||
"id": "p1",
|
||||
"class": "added",
|
||||
"bbox": [
|
||||
0,
|
||||
0,
|
||||
5,
|
||||
5
|
||||
],
|
||||
"confidence": 0.85
|
||||
},
|
||||
{
|
||||
"id": "p2",
|
||||
"class": "removed",
|
||||
"bbox": [
|
||||
20,
|
||||
0,
|
||||
25,
|
||||
5
|
||||
],
|
||||
"confidence": 0.65
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"sample_id": "terrain-test-flanders-rural",
|
||||
"split": "test",
|
||||
"task": "terrain_interpretation",
|
||||
"metadata": {
|
||||
"region": "flanders",
|
||||
"municipality": "Hasselt",
|
||||
"urbanity": "rural",
|
||||
"object_size": "not_applicable",
|
||||
"source": "fixture-dhmv",
|
||||
"sensor": "elevation-raster",
|
||||
"resolution_m": 5,
|
||||
"season": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"vegetation": "moderate",
|
||||
"difficulty": "normal",
|
||||
"context": "terrain_profile",
|
||||
"tile_edge": false,
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"ood": false,
|
||||
"date": "2023-01-01"
|
||||
},
|
||||
"config": {
|
||||
"missing_prediction_policy": "exclude_and_reduce_coverage"
|
||||
},
|
||||
"lineage": {
|
||||
"reference": {
|
||||
"source_id": "synthetic:terrain-test-flanders-rural:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture"
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": "synthetic:terrain-test-flanders-rural:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output"
|
||||
}
|
||||
},
|
||||
"units": "m_TAW",
|
||||
"references": [
|
||||
12,
|
||||
13,
|
||||
14,
|
||||
15,
|
||||
16
|
||||
],
|
||||
"predictions": [
|
||||
12.2,
|
||||
12.8,
|
||||
14.4,
|
||||
null,
|
||||
15.7
|
||||
]
|
||||
},
|
||||
{
|
||||
"sample_id": "validation-test-national",
|
||||
"split": "test",
|
||||
"task": "geospatial_data_validation",
|
||||
"metadata": {
|
||||
"region": "national",
|
||||
"municipality": "not_applicable",
|
||||
"urbanity": "mixed",
|
||||
"object_size": "not_applicable",
|
||||
"source": "fixture-contracts",
|
||||
"sensor": "mixed",
|
||||
"resolution_m": null,
|
||||
"season": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"vegetation": "mixed",
|
||||
"difficulty": "hard",
|
||||
"context": "data_contract",
|
||||
"tile_edge": false,
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"ood": false,
|
||||
"date": "2025-01-01"
|
||||
},
|
||||
"config": {
|
||||
"critical_severities": [
|
||||
"blocker",
|
||||
"critical"
|
||||
]
|
||||
},
|
||||
"lineage": {
|
||||
"reference": {
|
||||
"source_id": "synthetic:validation-test-national:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture"
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": "synthetic:validation-test-national:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output"
|
||||
}
|
||||
},
|
||||
"expected_anomalies": [
|
||||
{
|
||||
"code": "D-CRS-WRONG",
|
||||
"severity": "blocker"
|
||||
},
|
||||
{
|
||||
"code": "D-INVALID-GEOMETRY",
|
||||
"severity": "critical"
|
||||
},
|
||||
{
|
||||
"code": "D-MISSING-PROVENANCE",
|
||||
"severity": "critical"
|
||||
}
|
||||
],
|
||||
"observed_anomalies": [
|
||||
{
|
||||
"code": "D-CRS-WRONG",
|
||||
"severity": "blocker"
|
||||
},
|
||||
{
|
||||
"code": "D-INVALID-GEOMETRY",
|
||||
"severity": "critical"
|
||||
},
|
||||
{
|
||||
"code": "D-RESOLUTION",
|
||||
"severity": "major"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"sample_id": "background-test-pure-empty",
|
||||
"split": "background-test",
|
||||
"task": "object_detection",
|
||||
"metadata": {
|
||||
"region": "brussels",
|
||||
"municipality": "Brussel",
|
||||
"urbanity": "urban",
|
||||
"object_size": "empty",
|
||||
"source": "fixture-orthophoto",
|
||||
"sensor": "synthetic-rgb",
|
||||
"resolution_m": 0.25,
|
||||
"season": "winter",
|
||||
"occlusion": "none",
|
||||
"vegetation": "low",
|
||||
"difficulty": "hard",
|
||||
"context": "pure_background",
|
||||
"tile_edge": false,
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"ood": false,
|
||||
"date": "2025-02-01"
|
||||
},
|
||||
"config": {
|
||||
"confidence_threshold": 0.5,
|
||||
"match_iou": 0.5,
|
||||
"fixed_diagnostic_risk_thresholds": [
|
||||
0,
|
||||
0.5,
|
||||
0.7,
|
||||
0.9
|
||||
]
|
||||
},
|
||||
"lineage": {
|
||||
"reference": {
|
||||
"source_id": "synthetic:background-test-pure-empty:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture"
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": "synthetic:background-test-pure-empty:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output"
|
||||
}
|
||||
},
|
||||
"classes": [
|
||||
"building"
|
||||
],
|
||||
"references": [],
|
||||
"predictions": []
|
||||
},
|
||||
{
|
||||
"sample_id": "background-test-hard-negative",
|
||||
"split": "background-test",
|
||||
"task": "object_detection",
|
||||
"metadata": {
|
||||
"region": "flanders",
|
||||
"municipality": "Oostende",
|
||||
"urbanity": "urban",
|
||||
"object_size": "empty",
|
||||
"source": "fixture-orthophoto",
|
||||
"sensor": "synthetic-rgb",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"occlusion": "none",
|
||||
"vegetation": "moderate",
|
||||
"difficulty": "hard",
|
||||
"context": "coastal_hard_negative",
|
||||
"tile_edge": true,
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"ood": false,
|
||||
"date": "2025-07-01"
|
||||
},
|
||||
"config": {
|
||||
"confidence_threshold": 0.5,
|
||||
"match_iou": 0.5,
|
||||
"fixed_diagnostic_risk_thresholds": [
|
||||
0,
|
||||
0.5,
|
||||
0.7,
|
||||
0.9
|
||||
]
|
||||
},
|
||||
"lineage": {
|
||||
"reference": {
|
||||
"source_id": "synthetic:background-test-hard-negative:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture"
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": "synthetic:background-test-hard-negative:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output"
|
||||
}
|
||||
},
|
||||
"classes": [
|
||||
"building"
|
||||
],
|
||||
"references": [],
|
||||
"predictions": [
|
||||
{
|
||||
"id": "p-hard-fp",
|
||||
"class": "building",
|
||||
"bbox": [
|
||||
20,
|
||||
20,
|
||||
35,
|
||||
35
|
||||
],
|
||||
"confidence": 0.92
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"dataset_version": "geointel-p4-harness-fixture-v2",
|
||||
"crs": "EPSG:31370",
|
||||
"independence_buffer_m": 2000.0,
|
||||
"perceptual_hamming_threshold": 4,
|
||||
"label_geometry_hamming_threshold": 2,
|
||||
"required_splits": ["train", "val", "calibration", "test", "background-test", "challenge"],
|
||||
"assignment_mode": "preassigned",
|
||||
"claim_boundary": "Synthetic contract fixtures for evaluator regression only; never production accuracy evidence.",
|
||||
"samples": [
|
||||
{"sample_id":"det-train-a","task":"object_detection","split":"train","group_id":"g01","source_family":"scene-01","temporal_family":"temporal-01","object_ids":["o01"],"bbox":[10000,10000,10100,10100],"raw_image_sha256":"0101010101010101010101010101010101010101010101010101010101010101","processed_image_sha256":"2121212121212121212121212121212121212121212121212121212121212121","label_sha256":"4141414141414141414141414141414141414141414141414141414141414141","perceptual_image_hash":"dcef24412401d4f9","label_geometry_hash":"6161616161616161616161616161616161616161616161616161616161616161","label_geometry_fingerprint":"c0f62e16b4033ee9","native_feature_ids":["native-01-1"],"parent_raster_id":"parent-raster-01","acquisition_id":"acquisition-01","acquisition_date":"2025-01-01"},
|
||||
{"sample_id":"seg-train-a","task":"footprint_segmentation","split":"train","group_id":"g02","source_family":"scene-02","temporal_family":"temporal-02","object_ids":["o02"],"bbox":[20000,10000,20100,10100],"raw_image_sha256":"0202020202020202020202020202020202020202020202020202020202020202","processed_image_sha256":"2222222222222222222222222222222222222222222222222222222222222222","label_sha256":"4242424242424242424242424242424242424242424242424242424242424242","perceptual_image_hash":"030bcdd3bc87412e","label_geometry_hash":"6262626262626262626262626262626262626262626262626262626262626262","label_geometry_fingerprint":"666a0fc2d26265ee","native_feature_ids":["native-02-1"],"parent_raster_id":"parent-raster-02","acquisition_id":"acquisition-02","acquisition_date":"2025-01-02"},
|
||||
{"sample_id":"raster-train-a","task":"raster_classification","split":"train","group_id":"g03","source_family":"scene-03","temporal_family":"temporal-03","object_ids":["o03"],"bbox":[30000,10000,30100,10100],"raw_image_sha256":"0303030303030303030303030303030303030303030303030303030303030303","processed_image_sha256":"2323232323232323232323232323232323232323232323232323232323232323","label_sha256":"4343434343434343434343434343434343434343434343434343434343434343","perceptual_image_hash":"75e36e008b4a6702","label_geometry_hash":"6363636363636363636363636363636363636363636363636363636363636363","label_geometry_fingerprint":"30820f2c124a3ff0","native_feature_ids":["native-03-1"],"parent_raster_id":"parent-raster-03","acquisition_id":"acquisition-03","acquisition_date":"2025-01-03"},
|
||||
{"sample_id":"val-vector-a","task":"vector_comparison","split":"val","group_id":"g04","source_family":"scene-04","temporal_family":"temporal-04","object_ids":["o04"],"bbox":[10000,30000,10100,30100],"raw_image_sha256":"0404040404040404040404040404040404040404040404040404040404040404","processed_image_sha256":"2424242424242424242424242424242424242424242424242424242424242424","label_sha256":"4444444444444444444444444444444444444444444444444444444444444444","perceptual_image_hash":"2e3b331cc01536c9","label_geometry_hash":"6464646464646464646464646464646464646464646464646464646464646464","label_geometry_fingerprint":"a559b46cba64e501","native_feature_ids":["native-04-1"],"parent_raster_id":"parent-raster-04","acquisition_id":"acquisition-04","acquisition_date":"2025-01-04"},
|
||||
{"sample_id":"val-change-b","task":"change_detection","split":"val","group_id":"g05","source_family":"scene-05","temporal_family":"temporal-05","object_ids":["o05"],"bbox":[30000,30000,30100,30100],"raw_image_sha256":"0505050505050505050505050505050505050505050505050505050505050505","processed_image_sha256":"2525252525252525252525252525252525252525252525252525252525252525","label_sha256":"4545454545454545454545454545454545454545454545454545454545454545","perceptual_image_hash":"828b99671c3f2f39","label_geometry_hash":"6565656565656565656565656565656565656565656565656565656565656565","label_geometry_fingerprint":"14ceff3640a013eb","native_feature_ids":["native-05-1"],"parent_raster_id":"parent-raster-05","acquisition_id":"acquisition-05","acquisition_date":"2025-01-05"},
|
||||
{"sample_id":"val-terrain-c","task":"terrain_interpretation","split":"val","group_id":"g06","source_family":"scene-06","temporal_family":"temporal-06","object_ids":["o06"],"bbox":[50000,30000,50100,30100],"raw_image_sha256":"0606060606060606060606060606060606060606060606060606060606060606","processed_image_sha256":"2626262626262626262626262626262626262626262626262626262626262626","label_sha256":"4646464646464646464646464646464646464646464646464646464646464646","perceptual_image_hash":"11c11105b0d10635","label_geometry_hash":"6666666666666666666666666666666666666666666666666666666666666666","label_geometry_fingerprint":"a1d9bbb93ba6d6da","native_feature_ids":["native-06-1"],"parent_raster_id":"parent-raster-06","acquisition_id":"acquisition-06","acquisition_date":"2025-01-06"},
|
||||
{"sample_id":"calibration-det-a","task":"object_detection","split":"calibration","group_id":"g07","source_family":"scene-07","temporal_family":"temporal-07","object_ids":["o21"],"bbox":[70000,30000,70100,30100],"raw_image_sha256":"0707070707070707070707070707070707070707070707070707070707070707","processed_image_sha256":"2727272727272727272727272727272727272727272727272727272727272727","label_sha256":"4747474747474747474747474747474747474747474747474747474747474747","perceptual_image_hash":"c4316514fed98f6b","label_geometry_hash":"6767676767676767676767676767676767676767676767676767676767676767","label_geometry_fingerprint":"dba609f34cb9826b","native_feature_ids":["native-07-1"],"parent_raster_id":"parent-raster-07","acquisition_id":"acquisition-07","acquisition_date":"2025-01-07"},
|
||||
{"sample_id":"calibration-seg-b","task":"footprint_segmentation","split":"calibration","group_id":"g08","source_family":"scene-08","temporal_family":"temporal-08","object_ids":["o22"],"bbox":[90000,30000,90100,30100],"raw_image_sha256":"0808080808080808080808080808080808080808080808080808080808080808","processed_image_sha256":"2828282828282828282828282828282828282828282828282828282828282828","label_sha256":"4848484848484848484848484848484848484848484848484848484848484848","perceptual_image_hash":"518ba3145333c08c","label_geometry_hash":"6868686868686868686868686868686868686868686868686868686868686868","label_geometry_fingerprint":"627df87eea48aec0","native_feature_ids":["native-08-1"],"parent_raster_id":"parent-raster-08","acquisition_id":"acquisition-08","acquisition_date":"2025-01-08"},
|
||||
{"sample_id":"det-test-flanders-urban","task":"object_detection","split":"test","group_id":"g09","source_family":"scene-09","temporal_family":"temporal-09","object_ids":["o07","o08"],"bbox":[10000,50000,10100,50100],"raw_image_sha256":"0909090909090909090909090909090909090909090909090909090909090909","processed_image_sha256":"2929292929292929292929292929292929292929292929292929292929292929","label_sha256":"4949494949494949494949494949494949494949494949494949494949494949","perceptual_image_hash":"d6d20ed34050b89e","label_geometry_hash":"6969696969696969696969696969696969696969696969696969696969696969","label_geometry_fingerprint":"7ed221e11cc6828d","native_feature_ids":["native-09-1","native-09-2"],"parent_raster_id":"parent-raster-09","acquisition_id":"acquisition-09","acquisition_date":"2025-01-09"},
|
||||
{"sample_id":"seg-test-wallonia-rural","task":"footprint_segmentation","split":"test","group_id":"g10","source_family":"scene-10","temporal_family":"temporal-10","object_ids":["o09"],"bbox":[20000,50000,20100,50100],"raw_image_sha256":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","processed_image_sha256":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","label_sha256":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","perceptual_image_hash":"6c476d18e0539687","label_geometry_hash":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","label_geometry_fingerprint":"e9fb3c93de82d0be","native_feature_ids":["native-10-1"],"parent_raster_id":"parent-raster-10","acquisition_id":"acquisition-10","acquisition_date":"2025-01-10"},
|
||||
{"sample_id":"raster-test-brussels-urban","task":"raster_classification","split":"test","group_id":"g11","source_family":"scene-11","temporal_family":"temporal-11","object_ids":["o10"],"bbox":[30000,50000,30100,50100],"raw_image_sha256":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","processed_image_sha256":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","label_sha256":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","perceptual_image_hash":"4383fa06c328f326","label_geometry_hash":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","label_geometry_fingerprint":"f4269ac17b100571","native_feature_ids":["native-11-1"],"parent_raster_id":"parent-raster-11","acquisition_id":"acquisition-11","acquisition_date":"2025-01-11"},
|
||||
{"sample_id":"vector-test-flanders-suburban","task":"vector_comparison","split":"test","group_id":"g12","source_family":"scene-12","temporal_family":"temporal-12","object_ids":["o11","o12"],"bbox":[40000,50000,40100,50100],"raw_image_sha256":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","processed_image_sha256":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","label_sha256":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","perceptual_image_hash":"1236e1aa148c25d5","label_geometry_hash":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","label_geometry_fingerprint":"2db7d069691934ed","native_feature_ids":["native-12-1","native-12-2"],"parent_raster_id":"parent-raster-12","acquisition_id":"acquisition-12","acquisition_date":"2025-01-12"},
|
||||
{"sample_id":"change-test-wallonia-industrial","task":"change_detection","split":"test","group_id":"g13","source_family":"scene-13","temporal_family":"temporal-13","object_ids":["o13","o14"],"bbox":[50000,50000,50100,50100],"raw_image_sha256":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","processed_image_sha256":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","label_sha256":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","perceptual_image_hash":"14e347040ef16eac","label_geometry_hash":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","label_geometry_fingerprint":"6c5b5f9db0d73c31","native_feature_ids":["native-13-1","native-13-2"],"parent_raster_id":"parent-raster-13","acquisition_id":"acquisition-13","acquisition_date":"2025-01-13"},
|
||||
{"sample_id":"terrain-test-flanders-rural","task":"terrain_interpretation","split":"test","group_id":"g14","source_family":"scene-14","temporal_family":"temporal-14","object_ids":["o15"],"bbox":[60000,50000,60100,50100],"raw_image_sha256":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","processed_image_sha256":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","label_sha256":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","perceptual_image_hash":"c808940b0a325e9c","label_geometry_hash":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","label_geometry_fingerprint":"4ec9f1dd6c799ec7","native_feature_ids":["native-14-1"],"parent_raster_id":"parent-raster-14","acquisition_id":"acquisition-14","acquisition_date":"2025-01-14"},
|
||||
{"sample_id":"validation-test-national","task":"geospatial_data_validation","split":"test","group_id":"g15","source_family":"scene-15","temporal_family":"temporal-15","object_ids":["o16"],"bbox":[70000,50000,70100,50100],"raw_image_sha256":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","processed_image_sha256":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","label_sha256":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","perceptual_image_hash":"4365e6b7df6acf3d","label_geometry_hash":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","label_geometry_fingerprint":"616691796f62dbbb","native_feature_ids":["native-15-1"],"parent_raster_id":"parent-raster-15","acquisition_id":"acquisition-15","acquisition_date":"2025-01-15"},
|
||||
{"sample_id":"background-test-pure-empty","task":"object_detection","split":"background-test","group_id":"g16","source_family":"scene-16","temporal_family":"temporal-16","object_ids":[],"bbox":[20000,70000,20100,70100],"raw_image_sha256":"1010101010101010101010101010101010101010101010101010101010101010","processed_image_sha256":"3030303030303030303030303030303030303030303030303030303030303030","label_sha256":"5050505050505050505050505050505050505050505050505050505050505050","perceptual_image_hash":"be4078e1ef9fa296","label_geometry_hash":"7070707070707070707070707070707070707070707070707070707070707070","label_geometry_fingerprint":"004e762f898c4fad","native_feature_ids":[],"parent_raster_id":"parent-raster-16","acquisition_id":"acquisition-16","acquisition_date":"2025-01-16"},
|
||||
{"sample_id":"background-test-hard-negative","task":"object_detection","split":"background-test","group_id":"g17","source_family":"scene-17","temporal_family":"temporal-17","object_ids":[],"bbox":[60000,70000,60100,70100],"raw_image_sha256":"1111111111111111111111111111111111111111111111111111111111111111","processed_image_sha256":"3131313131313131313131313131313131313131313131313131313131313131","label_sha256":"5151515151515151515151515151515151515151515151515151515151515151","perceptual_image_hash":"6ada1b5aa46cc261","label_geometry_hash":"7171717171717171717171717171717171717171717171717171717171717171","label_geometry_fingerprint":"2b405bcc9c95acfb","native_feature_ids":[],"parent_raster_id":"parent-raster-17","acquisition_id":"acquisition-17","acquisition_date":"2025-01-17"},
|
||||
{"sample_id":"det-challenge-coast","task":"object_detection","split":"challenge","group_id":"g18","source_family":"scene-18","temporal_family":"temporal-18","object_ids":["o17"],"bbox":[10000,90000,10100,90100],"raw_image_sha256":"1212121212121212121212121212121212121212121212121212121212121212","processed_image_sha256":"3232323232323232323232323232323232323232323232323232323232323232","label_sha256":"5252525252525252525252525252525252525252525252525252525252525252","perceptual_image_hash":"15910b3a2d056648","label_geometry_hash":"7272727272727272727272727272727272727272727272727272727272727272","label_geometry_fingerprint":"95518e2583b3a7d3","native_feature_ids":["native-18-1"],"parent_raster_id":"parent-raster-18","acquisition_id":"acquisition-18","acquisition_date":"2025-01-18"},
|
||||
{"sample_id":"seg-challenge-brussels","task":"footprint_segmentation","split":"challenge","group_id":"g19","source_family":"scene-19","temporal_family":"temporal-19","object_ids":["o18"],"bbox":[30000,90000,30100,90100],"raw_image_sha256":"1313131313131313131313131313131313131313131313131313131313131313","processed_image_sha256":"3333333333333333333333333333333333333333333333333333333333333333","label_sha256":"5353535353535353535353535353535353535353535353535353535353535353","perceptual_image_hash":"b5c6e201d687cf9e","label_geometry_hash":"7373737373737373737373737373737373737373737373737373737373737373","label_geometry_fingerprint":"2625fe765bce389e","native_feature_ids":["native-19-1"],"parent_raster_id":"parent-raster-19","acquisition_id":"acquisition-19","acquisition_date":"2025-01-19"},
|
||||
{"sample_id":"raster-challenge-seasonal","task":"raster_classification","split":"challenge","group_id":"g20","source_family":"scene-20","temporal_family":"temporal-20","object_ids":["o19"],"bbox":[50000,90000,50100,90100],"raw_image_sha256":"1414141414141414141414141414141414141414141414141414141414141414","processed_image_sha256":"3434343434343434343434343434343434343434343434343434343434343434","label_sha256":"5454545454545454545454545454545454545454545454545454545454545454","perceptual_image_hash":"85f82f2f8eb2c2f5","label_geometry_hash":"7474747474747474747474747474747474747474747474747474747474747474","label_geometry_fingerprint":"e91bcaae1021dccc","native_feature_ids":["native-20-1"],"parent_raster_id":"parent-raster-20","acquisition_id":"acquisition-20","acquisition_date":"2025-01-20"},
|
||||
{"sample_id":"validation-challenge-corrupt","task":"geospatial_data_validation","split":"challenge","group_id":"g21","source_family":"scene-21","temporal_family":"temporal-21","object_ids":["o20"],"bbox":[70000,90000,70100,90100],"raw_image_sha256":"1515151515151515151515151515151515151515151515151515151515151515","processed_image_sha256":"3535353535353535353535353535353535353535353535353535353535353535","label_sha256":"5555555555555555555555555555555555555555555555555555555555555555","perceptual_image_hash":"c71055ef6baeee8b","label_geometry_hash":"7575757575757575757575757575757575757575757575757575757575757575","label_geometry_fingerprint":"686f1d8a92703709","native_feature_ids":["native-21-1"],"parent_raster_id":"parent-raster-21","acquisition_id":"acquisition-21","acquisition_date":"2025-01-21"}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user