883 lines
20 KiB
JSON
883 lines
20 KiB
JSON
{
|
|
"schema_version": 2,
|
|
"portfolio_kind": "synthetic_contract",
|
|
"portfolio_id": "geointel-p4-reference-harness-v2",
|
|
"split_roles": [
|
|
"test",
|
|
"background-test"
|
|
],
|
|
"selection_policy": "No data-dependent threshold, model or gate selection is performed from protected cases.",
|
|
"protected_policy": {
|
|
"operating_point_selection_allowed": false,
|
|
"diagnostic_curves_select_operating_point": false,
|
|
"test_feedback_allowed": false,
|
|
"threshold_selection_source": "pre_registered_configuration_only"
|
|
},
|
|
"portfolio_lineage": {
|
|
"origin": "repository_fixture",
|
|
"source_path": "fixtures/accuracy/p4/protected-baseline-cases.json",
|
|
"version": "1"
|
|
},
|
|
"claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.",
|
|
"cases": [
|
|
{
|
|
"sample_id": "det-test-flanders-urban",
|
|
"split": "test",
|
|
"task": "object_detection",
|
|
"metadata": {
|
|
"region": "flanders",
|
|
"municipality": "Antwerpen",
|
|
"urbanity": "urban",
|
|
"object_size": "mixed",
|
|
"source": "fixture-orthophoto",
|
|
"sensor": "synthetic-rgb",
|
|
"resolution_m": 0.25,
|
|
"season": "summer",
|
|
"occlusion": "partial",
|
|
"vegetation": "moderate",
|
|
"difficulty": "hard",
|
|
"context": "dense_urban",
|
|
"tile_edge": false,
|
|
"label_review_state": "fixture_reviewed",
|
|
"ood": false,
|
|
"date": "2025-06-15"
|
|
},
|
|
"config": {
|
|
"confidence_threshold": 0.5,
|
|
"match_iou": 0.5,
|
|
"fixed_diagnostic_risk_thresholds": [
|
|
0,
|
|
0.5,
|
|
0.7,
|
|
0.9
|
|
]
|
|
},
|
|
"lineage": {
|
|
"reference": {
|
|
"source_id": "synthetic:det-test-flanders-urban:reference",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_contract_fixture"
|
|
},
|
|
"prediction": {
|
|
"source_id": "synthetic:det-test-flanders-urban:prediction",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_fixed_output"
|
|
}
|
|
},
|
|
"classes": [
|
|
"building"
|
|
],
|
|
"references": [
|
|
{
|
|
"id": "r1",
|
|
"class": "building",
|
|
"bbox": [
|
|
10,
|
|
10,
|
|
30,
|
|
30
|
|
]
|
|
},
|
|
{
|
|
"id": "r2",
|
|
"class": "building",
|
|
"bbox": [
|
|
50,
|
|
50,
|
|
80,
|
|
80
|
|
]
|
|
}
|
|
],
|
|
"predictions": [
|
|
{
|
|
"id": "p1",
|
|
"class": "building",
|
|
"bbox": [
|
|
10,
|
|
10,
|
|
30,
|
|
30
|
|
],
|
|
"confidence": 0.9
|
|
},
|
|
{
|
|
"id": "p2",
|
|
"class": "building",
|
|
"bbox": [
|
|
52,
|
|
52,
|
|
80,
|
|
80
|
|
],
|
|
"confidence": 0.7
|
|
},
|
|
{
|
|
"id": "p3",
|
|
"class": "building",
|
|
"bbox": [
|
|
85,
|
|
85,
|
|
95,
|
|
95
|
|
],
|
|
"confidence": 0.6
|
|
},
|
|
{
|
|
"id": "p4-filtered",
|
|
"class": "building",
|
|
"bbox": [
|
|
2,
|
|
80,
|
|
7,
|
|
86
|
|
],
|
|
"confidence": 0.2
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"sample_id": "seg-test-wallonia-rural",
|
|
"split": "test",
|
|
"task": "footprint_segmentation",
|
|
"metadata": {
|
|
"region": "wallonia",
|
|
"municipality": "Namur",
|
|
"urbanity": "rural",
|
|
"object_size": "large",
|
|
"source": "fixture-picc",
|
|
"sensor": "synthetic-rgb",
|
|
"resolution_m": 0.5,
|
|
"season": "spring",
|
|
"occlusion": "none",
|
|
"vegetation": "low",
|
|
"difficulty": "normal",
|
|
"context": "rural_buildings",
|
|
"tile_edge": false,
|
|
"label_review_state": "fixture_reviewed",
|
|
"ood": false,
|
|
"date": "2025-04-20"
|
|
},
|
|
"config": {
|
|
"match_iou": 0.5,
|
|
"boundary_tolerance_m": 1
|
|
},
|
|
"lineage": {
|
|
"reference": {
|
|
"source_id": "synthetic:seg-test-wallonia-rural:reference",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_contract_fixture"
|
|
},
|
|
"prediction": {
|
|
"source_id": "synthetic:seg-test-wallonia-rural:prediction",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_fixed_output"
|
|
}
|
|
},
|
|
"spatial_context": {
|
|
"crs": "EPSG:31370",
|
|
"coordinate_units": "m",
|
|
"metric": true
|
|
},
|
|
"classes": [
|
|
"building"
|
|
],
|
|
"references": [
|
|
{
|
|
"id": "r1",
|
|
"class": "building",
|
|
"polygon": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
10,
|
|
0
|
|
],
|
|
[
|
|
10,
|
|
8
|
|
],
|
|
[
|
|
0,
|
|
8
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"predictions": [
|
|
{
|
|
"id": "p1",
|
|
"class": "building",
|
|
"polygon": [
|
|
[
|
|
0.5,
|
|
0.5
|
|
],
|
|
[
|
|
9.5,
|
|
0.5
|
|
],
|
|
[
|
|
9.5,
|
|
8.5
|
|
],
|
|
[
|
|
0.5,
|
|
8.5
|
|
],
|
|
[
|
|
0.5,
|
|
0.5
|
|
]
|
|
],
|
|
"confidence": 0.8
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"sample_id": "raster-test-brussels-urban",
|
|
"split": "test",
|
|
"task": "raster_classification",
|
|
"metadata": {
|
|
"region": "brussels",
|
|
"municipality": "Brussel",
|
|
"urbanity": "urban",
|
|
"object_size": "not_applicable",
|
|
"source": "fixture-thematic-raster",
|
|
"sensor": "synthetic-multispectral",
|
|
"resolution_m": 1,
|
|
"season": "autumn",
|
|
"occlusion": "not_applicable",
|
|
"vegetation": "moderate",
|
|
"difficulty": "hard",
|
|
"context": "dense_urban_raster",
|
|
"tile_edge": true,
|
|
"label_review_state": "fixture_reviewed",
|
|
"ood": false,
|
|
"date": "2025-10-05"
|
|
},
|
|
"config": {
|
|
"class_order": [
|
|
0,
|
|
1,
|
|
2
|
|
],
|
|
"masked_pixels_excluded": true
|
|
},
|
|
"lineage": {
|
|
"reference": {
|
|
"source_id": "synthetic:raster-test-brussels-urban:reference",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_contract_fixture"
|
|
},
|
|
"prediction": {
|
|
"source_id": "synthetic:raster-test-brussels-urban:prediction",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_fixed_output"
|
|
}
|
|
},
|
|
"classes": [
|
|
0,
|
|
1,
|
|
2
|
|
],
|
|
"raster_context": {
|
|
"reference": {
|
|
"crs": "EPSG:31370",
|
|
"transform": [
|
|
1,
|
|
0,
|
|
0,
|
|
0,
|
|
-1,
|
|
3
|
|
],
|
|
"shape": [
|
|
3,
|
|
4
|
|
],
|
|
"nodata": null,
|
|
"mask": [
|
|
[
|
|
true,
|
|
true,
|
|
true,
|
|
true
|
|
],
|
|
[
|
|
true,
|
|
true,
|
|
true,
|
|
true
|
|
],
|
|
[
|
|
true,
|
|
true,
|
|
true,
|
|
true
|
|
]
|
|
]
|
|
},
|
|
"prediction": {
|
|
"crs": "EPSG:31370",
|
|
"transform": [
|
|
1,
|
|
0,
|
|
0,
|
|
0,
|
|
-1,
|
|
3
|
|
],
|
|
"shape": [
|
|
3,
|
|
4
|
|
],
|
|
"nodata": null,
|
|
"mask": [
|
|
[
|
|
true,
|
|
true,
|
|
true,
|
|
true
|
|
],
|
|
[
|
|
true,
|
|
true,
|
|
true,
|
|
true
|
|
],
|
|
[
|
|
true,
|
|
true,
|
|
true,
|
|
true
|
|
]
|
|
]
|
|
}
|
|
},
|
|
"references": [
|
|
[
|
|
0,
|
|
0,
|
|
1,
|
|
1
|
|
],
|
|
[
|
|
0,
|
|
1,
|
|
1,
|
|
2
|
|
],
|
|
[
|
|
2,
|
|
2,
|
|
1,
|
|
0
|
|
]
|
|
],
|
|
"predictions": [
|
|
[
|
|
0,
|
|
0,
|
|
1,
|
|
2
|
|
],
|
|
[
|
|
0,
|
|
1,
|
|
1,
|
|
2
|
|
],
|
|
[
|
|
2,
|
|
1,
|
|
1,
|
|
0
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"sample_id": "vector-test-flanders-suburban",
|
|
"split": "test",
|
|
"task": "vector_comparison",
|
|
"metadata": {
|
|
"region": "flanders",
|
|
"municipality": "Mol",
|
|
"urbanity": "suburban",
|
|
"object_size": "mixed",
|
|
"source": "fixture-grb",
|
|
"sensor": "vector",
|
|
"resolution_m": 0.1,
|
|
"season": "not_applicable",
|
|
"occlusion": "not_applicable",
|
|
"vegetation": "low",
|
|
"difficulty": "normal",
|
|
"context": "suburban_buildings",
|
|
"tile_edge": false,
|
|
"label_review_state": "fixture_reviewed",
|
|
"ood": false,
|
|
"date": "2025-01-15"
|
|
},
|
|
"config": {
|
|
"match_iou": 0.5
|
|
},
|
|
"lineage": {
|
|
"reference": {
|
|
"source_id": "synthetic:vector-test-flanders-suburban:reference",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_contract_fixture"
|
|
},
|
|
"prediction": {
|
|
"source_id": "synthetic:vector-test-flanders-suburban:prediction",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_fixed_output"
|
|
}
|
|
},
|
|
"spatial_context": {
|
|
"crs": "EPSG:31370",
|
|
"coordinate_units": "m",
|
|
"metric": true
|
|
},
|
|
"classes": [
|
|
"building"
|
|
],
|
|
"references": [
|
|
{
|
|
"id": "r1",
|
|
"class": "building",
|
|
"polygon": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
4,
|
|
0
|
|
],
|
|
[
|
|
4,
|
|
4
|
|
],
|
|
[
|
|
0,
|
|
4
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "r2",
|
|
"class": "building",
|
|
"polygon": [
|
|
[
|
|
10,
|
|
0
|
|
],
|
|
[
|
|
14,
|
|
0
|
|
],
|
|
[
|
|
14,
|
|
4
|
|
],
|
|
[
|
|
10,
|
|
4
|
|
],
|
|
[
|
|
10,
|
|
0
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"predictions": [
|
|
{
|
|
"id": "p1",
|
|
"class": "building",
|
|
"polygon": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
4,
|
|
0
|
|
],
|
|
[
|
|
4,
|
|
4
|
|
],
|
|
[
|
|
0,
|
|
4
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "p2",
|
|
"class": "building",
|
|
"polygon": [
|
|
[
|
|
20,
|
|
0
|
|
],
|
|
[
|
|
24,
|
|
0
|
|
],
|
|
[
|
|
24,
|
|
4
|
|
],
|
|
[
|
|
20,
|
|
4
|
|
],
|
|
[
|
|
20,
|
|
0
|
|
]
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"sample_id": "change-test-wallonia-industrial",
|
|
"split": "test",
|
|
"task": "change_detection",
|
|
"metadata": {
|
|
"region": "wallonia",
|
|
"municipality": "Liège",
|
|
"urbanity": "industrial",
|
|
"object_size": "mixed",
|
|
"source": "fixture-picc-temporal",
|
|
"sensor": "vector",
|
|
"resolution_m": 0.5,
|
|
"season": "multi-date",
|
|
"occlusion": "none",
|
|
"vegetation": "low",
|
|
"difficulty": "hard",
|
|
"context": "industrial_change",
|
|
"tile_edge": false,
|
|
"label_review_state": "fixture_reviewed",
|
|
"ood": false,
|
|
"date": "2024-05-01/2025-05-01"
|
|
},
|
|
"config": {
|
|
"match_iou": 0.5
|
|
},
|
|
"lineage": {
|
|
"reference": {
|
|
"source_id": "synthetic:change-test-wallonia-industrial:reference",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_contract_fixture"
|
|
},
|
|
"prediction": {
|
|
"source_id": "synthetic:change-test-wallonia-industrial:prediction",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_fixed_output"
|
|
}
|
|
},
|
|
"classes": [
|
|
"added",
|
|
"removed"
|
|
],
|
|
"references": [
|
|
{
|
|
"id": "r1",
|
|
"class": "added",
|
|
"bbox": [
|
|
0,
|
|
0,
|
|
5,
|
|
5
|
|
]
|
|
},
|
|
{
|
|
"id": "r2",
|
|
"class": "removed",
|
|
"bbox": [
|
|
10,
|
|
0,
|
|
15,
|
|
5
|
|
]
|
|
}
|
|
],
|
|
"predictions": [
|
|
{
|
|
"id": "p1",
|
|
"class": "added",
|
|
"bbox": [
|
|
0,
|
|
0,
|
|
5,
|
|
5
|
|
],
|
|
"confidence": 0.85
|
|
},
|
|
{
|
|
"id": "p2",
|
|
"class": "removed",
|
|
"bbox": [
|
|
20,
|
|
0,
|
|
25,
|
|
5
|
|
],
|
|
"confidence": 0.65
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"sample_id": "terrain-test-flanders-rural",
|
|
"split": "test",
|
|
"task": "terrain_interpretation",
|
|
"metadata": {
|
|
"region": "flanders",
|
|
"municipality": "Hasselt",
|
|
"urbanity": "rural",
|
|
"object_size": "not_applicable",
|
|
"source": "fixture-dhmv",
|
|
"sensor": "elevation-raster",
|
|
"resolution_m": 5,
|
|
"season": "not_applicable",
|
|
"occlusion": "not_applicable",
|
|
"vegetation": "moderate",
|
|
"difficulty": "normal",
|
|
"context": "terrain_profile",
|
|
"tile_edge": false,
|
|
"label_review_state": "fixture_reviewed",
|
|
"ood": false,
|
|
"date": "2023-01-01"
|
|
},
|
|
"config": {
|
|
"missing_prediction_policy": "exclude_and_reduce_coverage"
|
|
},
|
|
"lineage": {
|
|
"reference": {
|
|
"source_id": "synthetic:terrain-test-flanders-rural:reference",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_contract_fixture"
|
|
},
|
|
"prediction": {
|
|
"source_id": "synthetic:terrain-test-flanders-rural:prediction",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_fixed_output"
|
|
}
|
|
},
|
|
"units": "m_TAW",
|
|
"references": [
|
|
12,
|
|
13,
|
|
14,
|
|
15,
|
|
16
|
|
],
|
|
"predictions": [
|
|
12.2,
|
|
12.8,
|
|
14.4,
|
|
null,
|
|
15.7
|
|
]
|
|
},
|
|
{
|
|
"sample_id": "validation-test-national",
|
|
"split": "test",
|
|
"task": "geospatial_data_validation",
|
|
"metadata": {
|
|
"region": "national",
|
|
"municipality": "not_applicable",
|
|
"urbanity": "mixed",
|
|
"object_size": "not_applicable",
|
|
"source": "fixture-contracts",
|
|
"sensor": "mixed",
|
|
"resolution_m": null,
|
|
"season": "not_applicable",
|
|
"occlusion": "not_applicable",
|
|
"vegetation": "mixed",
|
|
"difficulty": "hard",
|
|
"context": "data_contract",
|
|
"tile_edge": false,
|
|
"label_review_state": "fixture_reviewed",
|
|
"ood": false,
|
|
"date": "2025-01-01"
|
|
},
|
|
"config": {
|
|
"critical_severities": [
|
|
"blocker",
|
|
"critical"
|
|
]
|
|
},
|
|
"lineage": {
|
|
"reference": {
|
|
"source_id": "synthetic:validation-test-national:reference",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_contract_fixture"
|
|
},
|
|
"prediction": {
|
|
"source_id": "synthetic:validation-test-national:prediction",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_fixed_output"
|
|
}
|
|
},
|
|
"expected_anomalies": [
|
|
{
|
|
"code": "D-CRS-WRONG",
|
|
"severity": "blocker"
|
|
},
|
|
{
|
|
"code": "D-INVALID-GEOMETRY",
|
|
"severity": "critical"
|
|
},
|
|
{
|
|
"code": "D-MISSING-PROVENANCE",
|
|
"severity": "critical"
|
|
}
|
|
],
|
|
"observed_anomalies": [
|
|
{
|
|
"code": "D-CRS-WRONG",
|
|
"severity": "blocker"
|
|
},
|
|
{
|
|
"code": "D-INVALID-GEOMETRY",
|
|
"severity": "critical"
|
|
},
|
|
{
|
|
"code": "D-RESOLUTION",
|
|
"severity": "major"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"sample_id": "background-test-pure-empty",
|
|
"split": "background-test",
|
|
"task": "object_detection",
|
|
"metadata": {
|
|
"region": "brussels",
|
|
"municipality": "Brussel",
|
|
"urbanity": "urban",
|
|
"object_size": "empty",
|
|
"source": "fixture-orthophoto",
|
|
"sensor": "synthetic-rgb",
|
|
"resolution_m": 0.25,
|
|
"season": "winter",
|
|
"occlusion": "none",
|
|
"vegetation": "low",
|
|
"difficulty": "hard",
|
|
"context": "pure_background",
|
|
"tile_edge": false,
|
|
"label_review_state": "fixture_reviewed",
|
|
"ood": false,
|
|
"date": "2025-02-01"
|
|
},
|
|
"config": {
|
|
"confidence_threshold": 0.5,
|
|
"match_iou": 0.5,
|
|
"fixed_diagnostic_risk_thresholds": [
|
|
0,
|
|
0.5,
|
|
0.7,
|
|
0.9
|
|
]
|
|
},
|
|
"lineage": {
|
|
"reference": {
|
|
"source_id": "synthetic:background-test-pure-empty:reference",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_contract_fixture"
|
|
},
|
|
"prediction": {
|
|
"source_id": "synthetic:background-test-pure-empty:prediction",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_fixed_output"
|
|
}
|
|
},
|
|
"classes": [
|
|
"building"
|
|
],
|
|
"references": [],
|
|
"predictions": []
|
|
},
|
|
{
|
|
"sample_id": "background-test-hard-negative",
|
|
"split": "background-test",
|
|
"task": "object_detection",
|
|
"metadata": {
|
|
"region": "flanders",
|
|
"municipality": "Oostende",
|
|
"urbanity": "urban",
|
|
"object_size": "empty",
|
|
"source": "fixture-orthophoto",
|
|
"sensor": "synthetic-rgb",
|
|
"resolution_m": 0.25,
|
|
"season": "summer",
|
|
"occlusion": "none",
|
|
"vegetation": "moderate",
|
|
"difficulty": "hard",
|
|
"context": "coastal_hard_negative",
|
|
"tile_edge": true,
|
|
"label_review_state": "fixture_reviewed",
|
|
"ood": false,
|
|
"date": "2025-07-01"
|
|
},
|
|
"config": {
|
|
"confidence_threshold": 0.5,
|
|
"match_iou": 0.5,
|
|
"fixed_diagnostic_risk_thresholds": [
|
|
0,
|
|
0.5,
|
|
0.7,
|
|
0.9
|
|
]
|
|
},
|
|
"lineage": {
|
|
"reference": {
|
|
"source_id": "synthetic:background-test-hard-negative:reference",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_contract_fixture"
|
|
},
|
|
"prediction": {
|
|
"source_id": "synthetic:background-test-hard-negative:prediction",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_fixed_output"
|
|
}
|
|
},
|
|
"classes": [
|
|
"building"
|
|
],
|
|
"references": [],
|
|
"predictions": [
|
|
{
|
|
"id": "p-hard-fp",
|
|
"class": "building",
|
|
"bbox": [
|
|
20,
|
|
20,
|
|
35,
|
|
35
|
|
],
|
|
"confidence": 0.92
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|