docs(accuracy): publish phase 4 benchmark evidence
This commit is contained in:
Jens
+157
@@ -0,0 +1,157 @@
|
||||
{
|
||||
"benchmark_manifest_sha256": "0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642",
|
||||
"critical_subgroup_policy": "Any required subgroup with insufficient support, missing metrics, a failed non-inferiority comparison or regression blocks promotion; averages cannot override it.",
|
||||
"decision": "At least one mandatory Phase 4 gate failed; Phase 5 remains blocked.",
|
||||
"gate_policy": "geointel-p4-evaluation-harness-v2",
|
||||
"invalid_gate_states": {},
|
||||
"local_gates": {
|
||||
"all_declared_evaluator_families_exercised": {
|
||||
"declared": [
|
||||
"change_detection",
|
||||
"footprint_segmentation",
|
||||
"geospatial_data_validation",
|
||||
"object_detection",
|
||||
"raster_classification",
|
||||
"terrain_interpretation",
|
||||
"vector_comparison"
|
||||
],
|
||||
"observed": [
|
||||
"change_detection",
|
||||
"footprint_segmentation",
|
||||
"geospatial_data_validation",
|
||||
"object_detection",
|
||||
"raster_classification",
|
||||
"terrain_interpretation",
|
||||
"vector_comparison"
|
||||
],
|
||||
"status": "pass"
|
||||
},
|
||||
"complete_raw_predictions_retained": {
|
||||
"status": "pass",
|
||||
"violating_samples": []
|
||||
},
|
||||
"implemented_capability_inventory": {
|
||||
"capability_count": 15,
|
||||
"status": "pass"
|
||||
},
|
||||
"manifest_training_firewall_contract": {
|
||||
"checks": {
|
||||
"clean_train_allowed": true,
|
||||
"non_train_role_blocked": true,
|
||||
"protected_path_blocked": true,
|
||||
"renamed_protected_lineage_blocked": true
|
||||
},
|
||||
"status": "pass"
|
||||
},
|
||||
"normative_split_roles_and_leakage": {
|
||||
"leakage_status": "pass",
|
||||
"observed_roles": [
|
||||
"background-test",
|
||||
"calibration",
|
||||
"challenge",
|
||||
"test",
|
||||
"train",
|
||||
"val"
|
||||
],
|
||||
"required_roles": [
|
||||
"background-test",
|
||||
"calibration",
|
||||
"challenge",
|
||||
"test",
|
||||
"train",
|
||||
"val"
|
||||
],
|
||||
"status": "pass"
|
||||
},
|
||||
"protected_operating_point_contract": {
|
||||
"evidence": "Protected cases carry pre-registered configurations. Fixed AP/risk-coverage diagnostics cannot select an operating point or feed back into training.",
|
||||
"status": "pass"
|
||||
},
|
||||
"reference_implementation_baseline": {
|
||||
"status": "pass"
|
||||
},
|
||||
"stratified_metric_contract": {
|
||||
"observed_overall_status": "not_evaluable",
|
||||
"status": "pass"
|
||||
},
|
||||
"undefined_metric_truth_table": {
|
||||
"sample_id": "background-test-pure-empty",
|
||||
"status": "pass"
|
||||
}
|
||||
},
|
||||
"local_harness_status": "pass",
|
||||
"missing_gate_names": {
|
||||
"local": [],
|
||||
"product": []
|
||||
},
|
||||
"numeric_model_release_targets": "not_frozen_without_reviewed_representative_incumbent_baseline",
|
||||
"phase_decision": "blocked",
|
||||
"product_benchmark_status": "fail",
|
||||
"product_gates": {
|
||||
"active_model_available_and_hash_verified": {
|
||||
"configured_path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt",
|
||||
"configured_sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1",
|
||||
"reason": "Configured active model is not locally accessible.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"authoritative_reference_portfolio_available": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"executed_product_incumbent_baseline": {
|
||||
"checked_artifacts": [],
|
||||
"derived_gates": {
|
||||
"authoritative_reference_portfolio_available": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"human_review_complete": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"protected_storage_isolation": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"representative_product_subgroup_support": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"split_independence": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
}
|
||||
},
|
||||
"expected_path": "artifacts/evidence/accuracy/P4/product-baseline-manifest.json",
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"human_review_complete": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"phase3_leakage_resolved": {
|
||||
"observed": "attention",
|
||||
"status": "fail"
|
||||
},
|
||||
"protected_storage_isolation": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"representative_product_subgroup_support": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"split_independence": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
}
|
||||
},
|
||||
"promotion_allowed": false,
|
||||
"schema_version": 2,
|
||||
"status": "fail",
|
||||
"unexpected_gate_names": {
|
||||
"local": [],
|
||||
"product": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"reason": "Synthetic single-case fixtures do not provide independent product AOI clusters. AOI micro/macro and cluster-bootstrap evidence requires the protected product corpus.",
|
||||
"required_future_outputs": [
|
||||
"per-AOI primary metrics",
|
||||
"micro and macro aggregation",
|
||||
"paired candidate-minus-incumbent deltas",
|
||||
"cluster-bootstrap confidence intervals"
|
||||
],
|
||||
"schema_version": 2,
|
||||
"status": "not_evaluable"
|
||||
}
|
||||
+1489
File diff suppressed because it is too large
Load Diff
+105
@@ -0,0 +1,105 @@
|
||||
{
|
||||
"benchmark_id": "geointel-p4-reference-harness-v2",
|
||||
"claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.",
|
||||
"code": [
|
||||
{
|
||||
"path": "scripts/accuracy_phase4_evaluator.py",
|
||||
"sha256": "d2e40f7978d3ff4ce7025eaaaefb06ccc76c269d6874f8b0f528739d535a1da1",
|
||||
"size_bytes": 93353
|
||||
},
|
||||
{
|
||||
"path": "scripts/generate_accuracy_phase4_splits.py",
|
||||
"sha256": "d12842922bcd99b1672176df15a9cef250128e7d7fc662ab15e16f1c084ba570",
|
||||
"size_bytes": 75923
|
||||
},
|
||||
{
|
||||
"path": "scripts/run_accuracy_phase4_benchmark.py",
|
||||
"sha256": "003ecb4babc7f56a6d13e9e1c08dda902b9b26203cc7d53476d4d5e083b8b85b",
|
||||
"size_bytes": 123717
|
||||
},
|
||||
{
|
||||
"path": "scripts/run_golden_qa_benchmark.py",
|
||||
"sha256": "2c8a19fd92db8f07f491b8f7b34d9736384f81e02a6b83019fac71b101ebdaa0",
|
||||
"size_bytes": 10856
|
||||
},
|
||||
{
|
||||
"path": "backend/app/services/qa_service.py",
|
||||
"sha256": "2d4f645410fbf2972e55dddbf5a85db5368b2b1932792cd522ec51c483339244",
|
||||
"size_bytes": 15206
|
||||
}
|
||||
],
|
||||
"evaluation_results_canonical_json_sha256": "55cdba9cce4dcdc88cee0829cd510d7e624a3472feb9d33f7e681621a792eba3",
|
||||
"evaluator_version": "2.1.0",
|
||||
"evidence_run_id": "p4-2.0.1-9677d0ef37db82bcf39b",
|
||||
"inference_and_selection": {
|
||||
"background_test_used_for_selection": false,
|
||||
"challenge_labels_available": false,
|
||||
"production_model_inference_executed": false,
|
||||
"raw_predictions_retained": true,
|
||||
"synthetic_reference_harness": true,
|
||||
"test_used_for_selection": false,
|
||||
"threshold_source": "pre_registered_configuration_only"
|
||||
},
|
||||
"input_manifest": {
|
||||
"path": "input-manifest.json",
|
||||
"sha256": "44f1c81d76baf4dc965afc2cd109ac165ae8fe0308f202018ea21bac786bcd0b"
|
||||
},
|
||||
"inputs": {
|
||||
"golden_qa_manifest": {
|
||||
"path": "fixtures/golden/golden_qa_benchmarks.json",
|
||||
"sha256": "d60576ff7048ca3a048dc166e8977f3a31e9f8fa352be5ad222a017260e179e2",
|
||||
"size_bytes": 2758
|
||||
},
|
||||
"phase3_full_scan": {
|
||||
"path": "artifacts/evidence/accuracy/P3/full-scan-manifest.json",
|
||||
"sha256": "28c1022d125f30bfb4aa362b57d8a90cf8c88270b2a9ac6582b86b6a44a08038",
|
||||
"size_bytes": 755527
|
||||
},
|
||||
"phase3_leakage": {
|
||||
"path": "artifacts/evidence/accuracy/P3/leakage-report.json",
|
||||
"sha256": "8e1b7a24a840549d442bc03132ee1f21c6d50f8714cbb8d33430391a3769cd4e",
|
||||
"size_bytes": 580
|
||||
},
|
||||
"protected_cases": {
|
||||
"path": "fixtures/accuracy/p4/protected-baseline-cases.json",
|
||||
"sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473",
|
||||
"size_bytes": 20091
|
||||
},
|
||||
"split_source": {
|
||||
"path": "fixtures/accuracy/p4/split-source-manifest.json",
|
||||
"sha256": "27ec60406ac57a02f1335013ee344a43d4e0093e7ee1329ec22876da623f5bfc",
|
||||
"size_bytes": 17331
|
||||
}
|
||||
},
|
||||
"manifest_sha256": "0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642",
|
||||
"product_baseline": {
|
||||
"artifacts": [],
|
||||
"manifest": null,
|
||||
"product_gate_evidence_sha256": "c9f5e3de3ab6d64911f8807b26caa661ccf911a5fd6b995107edda2004836e0c",
|
||||
"requested_path": "artifacts/evidence/accuracy/P4/product-baseline-manifest.json",
|
||||
"validation_status": "not_evaluable"
|
||||
},
|
||||
"product_gate_evidence_sha256": "c9f5e3de3ab6d64911f8807b26caa661ccf911a5fd6b995107edda2004836e0c",
|
||||
"reference_baseline_sha256": "aeb0f6510352f4e205e4fc4e8c423110ca72c2142016c7af846d758865f258c6",
|
||||
"repository_commit": "70fb4b94e5cb7c248beec5a936ce186f38cc183c",
|
||||
"runtime": {
|
||||
"cuda_used_for_reference_harness": false,
|
||||
"dependencies": {
|
||||
"numpy": "2.2.2",
|
||||
"pyproj": "3.7.2",
|
||||
"shapely": "2.1.2"
|
||||
},
|
||||
"execution_device": "CPU deterministic evaluator arithmetic; no production model inference",
|
||||
"platform": "Windows-11-10.0.26200-SP0",
|
||||
"python": "3.13.2",
|
||||
"python_implementation": "CPython"
|
||||
},
|
||||
"schema_version": 2,
|
||||
"split_generator_version": "1.3.0",
|
||||
"split_manifests": {
|
||||
"development_sha256": "64e4fc986c2390bedfbe100062d4b06e1098689f4d4a9a9da13a7f67357c0493",
|
||||
"leakage_status": "pass",
|
||||
"protected_sha256": "e57ca1529654ef1c2527e84122ff26ec3533e7c70658250983968fbc5d4af410"
|
||||
},
|
||||
"workflow_version": "2.0.1"
|
||||
}
|
||||
+274
@@ -0,0 +1,274 @@
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"calibration": {
|
||||
"binning": "five fixed equal-width bins",
|
||||
"bins": [
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.0,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.2
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.2,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.4
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.4,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.6
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.6,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.8
|
||||
},
|
||||
{
|
||||
"accuracy": 0,
|
||||
"count": 1,
|
||||
"lower": 0.8,
|
||||
"mean_confidence": 0.92,
|
||||
"upper": 1.0
|
||||
}
|
||||
],
|
||||
"brier": 0.8464,
|
||||
"ece": 0.92,
|
||||
"status": "computed"
|
||||
},
|
||||
"coverage_risk": [
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.0,
|
||||
"total_prediction_count": 1
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.5,
|
||||
"total_prediction_count": 1
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.7,
|
||||
"total_prediction_count": 1
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.9,
|
||||
"total_prediction_count": 1
|
||||
}
|
||||
],
|
||||
"sample_id": "background-test-hard-negative",
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"calibration": {
|
||||
"bins": [],
|
||||
"brier": null,
|
||||
"ece": null,
|
||||
"status": "undefined"
|
||||
},
|
||||
"coverage_risk": [
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.0,
|
||||
"total_prediction_count": 0
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.5,
|
||||
"total_prediction_count": 0
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.7,
|
||||
"total_prediction_count": 0
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.9,
|
||||
"total_prediction_count": 0
|
||||
}
|
||||
],
|
||||
"sample_id": "background-test-pure-empty",
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"calibration": {
|
||||
"binning": "five fixed equal-width bins",
|
||||
"bins": [
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.0,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.2
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.2,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.4
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.4,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.6
|
||||
},
|
||||
{
|
||||
"accuracy": 0.5,
|
||||
"count": 2,
|
||||
"lower": 0.6,
|
||||
"mean_confidence": 0.6499999999999999,
|
||||
"upper": 0.8
|
||||
},
|
||||
{
|
||||
"accuracy": 1,
|
||||
"count": 1,
|
||||
"lower": 0.8,
|
||||
"mean_confidence": 0.9,
|
||||
"upper": 1.0
|
||||
}
|
||||
],
|
||||
"brier": 0.15333333333333332,
|
||||
"ece": 0.13333333333333325,
|
||||
"status": "computed"
|
||||
},
|
||||
"coverage_risk": [
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 2,
|
||||
"matched_reference_count": 2,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 1.0,
|
||||
"retained_prediction_count": 4,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 0.5,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.0,
|
||||
"total_prediction_count": 4
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 2,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 1.0,
|
||||
"retained_prediction_count": 3,
|
||||
"retained_prediction_coverage": 0.75,
|
||||
"risk": 0.3333333333333333,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.5,
|
||||
"total_prediction_count": 4
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 2,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 1.0,
|
||||
"retained_prediction_count": 2,
|
||||
"retained_prediction_coverage": 0.5,
|
||||
"risk": 0.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.7,
|
||||
"total_prediction_count": 4
|
||||
},
|
||||
{
|
||||
"false_negative_count": 1,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 1,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 0.5,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 0.25,
|
||||
"risk": 0.5,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.9,
|
||||
"total_prediction_count": 4
|
||||
}
|
||||
],
|
||||
"sample_id": "det-test-flanders-urban",
|
||||
"task": "object_detection"
|
||||
}
|
||||
],
|
||||
"note": "Fixed diagnostic bins and risk thresholds test metric arithmetic; they do not select or change any operating point.",
|
||||
"schema_version": 2,
|
||||
"selection_allowed": false,
|
||||
"status": "fixture_diagnostic_only"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"future_gate_contract": {
|
||||
"aggregate_improvement_may_mask_subgroup_regression": false,
|
||||
"global_and_critical_subgroups_required": true,
|
||||
"missing_or_insufficient_support": "not_evaluable",
|
||||
"numeric_margin": "to_be_frozen_before_protected_access",
|
||||
"unit": "paired independent AOI"
|
||||
},
|
||||
"reason": "Phase 4 has no valid real incumbent product baseline and no pre-registered candidate; synthetic fixture values cannot define non-inferiority.",
|
||||
"schema_version": 2,
|
||||
"status": "not_evaluable"
|
||||
}
|
||||
+334
@@ -0,0 +1,334 @@
|
||||
{
|
||||
"allowed_splits": [
|
||||
"calibration",
|
||||
"train",
|
||||
"val"
|
||||
],
|
||||
"assignment_algorithm": "group-before-split-deficit-balancer-v1",
|
||||
"assignment_mode": "preassigned",
|
||||
"assignment_seed": "geointel-p4-group-split-v1",
|
||||
"claim_boundary": "Synthetic contract fixtures for evaluator regression only; never production accuracy evidence.",
|
||||
"crs": "EPSG:31370",
|
||||
"dataset_version": "geointel-p4-harness-fixture-v2",
|
||||
"generator_version": "1.3.0",
|
||||
"independence_buffer_m": 2000.0,
|
||||
"label_geometry_hamming_threshold": 2,
|
||||
"mandatory_split_roles": [
|
||||
"train",
|
||||
"val",
|
||||
"calibration",
|
||||
"test",
|
||||
"background-test",
|
||||
"challenge"
|
||||
],
|
||||
"manifest_role": "development_and_calibration",
|
||||
"manifest_sha256": "64e4fc986c2390bedfbe100062d4b06e1098689f4d4a9a9da13a7f67357c0493",
|
||||
"perceptual_hamming_threshold": 4,
|
||||
"samples": [
|
||||
{
|
||||
"acquisition_date": "2025-01-07",
|
||||
"acquisition_id": "acquisition-07",
|
||||
"bbox": [
|
||||
70000.0,
|
||||
30000.0,
|
||||
70100.0,
|
||||
30100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g07",
|
||||
"label_geometry_fingerprint": "dba609f34cb9826b",
|
||||
"label_geometry_hash": "6767676767676767676767676767676767676767676767676767676767676767",
|
||||
"label_sha256": "4747474747474747474747474747474747474747474747474747474747474747",
|
||||
"native_feature_ids": [
|
||||
"native-07-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o21"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-07",
|
||||
"perceptual_image_hash": "c4316514fed98f6b",
|
||||
"processed_image_sha256": "2727272727272727272727272727272727272727272727272727272727272727",
|
||||
"raw_image_sha256": "0707070707070707070707070707070707070707070707070707070707070707",
|
||||
"record_sha256": "3f04e62eb97c0a067a4742084a9665607beab43a5c0392a89c77bc10cb472136",
|
||||
"sample_id": "calibration-det-a",
|
||||
"source_family": "scene-07",
|
||||
"split": "calibration",
|
||||
"task": "object_detection",
|
||||
"temporal_family": "temporal-07"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-08",
|
||||
"acquisition_id": "acquisition-08",
|
||||
"bbox": [
|
||||
90000.0,
|
||||
30000.0,
|
||||
90100.0,
|
||||
30100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g08",
|
||||
"label_geometry_fingerprint": "627df87eea48aec0",
|
||||
"label_geometry_hash": "6868686868686868686868686868686868686868686868686868686868686868",
|
||||
"label_sha256": "4848484848484848484848484848484848484848484848484848484848484848",
|
||||
"native_feature_ids": [
|
||||
"native-08-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o22"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-08",
|
||||
"perceptual_image_hash": "518ba3145333c08c",
|
||||
"processed_image_sha256": "2828282828282828282828282828282828282828282828282828282828282828",
|
||||
"raw_image_sha256": "0808080808080808080808080808080808080808080808080808080808080808",
|
||||
"record_sha256": "d3cec9ed2d1a4c7380958a960ac24d1dbc5a729ccb81cc0f22016ee348137a07",
|
||||
"sample_id": "calibration-seg-b",
|
||||
"source_family": "scene-08",
|
||||
"split": "calibration",
|
||||
"task": "footprint_segmentation",
|
||||
"temporal_family": "temporal-08"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-01",
|
||||
"acquisition_id": "acquisition-01",
|
||||
"bbox": [
|
||||
10000.0,
|
||||
10000.0,
|
||||
10100.0,
|
||||
10100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g01",
|
||||
"label_geometry_fingerprint": "c0f62e16b4033ee9",
|
||||
"label_geometry_hash": "6161616161616161616161616161616161616161616161616161616161616161",
|
||||
"label_sha256": "4141414141414141414141414141414141414141414141414141414141414141",
|
||||
"native_feature_ids": [
|
||||
"native-01-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o01"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-01",
|
||||
"perceptual_image_hash": "dcef24412401d4f9",
|
||||
"processed_image_sha256": "2121212121212121212121212121212121212121212121212121212121212121",
|
||||
"raw_image_sha256": "0101010101010101010101010101010101010101010101010101010101010101",
|
||||
"record_sha256": "16142e63a8e93d20d1fcfb66cfd8aeeebc6cbe7784f5ce8c403cd4b2f6e5dcac",
|
||||
"sample_id": "det-train-a",
|
||||
"source_family": "scene-01",
|
||||
"split": "train",
|
||||
"task": "object_detection",
|
||||
"temporal_family": "temporal-01"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-03",
|
||||
"acquisition_id": "acquisition-03",
|
||||
"bbox": [
|
||||
30000.0,
|
||||
10000.0,
|
||||
30100.0,
|
||||
10100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g03",
|
||||
"label_geometry_fingerprint": "30820f2c124a3ff0",
|
||||
"label_geometry_hash": "6363636363636363636363636363636363636363636363636363636363636363",
|
||||
"label_sha256": "4343434343434343434343434343434343434343434343434343434343434343",
|
||||
"native_feature_ids": [
|
||||
"native-03-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o03"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-03",
|
||||
"perceptual_image_hash": "75e36e008b4a6702",
|
||||
"processed_image_sha256": "2323232323232323232323232323232323232323232323232323232323232323",
|
||||
"raw_image_sha256": "0303030303030303030303030303030303030303030303030303030303030303",
|
||||
"record_sha256": "60c78f7f4cb086681fa38d3a2e160824003cb646890faf6ddde142eab585adc8",
|
||||
"sample_id": "raster-train-a",
|
||||
"source_family": "scene-03",
|
||||
"split": "train",
|
||||
"task": "raster_classification",
|
||||
"temporal_family": "temporal-03"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-02",
|
||||
"acquisition_id": "acquisition-02",
|
||||
"bbox": [
|
||||
20000.0,
|
||||
10000.0,
|
||||
20100.0,
|
||||
10100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g02",
|
||||
"label_geometry_fingerprint": "666a0fc2d26265ee",
|
||||
"label_geometry_hash": "6262626262626262626262626262626262626262626262626262626262626262",
|
||||
"label_sha256": "4242424242424242424242424242424242424242424242424242424242424242",
|
||||
"native_feature_ids": [
|
||||
"native-02-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o02"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-02",
|
||||
"perceptual_image_hash": "030bcdd3bc87412e",
|
||||
"processed_image_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
|
||||
"raw_image_sha256": "0202020202020202020202020202020202020202020202020202020202020202",
|
||||
"record_sha256": "b92bff2d0e039441711100b95178a6bebab0052db1b7e35259dcbcb227d50448",
|
||||
"sample_id": "seg-train-a",
|
||||
"source_family": "scene-02",
|
||||
"split": "train",
|
||||
"task": "footprint_segmentation",
|
||||
"temporal_family": "temporal-02"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-05",
|
||||
"acquisition_id": "acquisition-05",
|
||||
"bbox": [
|
||||
30000.0,
|
||||
30000.0,
|
||||
30100.0,
|
||||
30100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g05",
|
||||
"label_geometry_fingerprint": "14ceff3640a013eb",
|
||||
"label_geometry_hash": "6565656565656565656565656565656565656565656565656565656565656565",
|
||||
"label_sha256": "4545454545454545454545454545454545454545454545454545454545454545",
|
||||
"native_feature_ids": [
|
||||
"native-05-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o05"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-05",
|
||||
"perceptual_image_hash": "828b99671c3f2f39",
|
||||
"processed_image_sha256": "2525252525252525252525252525252525252525252525252525252525252525",
|
||||
"raw_image_sha256": "0505050505050505050505050505050505050505050505050505050505050505",
|
||||
"record_sha256": "bc142eb7eb7ed86621984ff89184b1b79cfaa13e1011832c8b8175312761ea91",
|
||||
"sample_id": "val-change-b",
|
||||
"source_family": "scene-05",
|
||||
"split": "val",
|
||||
"task": "change_detection",
|
||||
"temporal_family": "temporal-05"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-06",
|
||||
"acquisition_id": "acquisition-06",
|
||||
"bbox": [
|
||||
50000.0,
|
||||
30000.0,
|
||||
50100.0,
|
||||
30100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g06",
|
||||
"label_geometry_fingerprint": "a1d9bbb93ba6d6da",
|
||||
"label_geometry_hash": "6666666666666666666666666666666666666666666666666666666666666666",
|
||||
"label_sha256": "4646464646464646464646464646464646464646464646464646464646464646",
|
||||
"native_feature_ids": [
|
||||
"native-06-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o06"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-06",
|
||||
"perceptual_image_hash": "11c11105b0d10635",
|
||||
"processed_image_sha256": "2626262626262626262626262626262626262626262626262626262626262626",
|
||||
"raw_image_sha256": "0606060606060606060606060606060606060606060606060606060606060606",
|
||||
"record_sha256": "a861ffb75dba0e50e4082232620521465acecdbc0cf1f6dee9b7509d9c0d480e",
|
||||
"sample_id": "val-terrain-c",
|
||||
"source_family": "scene-06",
|
||||
"split": "val",
|
||||
"task": "terrain_interpretation",
|
||||
"temporal_family": "temporal-06"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-04",
|
||||
"acquisition_id": "acquisition-04",
|
||||
"bbox": [
|
||||
10000.0,
|
||||
30000.0,
|
||||
10100.0,
|
||||
30100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g04",
|
||||
"label_geometry_fingerprint": "a559b46cba64e501",
|
||||
"label_geometry_hash": "6464646464646464646464646464646464646464646464646464646464646464",
|
||||
"label_sha256": "4444444444444444444444444444444444444444444444444444444444444444",
|
||||
"native_feature_ids": [
|
||||
"native-04-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o04"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-04",
|
||||
"perceptual_image_hash": "2e3b331cc01536c9",
|
||||
"processed_image_sha256": "2424242424242424242424242424242424242424242424242424242424242424",
|
||||
"raw_image_sha256": "0404040404040404040404040404040404040404040404040404040404040404",
|
||||
"record_sha256": "e2a931aa255519d0bd87655d92132164923b369a31e12583c8aa499610044a30",
|
||||
"sample_id": "val-vector-a",
|
||||
"source_family": "scene-04",
|
||||
"split": "val",
|
||||
"task": "vector_comparison",
|
||||
"temporal_family": "temporal-04"
|
||||
}
|
||||
],
|
||||
"schema_version": 1,
|
||||
"selection_access_allowed_by_split": {
|
||||
"calibration": true,
|
||||
"train": false,
|
||||
"val": true
|
||||
},
|
||||
"source_manifest_sha256": "1c75c875d4c297d046117bb9393ba5e648f13d3358ea92cb65374869c8f5fc01",
|
||||
"source_trust": {
|
||||
"mode": "synthetic_fixture",
|
||||
"policy_id": "geointel-p4-synthetic-fixture-v2",
|
||||
"production_accuracy_use_allowed": false
|
||||
},
|
||||
"training_access_allowed_by_split": {
|
||||
"calibration": false,
|
||||
"train": true,
|
||||
"val": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.",
|
||||
"observed_codes": [
|
||||
{
|
||||
"error_code": "D-VALIDATION-FN",
|
||||
"kind": "validation_false_negative"
|
||||
},
|
||||
{
|
||||
"error_code": "D-VALIDATION-FP",
|
||||
"kind": "validation_false_positive"
|
||||
},
|
||||
{
|
||||
"error_code": "M-AREA-BIAS",
|
||||
"kind": "area_bias"
|
||||
},
|
||||
{
|
||||
"error_code": "M-CLASS",
|
||||
"kind": "raster_misclassification"
|
||||
},
|
||||
{
|
||||
"error_code": "M-FN-MISSED",
|
||||
"kind": "event_false_negative"
|
||||
},
|
||||
{
|
||||
"error_code": "M-FN-MISSED",
|
||||
"kind": "false_negative"
|
||||
},
|
||||
{
|
||||
"error_code": "M-FP-CONFUSER",
|
||||
"kind": "event_false_positive"
|
||||
},
|
||||
{
|
||||
"error_code": "M-FP-CONFUSER",
|
||||
"kind": "false_positive"
|
||||
},
|
||||
{
|
||||
"error_code": "M-MISCALIBRATED",
|
||||
"kind": "miscalibrated"
|
||||
},
|
||||
{
|
||||
"error_code": "P-PARTIAL",
|
||||
"kind": "terrain_missing"
|
||||
}
|
||||
],
|
||||
"observed_failure_count": 14,
|
||||
"schema_version": 2,
|
||||
"source": "docs/accuracy-program/05-metric-framework.md section 4"
|
||||
}
|
||||
+300
@@ -0,0 +1,300 @@
|
||||
{
|
||||
"benchmark_id": "geointel-p4-reference-harness-v2",
|
||||
"claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.",
|
||||
"evaluator_families": [
|
||||
"change_detection",
|
||||
"footprint_segmentation",
|
||||
"geospatial_data_validation",
|
||||
"object_detection",
|
||||
"raster_classification",
|
||||
"terrain_interpretation",
|
||||
"vector_comparison"
|
||||
],
|
||||
"evaluator_version": "2.1.0",
|
||||
"gate_states": [
|
||||
"fail",
|
||||
"not_evaluable",
|
||||
"pass"
|
||||
],
|
||||
"implemented_capabilities": [
|
||||
{
|
||||
"capability_id": "model_object_detection",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "model_inference_pipeline",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/detection_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"precision",
|
||||
"recall",
|
||||
"F1",
|
||||
"AP50",
|
||||
"AP50-95",
|
||||
"IoU",
|
||||
"ECE",
|
||||
"Brier",
|
||||
"coverage-risk"
|
||||
],
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"capability_id": "building_proposal_filtering",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "not_separately_benchmarked",
|
||||
"implementation_kind": "supporting_candidate_classifier",
|
||||
"implementation_paths": [
|
||||
"scripts/train_building_proposal_classifier.py",
|
||||
"scripts/evaluate_belgium_building_candidate.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"candidate precision",
|
||||
"candidate recall",
|
||||
"F1",
|
||||
"calibration"
|
||||
],
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"capability_id": "footprint_segmentation",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "model_inference_pipeline",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/segmentation_service.py",
|
||||
"backend/app/services/segmentation_adapter.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"object precision",
|
||||
"object recall",
|
||||
"F1",
|
||||
"IoU",
|
||||
"Dice",
|
||||
"boundary F1",
|
||||
"centroid distance",
|
||||
"area error",
|
||||
"topology"
|
||||
],
|
||||
"task": "footprint_segmentation"
|
||||
},
|
||||
{
|
||||
"capability_id": "detection_and_vector_qa",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "deterministic_geospatial_comparison",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/qa_service.py",
|
||||
"backend/app/services/detection_qa_service.py",
|
||||
"backend/app/services/quality_check_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"precision",
|
||||
"recall",
|
||||
"F1",
|
||||
"IoU",
|
||||
"topology",
|
||||
"coverage"
|
||||
],
|
||||
"task": "vector_comparison"
|
||||
},
|
||||
{
|
||||
"capability_id": "vector_clip_buffer_intersect",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_vector_processing",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/vector_operations_service.py",
|
||||
"backend/app/services/vector_feature_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"geometry validity",
|
||||
"CRS correctness",
|
||||
"area conservation",
|
||||
"feature counts",
|
||||
"topology"
|
||||
],
|
||||
"task": "vector_comparison"
|
||||
},
|
||||
{
|
||||
"capability_id": "temporal_vector_change",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "deterministic_change_detection",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/change_detection_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"event precision",
|
||||
"event recall",
|
||||
"event F1",
|
||||
"IoU"
|
||||
],
|
||||
"task": "change_detection"
|
||||
},
|
||||
{
|
||||
"capability_id": "thematic_raster_interpretation",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_metric_contract_only_no_generic_learned_classifier_claim",
|
||||
"implementation_kind": "deterministic_source_interpretation",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/thematic_raster_analysis_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"pixel accuracy",
|
||||
"per-class F1",
|
||||
"per-class IoU",
|
||||
"mean IoU"
|
||||
],
|
||||
"task": "raster_classification"
|
||||
},
|
||||
{
|
||||
"capability_id": "raster_clip_reproject_indices",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_raster_processing",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/raster_service.py",
|
||||
"backend/app/services/raster_operations_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"CRS/transform preservation",
|
||||
"pixel alignment",
|
||||
"nodata",
|
||||
"numeric tolerance"
|
||||
],
|
||||
"task": "raster_classification"
|
||||
},
|
||||
{
|
||||
"capability_id": "raster_partition_mosaic",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_raster_partitioning",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/raster_partition_analysis_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"seam equality",
|
||||
"coverage completeness",
|
||||
"resolution consistency"
|
||||
],
|
||||
"task": "raster_classification"
|
||||
},
|
||||
{
|
||||
"capability_id": "terrain_height_interpretation",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "deterministic_continuous_raster_analysis",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/terrain_analysis_service.py",
|
||||
"backend/app/services/spw_terrain_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"MAE",
|
||||
"RMSE",
|
||||
"bias",
|
||||
"coverage",
|
||||
"unit integrity"
|
||||
],
|
||||
"task": "terrain_interpretation"
|
||||
},
|
||||
{
|
||||
"capability_id": "flood_hazard_interpretation",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_scenario_raster_analysis",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/flood_hazard_analysis_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"depth MAE/RMSE",
|
||||
"hazard-class IoU",
|
||||
"coverage",
|
||||
"scenario identity"
|
||||
],
|
||||
"task": "terrain_interpretation"
|
||||
},
|
||||
{
|
||||
"capability_id": "bathymetry_interpretation",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "covered_by_family_not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_vertical_reference_analysis",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/bathymetry_raster_analysis_service.py",
|
||||
"backend/app/services/mdk_bathymetry_probe_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"MAE",
|
||||
"RMSE",
|
||||
"bias",
|
||||
"coverage",
|
||||
"vertical-datum integrity"
|
||||
],
|
||||
"task": "terrain_interpretation"
|
||||
},
|
||||
{
|
||||
"capability_id": "data_contract_validation_and_scan",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "synthetic_contract_case_only",
|
||||
"implementation_kind": "deterministic_validation",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/data_contract_validation.py",
|
||||
"scripts/run_accuracy_phase3_full_data_scan.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"anomaly precision",
|
||||
"anomaly recall",
|
||||
"anomaly F1",
|
||||
"critical misses"
|
||||
],
|
||||
"task": "geospatial_data_validation"
|
||||
},
|
||||
{
|
||||
"capability_id": "aoi_partition_orchestration",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "not_separately_benchmarked",
|
||||
"implementation_kind": "deterministic_orchestration",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/aoi_operation_service.py",
|
||||
"backend/app/services/aoi_operation_executor.py",
|
||||
"backend/app/services/aoi_operation_worker.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"partition completeness",
|
||||
"overlap/gap",
|
||||
"idempotency",
|
||||
"resume correctness"
|
||||
],
|
||||
"task": "geospatial_data_validation"
|
||||
},
|
||||
{
|
||||
"capability_id": "geo_assistant_orchestration",
|
||||
"claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.",
|
||||
"evaluation_status": "no_independent_accuracy_score_underlying_tool_results_are_authoritative",
|
||||
"implementation_kind": "tool_orchestration_interface",
|
||||
"implementation_paths": [
|
||||
"backend/app/services/geo_assistant_service.py"
|
||||
],
|
||||
"suitable_metrics": [
|
||||
"tool-selection correctness",
|
||||
"grounding",
|
||||
"unsupported-claim rate"
|
||||
],
|
||||
"task": "geospatial_data_validation"
|
||||
}
|
||||
],
|
||||
"metric_contract": {
|
||||
"path": "docs/accuracy-program/05-metric-framework.md",
|
||||
"sha256": "18abfc21964b7d98848d8ae513afce242184980c9517c9184d40efd9165ea00d",
|
||||
"size_bytes": 28053
|
||||
},
|
||||
"protected_policy": {
|
||||
"diagnostic_curves_select_operating_point": false,
|
||||
"operating_point_selection_allowed": false,
|
||||
"test_feedback_allowed": false,
|
||||
"threshold_selection_source": "pre_registered_configuration_only"
|
||||
},
|
||||
"schema_version": 2,
|
||||
"split_generator_version": "1.3.0",
|
||||
"undefined_value_policy": "Undefined denominators are null with numerator, denominator and support; they are never coerced to a perfect score.",
|
||||
"workflow_version": "2.0.1"
|
||||
}
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
{
|
||||
"artifact_count": 24,
|
||||
"artifacts": [
|
||||
{
|
||||
"path": "acceptance-gates.json",
|
||||
"sha256": "e5f1c9c43c8da22acfa5486e641c010a278431f90634f91335b039ab7bdb50a4",
|
||||
"size_bytes": 5517
|
||||
},
|
||||
{
|
||||
"path": "aoi-metrics.json",
|
||||
"sha256": "2f5424c4a624316d0c8e71696440080d535ec0682831a80d664694f7cc198e94",
|
||||
"size_bytes": 430
|
||||
},
|
||||
{
|
||||
"path": "baseline-raw-predictions.json",
|
||||
"sha256": "d03ec665fcb3efb9546c1e5a42888cd2775307e3c7ba90d225389caca7e1140f",
|
||||
"size_bytes": 42170
|
||||
},
|
||||
{
|
||||
"path": "benchmark-manifest.json",
|
||||
"sha256": "868dd7eb2dfa8344e417879ed3f5dd7c7d9e5673add7e3e2294c5cbde4b75d57",
|
||||
"size_bytes": 4212
|
||||
},
|
||||
{
|
||||
"path": "calibration-metrics.json",
|
||||
"sha256": "ad54f924512529deba6d869736ca3f72dde8651227943c0b9e8695912e7a0c3d",
|
||||
"size_bytes": 8858
|
||||
},
|
||||
{
|
||||
"path": "candidate-vs-incumbent.json",
|
||||
"sha256": "7ee6fb5722b43fa0dc0a8e966547c954c62c7401b10e6f727e21d332b34f077f",
|
||||
"size_bytes": 517
|
||||
},
|
||||
{
|
||||
"path": "development-split-manifest.json",
|
||||
"sha256": "636b1d38695ade99ee3896a2d870fcaf0635168e8e25657353ac51a7bfba2248",
|
||||
"size_bytes": 12424
|
||||
},
|
||||
{
|
||||
"path": "error-taxonomy.json",
|
||||
"sha256": "acf2804e482dfdca1df05ef2c6e40701b82877466761b880b314e46699f470dc",
|
||||
"size_bytes": 1116
|
||||
},
|
||||
{
|
||||
"path": "evaluation-contract.json",
|
||||
"sha256": "797598cf3060cffb30f352ed022459c3be2e656c19b190c38b66fb99dbe54bd2",
|
||||
"size_bytes": 11057
|
||||
},
|
||||
{
|
||||
"path": "failure-gallery.json",
|
||||
"sha256": "eaf19cbad22a44fc2b6420ff0591536a61560815f472bf7fb54ed0179bd93986",
|
||||
"size_bytes": 14852
|
||||
},
|
||||
{
|
||||
"path": "generation-status.json",
|
||||
"sha256": "3311ae1ddf622f94c1a2e596b24454e0c033428b5d767d8cc3c67ee4e33a1a63",
|
||||
"size_bytes": 374
|
||||
},
|
||||
{
|
||||
"path": "human-review-summary.json",
|
||||
"sha256": "1ff22096bccb03ca5188d53bfe5f5d4a8ac937fdb43265c20b676f9211d1abbd",
|
||||
"size_bytes": 201
|
||||
},
|
||||
{
|
||||
"path": "input-manifest.json",
|
||||
"sha256": "44f1c81d76baf4dc965afc2cd109ac165ae8fe0308f202018ea21bac786bcd0b",
|
||||
"size_bytes": 6092
|
||||
},
|
||||
{
|
||||
"path": "latency-and-reliability.json",
|
||||
"sha256": "7c89bc0562b99565f6b1c54eac562f129192481ac61571ab48fb69a1f8553f95",
|
||||
"size_bytes": 263
|
||||
},
|
||||
{
|
||||
"path": "leakage-gate-report.json",
|
||||
"sha256": "ea532fc95b4e3be30299b6ec1939a42166c8107214b4264208c1a10c2abc0ff8",
|
||||
"size_bytes": 1221
|
||||
},
|
||||
{
|
||||
"path": "metric-report.json",
|
||||
"sha256": "6c92f31c1b0363d031e03968a50985e4e27075dc33ef03e9ce5e6a7fbb30c4f0",
|
||||
"size_bytes": 403544
|
||||
},
|
||||
{
|
||||
"path": "object-metrics.json",
|
||||
"sha256": "30cb52b1ee18a75bb68304ec6b88f54455a53e68052392232dec370ac3a68a10",
|
||||
"size_bytes": 24826
|
||||
},
|
||||
{
|
||||
"path": "protected-split-manifest.json",
|
||||
"sha256": "f294af0b2957af7de6ec533e4495d9d0d75da86cc0d29e265ed6998104829c64",
|
||||
"size_bytes": 20986
|
||||
},
|
||||
{
|
||||
"path": "reference-implementation-baseline.json",
|
||||
"sha256": "1c8fd9970d91449842922ba38f733da9d99142ce5f2a03736d14f9f61ad849a6",
|
||||
"size_bytes": 4710
|
||||
},
|
||||
{
|
||||
"path": "release-gate-report.json",
|
||||
"sha256": "e5f1c9c43c8da22acfa5486e641c010a278431f90634f91335b039ab7bdb50a4",
|
||||
"size_bytes": 5517
|
||||
},
|
||||
{
|
||||
"path": "split-and-leakage-audit.json",
|
||||
"sha256": "ea532fc95b4e3be30299b6ec1939a42166c8107214b4264208c1a10c2abc0ff8",
|
||||
"size_bytes": 1221
|
||||
},
|
||||
{
|
||||
"path": "stratified-metrics.json",
|
||||
"sha256": "66f7b7ed639cef400e52cada5d2274f8c185616b4beb6b44bcc89a8aa9929048",
|
||||
"size_bytes": 326587
|
||||
},
|
||||
{
|
||||
"path": "tile-metrics.json",
|
||||
"sha256": "0920c5d0b2076ed1ebcc06144f4cf786c3a18d1d67f9e388314f7f9ccb857256",
|
||||
"size_bytes": 6344
|
||||
},
|
||||
{
|
||||
"path": "workflow-summary.json",
|
||||
"sha256": "bcb4469fcaf4c360ed716037484117edf4e93c6b79bbdcf7f75cd3f79ca093a6",
|
||||
"size_bytes": 1035
|
||||
}
|
||||
],
|
||||
"benchmark_id": "geointel-p4-reference-harness-v2",
|
||||
"claim_boundary": "Immutable local reference-harness evidence only; product accuracy and release remain blocked while product gates are not evaluable or fail.",
|
||||
"phase": "P4",
|
||||
"schema_version": 2
|
||||
}
|
||||
+537
@@ -0,0 +1,537 @@
|
||||
{
|
||||
"failure_count": 14,
|
||||
"items": [
|
||||
{
|
||||
"contexts": [],
|
||||
"error_code": "P-PARTIAL",
|
||||
"evidence": {
|
||||
"missing_index": 3
|
||||
},
|
||||
"failure_id": "1397377fd37cd400f01a",
|
||||
"kind": "terrain_missing",
|
||||
"metadata": {
|
||||
"context": "terrain_profile",
|
||||
"date": "2023-01-01",
|
||||
"difficulty": "normal",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Hasselt",
|
||||
"object_size": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 5,
|
||||
"season": "not_applicable",
|
||||
"sensor": "elevation-raster",
|
||||
"source": "fixture-dhmv",
|
||||
"tile_edge": false,
|
||||
"urbanity": "rural",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"sample_id": "terrain-test-flanders-rural",
|
||||
"secondary_error_codes": [],
|
||||
"task": "terrain_interpretation"
|
||||
},
|
||||
{
|
||||
"contexts": [],
|
||||
"error_code": "M-FN-MISSED",
|
||||
"evidence": {
|
||||
"bbox": [
|
||||
10,
|
||||
0,
|
||||
15,
|
||||
5
|
||||
],
|
||||
"class": "removed",
|
||||
"id": "r2"
|
||||
},
|
||||
"failure_id": "28a0be1f9d4960036f3a",
|
||||
"kind": "event_false_negative",
|
||||
"metadata": {
|
||||
"context": "industrial_change",
|
||||
"date": "2024-05-01/2025-05-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Liège",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "wallonia",
|
||||
"resolution_m": 0.5,
|
||||
"season": "multi-date",
|
||||
"sensor": "vector",
|
||||
"source": "fixture-picc-temporal",
|
||||
"tile_edge": false,
|
||||
"urbanity": "industrial",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"sample_id": "change-test-wallonia-industrial",
|
||||
"secondary_error_codes": [],
|
||||
"task": "change_detection"
|
||||
},
|
||||
{
|
||||
"contexts": [],
|
||||
"error_code": "M-AREA-BIAS",
|
||||
"evidence": {
|
||||
"prediction_id": "p1",
|
||||
"reference_id": "r1",
|
||||
"relative_area_error": -0.1
|
||||
},
|
||||
"failure_id": "2d53b69d95b17bf49580",
|
||||
"kind": "area_bias",
|
||||
"metadata": {
|
||||
"context": "rural_buildings",
|
||||
"date": "2025-04-20",
|
||||
"difficulty": "normal",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Namur",
|
||||
"object_size": "large",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "wallonia",
|
||||
"resolution_m": 0.5,
|
||||
"season": "spring",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-picc",
|
||||
"tile_edge": false,
|
||||
"urbanity": "rural",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"sample_id": "seg-test-wallonia-rural",
|
||||
"secondary_error_codes": [],
|
||||
"task": "footprint_segmentation"
|
||||
},
|
||||
{
|
||||
"contexts": [
|
||||
"tile_edge"
|
||||
],
|
||||
"error_code": "M-CLASS",
|
||||
"evidence": {
|
||||
"column": 1,
|
||||
"pixel_index": 9,
|
||||
"prediction": 1,
|
||||
"reason": "class_mismatch",
|
||||
"reference": 2,
|
||||
"row": 2
|
||||
},
|
||||
"failure_id": "2fabc83865f9e1f49efc",
|
||||
"kind": "raster_misclassification",
|
||||
"metadata": {
|
||||
"context": "dense_urban_raster",
|
||||
"date": "2025-10-05",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Brussel",
|
||||
"object_size": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "brussels",
|
||||
"resolution_m": 1,
|
||||
"season": "autumn",
|
||||
"sensor": "synthetic-multispectral",
|
||||
"source": "fixture-thematic-raster",
|
||||
"tile_edge": true,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"sample_id": "raster-test-brussels-urban",
|
||||
"secondary_error_codes": [],
|
||||
"task": "raster_classification"
|
||||
},
|
||||
{
|
||||
"contexts": [
|
||||
"high_confidence",
|
||||
"tile_edge"
|
||||
],
|
||||
"error_code": "M-FP-CONFUSER",
|
||||
"evidence": {
|
||||
"bbox": [
|
||||
20,
|
||||
20,
|
||||
35,
|
||||
35
|
||||
],
|
||||
"class": "building",
|
||||
"confidence": 0.92,
|
||||
"id": "p-hard-fp"
|
||||
},
|
||||
"failure_id": "39eae35c6660b1ac1bcd",
|
||||
"kind": "false_positive",
|
||||
"metadata": {
|
||||
"context": "coastal_hard_negative",
|
||||
"date": "2025-07-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Oostende",
|
||||
"object_size": "empty",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-orthophoto",
|
||||
"tile_edge": true,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"sample_id": "background-test-hard-negative",
|
||||
"secondary_error_codes": [
|
||||
"M-MISCALIBRATED"
|
||||
],
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"contexts": [],
|
||||
"error_code": "D-VALIDATION-FN",
|
||||
"evidence": {
|
||||
"anomaly": "D-MISSING-PROVENANCE",
|
||||
"severity": "critical"
|
||||
},
|
||||
"failure_id": "40e33e21c399cc657f26",
|
||||
"kind": "validation_false_negative",
|
||||
"metadata": {
|
||||
"context": "data_contract",
|
||||
"date": "2025-01-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "not_applicable",
|
||||
"object_size": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "national",
|
||||
"resolution_m": null,
|
||||
"season": "not_applicable",
|
||||
"sensor": "mixed",
|
||||
"source": "fixture-contracts",
|
||||
"tile_edge": false,
|
||||
"urbanity": "mixed",
|
||||
"vegetation": "mixed"
|
||||
},
|
||||
"sample_id": "validation-test-national",
|
||||
"secondary_error_codes": [],
|
||||
"task": "geospatial_data_validation"
|
||||
},
|
||||
{
|
||||
"contexts": [],
|
||||
"error_code": "M-FP-CONFUSER",
|
||||
"evidence": {
|
||||
"bbox": [
|
||||
20,
|
||||
0,
|
||||
25,
|
||||
5
|
||||
],
|
||||
"class": "removed",
|
||||
"confidence": 0.65,
|
||||
"id": "p2"
|
||||
},
|
||||
"failure_id": "49560fb2dcb347c4cff5",
|
||||
"kind": "event_false_positive",
|
||||
"metadata": {
|
||||
"context": "industrial_change",
|
||||
"date": "2024-05-01/2025-05-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Liège",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "wallonia",
|
||||
"resolution_m": 0.5,
|
||||
"season": "multi-date",
|
||||
"sensor": "vector",
|
||||
"source": "fixture-picc-temporal",
|
||||
"tile_edge": false,
|
||||
"urbanity": "industrial",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"sample_id": "change-test-wallonia-industrial",
|
||||
"secondary_error_codes": [],
|
||||
"task": "change_detection"
|
||||
},
|
||||
{
|
||||
"contexts": [],
|
||||
"error_code": "M-MISCALIBRATED",
|
||||
"evidence": {
|
||||
"brier": 0.15333333333333332,
|
||||
"claim_boundary": "diagnostic deviation only; not a representative population-calibration claim",
|
||||
"ece": 0.13333333333333325,
|
||||
"prediction_support": 3
|
||||
},
|
||||
"failure_id": "8d4113e9688207b68f98",
|
||||
"kind": "miscalibrated",
|
||||
"metadata": {
|
||||
"context": "dense_urban",
|
||||
"date": "2025-06-15",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Antwerpen",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "partial",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-orthophoto",
|
||||
"tile_edge": false,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"sample_id": "det-test-flanders-urban",
|
||||
"secondary_error_codes": [],
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"contexts": [
|
||||
"tile_edge"
|
||||
],
|
||||
"error_code": "M-CLASS",
|
||||
"evidence": {
|
||||
"column": 3,
|
||||
"pixel_index": 3,
|
||||
"prediction": 2,
|
||||
"reason": "class_mismatch",
|
||||
"reference": 1,
|
||||
"row": 0
|
||||
},
|
||||
"failure_id": "a7c91430a7766b02755e",
|
||||
"kind": "raster_misclassification",
|
||||
"metadata": {
|
||||
"context": "dense_urban_raster",
|
||||
"date": "2025-10-05",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Brussel",
|
||||
"object_size": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "brussels",
|
||||
"resolution_m": 1,
|
||||
"season": "autumn",
|
||||
"sensor": "synthetic-multispectral",
|
||||
"source": "fixture-thematic-raster",
|
||||
"tile_edge": true,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"sample_id": "raster-test-brussels-urban",
|
||||
"secondary_error_codes": [],
|
||||
"task": "raster_classification"
|
||||
},
|
||||
{
|
||||
"contexts": [],
|
||||
"error_code": "M-FP-CONFUSER",
|
||||
"evidence": {
|
||||
"bbox": [
|
||||
85,
|
||||
85,
|
||||
95,
|
||||
95
|
||||
],
|
||||
"class": "building",
|
||||
"confidence": 0.6,
|
||||
"id": "p3"
|
||||
},
|
||||
"failure_id": "b7e154a0f6ddb9d8653f",
|
||||
"kind": "false_positive",
|
||||
"metadata": {
|
||||
"context": "dense_urban",
|
||||
"date": "2025-06-15",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Antwerpen",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "partial",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-orthophoto",
|
||||
"tile_edge": false,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"sample_id": "det-test-flanders-urban",
|
||||
"secondary_error_codes": [],
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"contexts": [],
|
||||
"error_code": "M-FP-CONFUSER",
|
||||
"evidence": {
|
||||
"class": "building",
|
||||
"id": "p2",
|
||||
"polygon": [
|
||||
[
|
||||
20,
|
||||
0
|
||||
],
|
||||
[
|
||||
24,
|
||||
0
|
||||
],
|
||||
[
|
||||
24,
|
||||
4
|
||||
],
|
||||
[
|
||||
20,
|
||||
4
|
||||
],
|
||||
[
|
||||
20,
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
"failure_id": "bf43633497be5101cddb",
|
||||
"kind": "false_positive",
|
||||
"metadata": {
|
||||
"context": "suburban_buildings",
|
||||
"date": "2025-01-15",
|
||||
"difficulty": "normal",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Mol",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.1,
|
||||
"season": "not_applicable",
|
||||
"sensor": "vector",
|
||||
"source": "fixture-grb",
|
||||
"tile_edge": false,
|
||||
"urbanity": "suburban",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"sample_id": "vector-test-flanders-suburban",
|
||||
"secondary_error_codes": [],
|
||||
"task": "vector_comparison"
|
||||
},
|
||||
{
|
||||
"contexts": [],
|
||||
"error_code": "M-FN-MISSED",
|
||||
"evidence": {
|
||||
"class": "building",
|
||||
"id": "r2",
|
||||
"polygon": [
|
||||
[
|
||||
10,
|
||||
0
|
||||
],
|
||||
[
|
||||
14,
|
||||
0
|
||||
],
|
||||
[
|
||||
14,
|
||||
4
|
||||
],
|
||||
[
|
||||
10,
|
||||
4
|
||||
],
|
||||
[
|
||||
10,
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
"failure_id": "c9a80d5bb196dc7113e2",
|
||||
"kind": "false_negative",
|
||||
"metadata": {
|
||||
"context": "suburban_buildings",
|
||||
"date": "2025-01-15",
|
||||
"difficulty": "normal",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Mol",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.1,
|
||||
"season": "not_applicable",
|
||||
"sensor": "vector",
|
||||
"source": "fixture-grb",
|
||||
"tile_edge": false,
|
||||
"urbanity": "suburban",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"sample_id": "vector-test-flanders-suburban",
|
||||
"secondary_error_codes": [],
|
||||
"task": "vector_comparison"
|
||||
},
|
||||
{
|
||||
"contexts": [],
|
||||
"error_code": "D-VALIDATION-FP",
|
||||
"evidence": {
|
||||
"anomaly": "D-RESOLUTION",
|
||||
"severity": "major"
|
||||
},
|
||||
"failure_id": "e00a10875bea399c62c3",
|
||||
"kind": "validation_false_positive",
|
||||
"metadata": {
|
||||
"context": "data_contract",
|
||||
"date": "2025-01-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "not_applicable",
|
||||
"object_size": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "national",
|
||||
"resolution_m": null,
|
||||
"season": "not_applicable",
|
||||
"sensor": "mixed",
|
||||
"source": "fixture-contracts",
|
||||
"tile_edge": false,
|
||||
"urbanity": "mixed",
|
||||
"vegetation": "mixed"
|
||||
},
|
||||
"sample_id": "validation-test-national",
|
||||
"secondary_error_codes": [],
|
||||
"task": "geospatial_data_validation"
|
||||
},
|
||||
{
|
||||
"contexts": [
|
||||
"tile_edge"
|
||||
],
|
||||
"error_code": "M-MISCALIBRATED",
|
||||
"evidence": {
|
||||
"brier": 0.8464,
|
||||
"claim_boundary": "diagnostic deviation only; not a representative population-calibration claim",
|
||||
"ece": 0.92,
|
||||
"prediction_support": 1
|
||||
},
|
||||
"failure_id": "edae8689bdec601bca2b",
|
||||
"kind": "miscalibrated",
|
||||
"metadata": {
|
||||
"context": "coastal_hard_negative",
|
||||
"date": "2025-07-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Oostende",
|
||||
"object_size": "empty",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-orthophoto",
|
||||
"tile_edge": true,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"sample_id": "background-test-hard-negative",
|
||||
"secondary_error_codes": [],
|
||||
"task": "object_detection"
|
||||
}
|
||||
],
|
||||
"items_canonical_json_sha256": "2002025055588195db73e20017882e412e988f3c55dedc89a1fd0c1dcd7a7e3e",
|
||||
"rendering_status": "machine_readable_examples_retained; a visual production gallery requires controlled access to protected imagery",
|
||||
"schema_version": 2,
|
||||
"taxonomy": "docs/accuracy-program/05-metric-framework.md section 4"
|
||||
}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"development_manifest_sha256": "64e4fc986c2390bedfbe100062d4b06e1098689f4d4a9a9da13a7f67357c0493",
|
||||
"generator_version": "1.3.0",
|
||||
"protected_manifest_sha256": "e57ca1529654ef1c2527e84122ff26ec3533e7c70658250983968fbc5d4af410",
|
||||
"schema_version": 1,
|
||||
"source_manifest_sha256": "1c75c875d4c297d046117bb9393ba5e648f13d3358ea92cb65374869c8f5fc01",
|
||||
"status": "pass"
|
||||
}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"ai_review_is_human_signoff": false,
|
||||
"required": null,
|
||||
"reviewed": null,
|
||||
"schema_version": 2,
|
||||
"source": "readiness-snapshot.json bound to Phase-1/3 evidence",
|
||||
"status": "not_evaluable"
|
||||
}
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
{
|
||||
"code": [
|
||||
{
|
||||
"path": "scripts/accuracy_phase4_evaluator.py",
|
||||
"sha256": "d2e40f7978d3ff4ce7025eaaaefb06ccc76c269d6874f8b0f528739d535a1da1",
|
||||
"size_bytes": 93353
|
||||
},
|
||||
{
|
||||
"path": "scripts/generate_accuracy_phase4_splits.py",
|
||||
"sha256": "d12842922bcd99b1672176df15a9cef250128e7d7fc662ab15e16f1c084ba570",
|
||||
"size_bytes": 75923
|
||||
},
|
||||
{
|
||||
"path": "scripts/run_accuracy_phase4_benchmark.py",
|
||||
"sha256": "003ecb4babc7f56a6d13e9e1c08dda902b9b26203cc7d53476d4d5e083b8b85b",
|
||||
"size_bytes": 123717
|
||||
},
|
||||
{
|
||||
"path": "scripts/run_golden_qa_benchmark.py",
|
||||
"sha256": "2c8a19fd92db8f07f491b8f7b34d9736384f81e02a6b83019fac71b101ebdaa0",
|
||||
"size_bytes": 10856
|
||||
},
|
||||
{
|
||||
"path": "backend/app/services/qa_service.py",
|
||||
"sha256": "2d4f645410fbf2972e55dddbf5a85db5368b2b1932792cd522ec51c483339244",
|
||||
"size_bytes": 15206
|
||||
}
|
||||
],
|
||||
"inputs": [
|
||||
{
|
||||
"path": "fixtures/accuracy/p4/split-source-manifest.json",
|
||||
"sha256": "27ec60406ac57a02f1335013ee344a43d4e0093e7ee1329ec22876da623f5bfc",
|
||||
"size_bytes": 17331
|
||||
},
|
||||
{
|
||||
"path": "fixtures/accuracy/p4/protected-baseline-cases.json",
|
||||
"sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473",
|
||||
"size_bytes": 20091
|
||||
},
|
||||
{
|
||||
"path": "fixtures/golden/golden_qa_benchmarks.json",
|
||||
"sha256": "d60576ff7048ca3a048dc166e8977f3a31e9f8fa352be5ad222a017260e179e2",
|
||||
"size_bytes": 2758
|
||||
},
|
||||
{
|
||||
"path": "docs/accuracy-program/05-metric-framework.md",
|
||||
"sha256": "18abfc21964b7d98848d8ae513afce242184980c9517c9184d40efd9165ea00d",
|
||||
"size_bytes": 28053
|
||||
},
|
||||
{
|
||||
"path": "docs/accuracy-program/07-source-authority-matrix.md",
|
||||
"sha256": "42fc59ec012b3d4eff5674ba347b11fb418a42b0178a12c9de16eac873ef6a9c",
|
||||
"size_bytes": 30807
|
||||
},
|
||||
{
|
||||
"path": "artifacts/evidence/accuracy/P3/full-scan-manifest.json",
|
||||
"sha256": "28c1022d125f30bfb4aa362b57d8a90cf8c88270b2a9ac6582b86b6a44a08038",
|
||||
"size_bytes": 755527
|
||||
},
|
||||
{
|
||||
"path": "artifacts/evidence/accuracy/P3/leakage-report.json",
|
||||
"sha256": "8e1b7a24a840549d442bc03132ee1f21c6d50f8714cbb8d33430391a3769cd4e",
|
||||
"size_bytes": 580
|
||||
}
|
||||
],
|
||||
"model_execution": {
|
||||
"configured_active_model": {
|
||||
"model_id": "yolo-configured",
|
||||
"model_version": "",
|
||||
"nationally_validated": false,
|
||||
"path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt",
|
||||
"sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1",
|
||||
"size_bytes": 22516074,
|
||||
"validated_area_names": [
|
||||
"Mol",
|
||||
"Kempen"
|
||||
]
|
||||
},
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"product_baseline": {
|
||||
"artifacts": [],
|
||||
"manifest": null,
|
||||
"product_gate_evidence_sha256": "c9f5e3de3ab6d64911f8807b26caa661ccf911a5fd6b995107edda2004836e0c",
|
||||
"requested_path": "artifacts/evidence/accuracy/P4/product-baseline-manifest.json",
|
||||
"validation_status": "not_evaluable"
|
||||
},
|
||||
"readiness_snapshot": {
|
||||
"active_model": {
|
||||
"model_id": "yolo-configured",
|
||||
"model_version": "",
|
||||
"nationally_validated": false,
|
||||
"path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt",
|
||||
"sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1",
|
||||
"size_bytes": 22516074,
|
||||
"validated_area_names": [
|
||||
"Mol",
|
||||
"Kempen"
|
||||
]
|
||||
},
|
||||
"authority_requirements": [
|
||||
{
|
||||
"primary": "grb",
|
||||
"task": "building_validation",
|
||||
"zone": "flanders"
|
||||
},
|
||||
{
|
||||
"primary": "picc",
|
||||
"task": "building_validation",
|
||||
"zone": "wallonia"
|
||||
},
|
||||
{
|
||||
"primary": "urbis",
|
||||
"task": "building_validation",
|
||||
"zone": "brussels"
|
||||
},
|
||||
{
|
||||
"primary": "dhmv",
|
||||
"task": "terrain_height",
|
||||
"zone": "flanders"
|
||||
},
|
||||
{
|
||||
"primary": "spw_terrain",
|
||||
"task": "terrain_height",
|
||||
"zone": "wallonia"
|
||||
},
|
||||
{
|
||||
"primary": "mdk",
|
||||
"task": "north_sea_bathymetry",
|
||||
"zone": "belgian_north_sea"
|
||||
},
|
||||
{
|
||||
"contextual": "sentinel-2",
|
||||
"primary": "official_orthophoto",
|
||||
"task": "imagery_corroboration",
|
||||
"zone": "belgium"
|
||||
}
|
||||
],
|
||||
"phase3_leakage_status": "attention",
|
||||
"phase3_scan": {
|
||||
"content_hash": null,
|
||||
"grb_consistency": {
|
||||
"reason": "No authoritative GRB snapshot was accessible in the configured local roots; no derived result was marked as GRB ground truth.",
|
||||
"status": "unavailable"
|
||||
},
|
||||
"scan_id": "p3-46ee3f8d3a2dc52b"
|
||||
},
|
||||
"schema_version": 1,
|
||||
"source_paths": {
|
||||
"accuracy_status_projection": {
|
||||
"path": "docs/accuracy-program/status.json",
|
||||
"selected_json_pointers": [
|
||||
"/runtime/active_model"
|
||||
],
|
||||
"sha256": "441dc0302156d8a1293fc330318bed5a39a6e6c558ee52dc9ada4965a5a516f6"
|
||||
},
|
||||
"phase3_full_scan": {
|
||||
"path": "artifacts/evidence/accuracy/P3/full-scan-manifest.json",
|
||||
"sha256": "28c1022d125f30bfb4aa362b57d8a90cf8c88270b2a9ac6582b86b6a44a08038",
|
||||
"size_bytes": 755527
|
||||
},
|
||||
"phase3_leakage": {
|
||||
"path": "artifacts/evidence/accuracy/P3/leakage-report.json",
|
||||
"sha256": "8e1b7a24a840549d442bc03132ee1f21c6d50f8714cbb8d33430391a3769cd4e",
|
||||
"size_bytes": 580
|
||||
}
|
||||
}
|
||||
},
|
||||
"readiness_snapshot_sha256": "5fab098e77e8537d95d83adeda5ec081106fea760b4b476d3517079c33d1cd9f",
|
||||
"repository_commit": "70fb4b94e5cb7c248beec5a936ce186f38cc183c",
|
||||
"runtime": {
|
||||
"cuda_used_for_reference_harness": false,
|
||||
"dependencies": {
|
||||
"numpy": "2.2.2",
|
||||
"pyproj": "3.7.2",
|
||||
"shapely": "2.1.2"
|
||||
},
|
||||
"execution_device": "CPU deterministic evaluator arithmetic; no production model inference",
|
||||
"platform": "Windows-11-10.0.26200-SP0",
|
||||
"python": "3.13.2",
|
||||
"python_implementation": "CPython"
|
||||
},
|
||||
"schema_version": 2
|
||||
}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"model_inference_executed": false,
|
||||
"reason": "The reference harness performs deterministic evaluator arithmetic only. GPU latency, VRAM, throughput and failure-rate gates require the real active model.",
|
||||
"schema_version": 2,
|
||||
"status": "not_evaluable"
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"checked_identity_fields": [
|
||||
"sample_id",
|
||||
"group_id",
|
||||
"source_family",
|
||||
"acquisition_date",
|
||||
"temporal_family",
|
||||
"object_ids",
|
||||
"native_feature_ids",
|
||||
"raw_image_sha256",
|
||||
"processed_image_sha256",
|
||||
"label_sha256",
|
||||
"record_sha256",
|
||||
"perceptual_image_hash",
|
||||
"label_geometry_hash",
|
||||
"label_geometry_fingerprint",
|
||||
"parent_raster_id",
|
||||
"acquisition_id",
|
||||
"bbox_distance"
|
||||
],
|
||||
"crs_validation": {
|
||||
"crs": "EPSG:31370",
|
||||
"distance_units": "m",
|
||||
"status": "pass"
|
||||
},
|
||||
"development_manifest_sha256": "64e4fc986c2390bedfbe100062d4b06e1098689f4d4a9a9da13a7f67357c0493",
|
||||
"finding_count": 0,
|
||||
"findings": [],
|
||||
"generator_version": "1.3.0",
|
||||
"independence_buffer_m": 2000.0,
|
||||
"inventory_total": 21,
|
||||
"label_geometry_hamming_threshold": 2,
|
||||
"perceptual_hamming_threshold": 4,
|
||||
"protected_manifest_sha256": "e57ca1529654ef1c2527e84122ff26ec3533e7c70658250983968fbc5d4af410",
|
||||
"schema_version": 1,
|
||||
"source_manifest_sha256": "1c75c875d4c297d046117bb9393ba5e648f13d3358ea92cb65374869c8f5fc01",
|
||||
"split_counts": {
|
||||
"background-test": 2,
|
||||
"calibration": 2,
|
||||
"challenge": 4,
|
||||
"test": 7,
|
||||
"train": 3,
|
||||
"val": 3
|
||||
},
|
||||
"status": "pass"
|
||||
}
|
||||
+10635
File diff suppressed because it is too large
Load Diff
+806
@@ -0,0 +1,806 @@
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"metadata": {
|
||||
"context": "coastal_hard_negative",
|
||||
"date": "2025-07-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Oostende",
|
||||
"object_size": "empty",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-orthophoto",
|
||||
"tile_edge": true,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"metrics": {
|
||||
"ap50": null,
|
||||
"ap50_95": null,
|
||||
"calibration": {
|
||||
"binning": "five fixed equal-width bins",
|
||||
"bins": [
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.0,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.2
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.2,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.4
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.4,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.6
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.6,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.8
|
||||
},
|
||||
{
|
||||
"accuracy": 0,
|
||||
"count": 1,
|
||||
"lower": 0.8,
|
||||
"mean_confidence": 0.92,
|
||||
"upper": 1.0
|
||||
}
|
||||
],
|
||||
"brier": 0.8464,
|
||||
"ece": 0.92,
|
||||
"status": "computed"
|
||||
},
|
||||
"coverage_risk": [
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.0,
|
||||
"total_prediction_count": 1
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.5,
|
||||
"total_prediction_count": 1
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.7,
|
||||
"total_prediction_count": 1
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 1.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.9,
|
||||
"total_prediction_count": 1
|
||||
}
|
||||
],
|
||||
"f1": null,
|
||||
"false_discovery_rate": 1.0,
|
||||
"false_negative": 0,
|
||||
"false_positive": 1,
|
||||
"map50": null,
|
||||
"map50_95": null,
|
||||
"match_iou": 0.5,
|
||||
"matched_iou": {
|
||||
"count": 0,
|
||||
"max": null,
|
||||
"mean": null,
|
||||
"median": null,
|
||||
"min": null
|
||||
},
|
||||
"miss_rate": null,
|
||||
"operating_confidence": 0.5,
|
||||
"per_class": {
|
||||
"building": {
|
||||
"ap50": null,
|
||||
"ap50_95": null,
|
||||
"f1": null,
|
||||
"false_discovery_rate": 1.0,
|
||||
"false_negative": 0,
|
||||
"false_positive": 1,
|
||||
"miss_rate": null,
|
||||
"precision": 0.0,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.0,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 0.7934506856227626
|
||||
},
|
||||
"prediction_count": 1,
|
||||
"recall": null,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"reference_count": 0,
|
||||
"true_positive": 0
|
||||
}
|
||||
},
|
||||
"precision": 0.0,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.0,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 0.7934506856227626
|
||||
},
|
||||
"prediction_count": 1,
|
||||
"recall": null,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"reference_count": 0,
|
||||
"true_positive": 0
|
||||
},
|
||||
"sample_id": "background-test-hard-negative",
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "pure_background",
|
||||
"date": "2025-02-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Brussel",
|
||||
"object_size": "empty",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "brussels",
|
||||
"resolution_m": 0.25,
|
||||
"season": "winter",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-orthophoto",
|
||||
"tile_edge": false,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"metrics": {
|
||||
"ap50": null,
|
||||
"ap50_95": null,
|
||||
"calibration": {
|
||||
"bins": [],
|
||||
"brier": null,
|
||||
"ece": null,
|
||||
"status": "undefined"
|
||||
},
|
||||
"coverage_risk": [
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.0,
|
||||
"total_prediction_count": 0
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.5,
|
||||
"total_prediction_count": 0
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.7,
|
||||
"total_prediction_count": 0
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 0,
|
||||
"reference_count": 0,
|
||||
"reference_coverage": null,
|
||||
"retained_prediction_count": 0,
|
||||
"retained_prediction_coverage": null,
|
||||
"risk": null,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.9,
|
||||
"total_prediction_count": 0
|
||||
}
|
||||
],
|
||||
"f1": null,
|
||||
"false_discovery_rate": null,
|
||||
"false_negative": 0,
|
||||
"false_positive": 0,
|
||||
"map50": null,
|
||||
"map50_95": null,
|
||||
"match_iou": 0.5,
|
||||
"matched_iou": {
|
||||
"count": 0,
|
||||
"max": null,
|
||||
"mean": null,
|
||||
"median": null,
|
||||
"min": null
|
||||
},
|
||||
"miss_rate": null,
|
||||
"operating_confidence": 0.5,
|
||||
"per_class": {
|
||||
"building": {
|
||||
"ap50": null,
|
||||
"ap50_95": null,
|
||||
"f1": null,
|
||||
"false_discovery_rate": null,
|
||||
"false_negative": 0,
|
||||
"false_positive": 0,
|
||||
"miss_rate": null,
|
||||
"precision": null,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"prediction_count": 0,
|
||||
"recall": null,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"reference_count": 0,
|
||||
"true_positive": 0
|
||||
}
|
||||
},
|
||||
"precision": null,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"prediction_count": 0,
|
||||
"recall": null,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": null,
|
||||
"status": "undefined",
|
||||
"support": 0,
|
||||
"upper": null
|
||||
},
|
||||
"reference_count": 0,
|
||||
"true_positive": 0
|
||||
},
|
||||
"sample_id": "background-test-pure-empty",
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "industrial_change",
|
||||
"date": "2024-05-01/2025-05-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Liège",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "wallonia",
|
||||
"resolution_m": 0.5,
|
||||
"season": "multi-date",
|
||||
"sensor": "vector",
|
||||
"source": "fixture-picc-temporal",
|
||||
"tile_edge": false,
|
||||
"urbanity": "industrial",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"metrics": {
|
||||
"event_metrics": {
|
||||
"added": {
|
||||
"f1": 1.0,
|
||||
"false_discovery_rate": 0.0,
|
||||
"false_negative": 0,
|
||||
"false_positive": 0,
|
||||
"miss_rate": 0.0,
|
||||
"precision": 1.0,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.20654931437723745,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 1.0
|
||||
},
|
||||
"prediction_count": 1,
|
||||
"recall": 1.0,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.20654931437723745,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 1.0
|
||||
},
|
||||
"reference_count": 1,
|
||||
"true_positive": 1
|
||||
},
|
||||
"removed": {
|
||||
"f1": null,
|
||||
"false_discovery_rate": 1.0,
|
||||
"false_negative": 1,
|
||||
"false_positive": 1,
|
||||
"miss_rate": 1.0,
|
||||
"precision": 0.0,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.0,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 0.7934506856227626
|
||||
},
|
||||
"prediction_count": 1,
|
||||
"recall": 0.0,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.0,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 0.7934506856227626
|
||||
},
|
||||
"reference_count": 1,
|
||||
"true_positive": 0
|
||||
}
|
||||
},
|
||||
"f1": 0.5,
|
||||
"false_discovery_rate": 0.5,
|
||||
"false_negative": 1,
|
||||
"false_positive": 1,
|
||||
"miss_rate": 0.5,
|
||||
"precision": 0.5,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.09453120573423074,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 0.9054687942657693
|
||||
},
|
||||
"prediction_count": 2,
|
||||
"recall": 0.5,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.09453120573423074,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 0.9054687942657693
|
||||
},
|
||||
"reference_count": 2,
|
||||
"true_positive": 1
|
||||
},
|
||||
"sample_id": "change-test-wallonia-industrial",
|
||||
"task": "change_detection"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "dense_urban",
|
||||
"date": "2025-06-15",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Antwerpen",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "partial",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-orthophoto",
|
||||
"tile_edge": false,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"metrics": {
|
||||
"ap50": 1.0,
|
||||
"ap50_95": 0.900990099009901,
|
||||
"calibration": {
|
||||
"binning": "five fixed equal-width bins",
|
||||
"bins": [
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.0,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.2
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.2,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.4
|
||||
},
|
||||
{
|
||||
"accuracy": null,
|
||||
"count": 0,
|
||||
"lower": 0.4,
|
||||
"mean_confidence": null,
|
||||
"upper": 0.6
|
||||
},
|
||||
{
|
||||
"accuracy": 0.5,
|
||||
"count": 2,
|
||||
"lower": 0.6,
|
||||
"mean_confidence": 0.6499999999999999,
|
||||
"upper": 0.8
|
||||
},
|
||||
{
|
||||
"accuracy": 1,
|
||||
"count": 1,
|
||||
"lower": 0.8,
|
||||
"mean_confidence": 0.9,
|
||||
"upper": 1.0
|
||||
}
|
||||
],
|
||||
"brier": 0.15333333333333332,
|
||||
"ece": 0.13333333333333325,
|
||||
"status": "computed"
|
||||
},
|
||||
"coverage_risk": [
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 2,
|
||||
"matched_reference_count": 2,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 1.0,
|
||||
"retained_prediction_count": 4,
|
||||
"retained_prediction_coverage": 1.0,
|
||||
"risk": 0.5,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.0,
|
||||
"total_prediction_count": 4
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 1,
|
||||
"matched_reference_count": 2,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 1.0,
|
||||
"retained_prediction_count": 3,
|
||||
"retained_prediction_coverage": 0.75,
|
||||
"risk": 0.3333333333333333,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.5,
|
||||
"total_prediction_count": 4
|
||||
},
|
||||
{
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 2,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 1.0,
|
||||
"retained_prediction_count": 2,
|
||||
"retained_prediction_coverage": 0.5,
|
||||
"risk": 0.0,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.7,
|
||||
"total_prediction_count": 4
|
||||
},
|
||||
{
|
||||
"false_negative_count": 1,
|
||||
"false_positive_count": 0,
|
||||
"matched_reference_count": 1,
|
||||
"reference_count": 2,
|
||||
"reference_coverage": 0.5,
|
||||
"retained_prediction_count": 1,
|
||||
"retained_prediction_coverage": 0.25,
|
||||
"risk": 0.5,
|
||||
"risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)",
|
||||
"threshold": 0.9,
|
||||
"total_prediction_count": 4
|
||||
}
|
||||
],
|
||||
"f1": 0.8,
|
||||
"false_discovery_rate": 0.3333333333333333,
|
||||
"false_negative": 0,
|
||||
"false_positive": 1,
|
||||
"map50": 1.0,
|
||||
"map50_95": 0.900990099009901,
|
||||
"match_iou": 0.5,
|
||||
"matched_iou": {
|
||||
"count": 2,
|
||||
"max": 1.0,
|
||||
"mean": 0.9355555555555555,
|
||||
"median": 0.9355555555555555,
|
||||
"min": 0.8711111111111111
|
||||
},
|
||||
"miss_rate": 0.0,
|
||||
"operating_confidence": 0.5,
|
||||
"per_class": {
|
||||
"building": {
|
||||
"ap50": 1.0,
|
||||
"ap50_95": 0.900990099009901,
|
||||
"f1": 0.8,
|
||||
"false_discovery_rate": 0.3333333333333333,
|
||||
"false_negative": 0,
|
||||
"false_positive": 1,
|
||||
"miss_rate": 0.0,
|
||||
"precision": 0.6666666666666666,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.2076596008020477,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 3,
|
||||
"upper": 0.9385080552796037
|
||||
},
|
||||
"prediction_count": 3,
|
||||
"recall": 1.0,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.34238022750665303,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 1.0
|
||||
},
|
||||
"reference_count": 2,
|
||||
"true_positive": 2
|
||||
}
|
||||
},
|
||||
"precision": 0.6666666666666666,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.2076596008020477,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 3,
|
||||
"upper": 0.9385080552796037
|
||||
},
|
||||
"prediction_count": 3,
|
||||
"recall": 1.0,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.34238022750665303,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 1.0
|
||||
},
|
||||
"reference_count": 2,
|
||||
"true_positive": 2
|
||||
},
|
||||
"sample_id": "det-test-flanders-urban",
|
||||
"task": "object_detection"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "rural_buildings",
|
||||
"date": "2025-04-20",
|
||||
"difficulty": "normal",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Namur",
|
||||
"object_size": "large",
|
||||
"occlusion": "none",
|
||||
"ood": false,
|
||||
"region": "wallonia",
|
||||
"resolution_m": 0.5,
|
||||
"season": "spring",
|
||||
"sensor": "synthetic-rgb",
|
||||
"source": "fixture-picc",
|
||||
"tile_edge": false,
|
||||
"urbanity": "rural",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"metrics": {
|
||||
"centroid_distance_m": {
|
||||
"count": 1,
|
||||
"max": 0.5,
|
||||
"mean": 0.5,
|
||||
"median": 0.5,
|
||||
"min": 0.5
|
||||
},
|
||||
"f1": 1.0,
|
||||
"false_discovery_rate": 0.0,
|
||||
"false_negative": 0,
|
||||
"false_positive": 0,
|
||||
"mean_boundary_f1": 1.0,
|
||||
"mean_dice": 0.8881578947368421,
|
||||
"mean_iou": 0.7988165680473372,
|
||||
"miss_rate": 0.0,
|
||||
"precision": 1.0,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.20654931437723745,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 1.0
|
||||
},
|
||||
"prediction_count": 1,
|
||||
"recall": 1.0,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.20654931437723745,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 1,
|
||||
"upper": 1.0
|
||||
},
|
||||
"reference_count": 1,
|
||||
"relative_area_error": {
|
||||
"count": 1,
|
||||
"max": -0.1,
|
||||
"mean": -0.1,
|
||||
"median": -0.1,
|
||||
"min": -0.1
|
||||
},
|
||||
"spatial_context_validated": true,
|
||||
"topologically_invalid_predictions": 0,
|
||||
"topologically_valid_predictions": 1,
|
||||
"true_positive": 1
|
||||
},
|
||||
"sample_id": "seg-test-wallonia-rural",
|
||||
"task": "footprint_segmentation"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "data_contract",
|
||||
"date": "2025-01-01",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "not_applicable",
|
||||
"object_size": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "national",
|
||||
"resolution_m": null,
|
||||
"season": "not_applicable",
|
||||
"sensor": "mixed",
|
||||
"source": "fixture-contracts",
|
||||
"tile_edge": false,
|
||||
"urbanity": "mixed",
|
||||
"vegetation": "mixed"
|
||||
},
|
||||
"metrics": {
|
||||
"blocker_or_critical_miss_count": 1,
|
||||
"f1": 0.6666666666666666,
|
||||
"false_discovery_rate": 0.3333333333333333,
|
||||
"false_negative": 1,
|
||||
"false_positive": 1,
|
||||
"miss_rate": 0.3333333333333333,
|
||||
"misses_by_severity": {
|
||||
"blocker": 0,
|
||||
"critical": 1,
|
||||
"informational": 0,
|
||||
"major": 0,
|
||||
"minor": 0
|
||||
},
|
||||
"precision": 0.6666666666666666,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.2076596008020477,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 3,
|
||||
"upper": 0.9385080552796037
|
||||
},
|
||||
"prediction_count": 3,
|
||||
"recall": 0.6666666666666666,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.2076596008020477,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 3,
|
||||
"upper": 0.9385080552796037
|
||||
},
|
||||
"reference_count": 3,
|
||||
"severity_mismatch_count": 0,
|
||||
"severity_mismatches": [],
|
||||
"true_positive": 2
|
||||
},
|
||||
"sample_id": "validation-test-national",
|
||||
"task": "geospatial_data_validation"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "suburban_buildings",
|
||||
"date": "2025-01-15",
|
||||
"difficulty": "normal",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Mol",
|
||||
"object_size": "mixed",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 0.1,
|
||||
"season": "not_applicable",
|
||||
"sensor": "vector",
|
||||
"source": "fixture-grb",
|
||||
"tile_edge": false,
|
||||
"urbanity": "suburban",
|
||||
"vegetation": "low"
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.5,
|
||||
"false_discovery_rate": 0.5,
|
||||
"false_negative": 1,
|
||||
"false_positive": 1,
|
||||
"mean_iou": 1.0,
|
||||
"miss_rate": 0.5,
|
||||
"precision": 0.5,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.09453120573423074,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 0.9054687942657693
|
||||
},
|
||||
"prediction_count": 2,
|
||||
"recall": 0.5,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.09453120573423074,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 2,
|
||||
"upper": 0.9054687942657693
|
||||
},
|
||||
"reference_count": 2,
|
||||
"spatial_context_validated": true,
|
||||
"topologically_valid": true,
|
||||
"true_positive": 1
|
||||
},
|
||||
"sample_id": "vector-test-flanders-suburban",
|
||||
"task": "vector_comparison"
|
||||
}
|
||||
],
|
||||
"schema_version": 2,
|
||||
"status": "fixture_contract_only"
|
||||
}
|
||||
+589
@@ -0,0 +1,589 @@
|
||||
{
|
||||
"access_policy": "Only the frozen release evaluator may consume test/background-test labels; challenge labels remain external/sealed.",
|
||||
"allowed_splits": [
|
||||
"background-test",
|
||||
"challenge",
|
||||
"test"
|
||||
],
|
||||
"assignment_algorithm": "group-before-split-deficit-balancer-v1",
|
||||
"assignment_mode": "preassigned",
|
||||
"assignment_seed": "geointel-p4-group-split-v1",
|
||||
"claim_boundary": "Synthetic contract fixtures for evaluator regression only; never production accuracy evidence.",
|
||||
"crs": "EPSG:31370",
|
||||
"dataset_version": "geointel-p4-harness-fixture-v2",
|
||||
"generator_version": "1.3.0",
|
||||
"implemented_tasks": [
|
||||
"change_detection",
|
||||
"footprint_segmentation",
|
||||
"geospatial_data_validation",
|
||||
"object_detection",
|
||||
"raster_classification",
|
||||
"terrain_interpretation",
|
||||
"vector_comparison"
|
||||
],
|
||||
"independence_buffer_m": 2000.0,
|
||||
"label_geometry_hamming_threshold": 2,
|
||||
"labels_available_by_split": {
|
||||
"background-test": "frozen_evaluator_only",
|
||||
"challenge": "sealed_external",
|
||||
"test": "frozen_evaluator_only"
|
||||
},
|
||||
"mandatory_split_roles": [
|
||||
"train",
|
||||
"val",
|
||||
"calibration",
|
||||
"test",
|
||||
"background-test",
|
||||
"challenge"
|
||||
],
|
||||
"manifest_role": "protected_release_only",
|
||||
"manifest_sha256": "e57ca1529654ef1c2527e84122ff26ec3533e7c70658250983968fbc5d4af410",
|
||||
"perceptual_hamming_threshold": 4,
|
||||
"protected_task_coverage": [
|
||||
{
|
||||
"background_test_count": 0,
|
||||
"exemption_reason": null,
|
||||
"sealed_challenge_count": 0,
|
||||
"status": "covered",
|
||||
"task": "change_detection",
|
||||
"test_count": 1
|
||||
},
|
||||
{
|
||||
"background_test_count": 0,
|
||||
"exemption_reason": null,
|
||||
"sealed_challenge_count": 1,
|
||||
"status": "covered",
|
||||
"task": "footprint_segmentation",
|
||||
"test_count": 1
|
||||
},
|
||||
{
|
||||
"background_test_count": 0,
|
||||
"exemption_reason": null,
|
||||
"sealed_challenge_count": 1,
|
||||
"status": "covered",
|
||||
"task": "geospatial_data_validation",
|
||||
"test_count": 1
|
||||
},
|
||||
{
|
||||
"background_test_count": 2,
|
||||
"exemption_reason": null,
|
||||
"sealed_challenge_count": 1,
|
||||
"status": "covered",
|
||||
"task": "object_detection",
|
||||
"test_count": 1
|
||||
},
|
||||
{
|
||||
"background_test_count": 0,
|
||||
"exemption_reason": null,
|
||||
"sealed_challenge_count": 1,
|
||||
"status": "covered",
|
||||
"task": "raster_classification",
|
||||
"test_count": 1
|
||||
},
|
||||
{
|
||||
"background_test_count": 0,
|
||||
"exemption_reason": null,
|
||||
"sealed_challenge_count": 0,
|
||||
"status": "covered",
|
||||
"task": "terrain_interpretation",
|
||||
"test_count": 1
|
||||
},
|
||||
{
|
||||
"background_test_count": 0,
|
||||
"exemption_reason": null,
|
||||
"sealed_challenge_count": 0,
|
||||
"status": "covered",
|
||||
"task": "vector_comparison",
|
||||
"test_count": 1
|
||||
}
|
||||
],
|
||||
"samples": [
|
||||
{
|
||||
"acquisition_date": "2025-01-17",
|
||||
"acquisition_id": "acquisition-17",
|
||||
"bbox": [
|
||||
60000.0,
|
||||
70000.0,
|
||||
60100.0,
|
||||
70100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g17",
|
||||
"label_geometry_fingerprint": "2b405bcc9c95acfb",
|
||||
"label_geometry_hash": "7171717171717171717171717171717171717171717171717171717171717171",
|
||||
"label_sha256": "5151515151515151515151515151515151515151515151515151515151515151",
|
||||
"native_feature_ids": [],
|
||||
"object_ids": [],
|
||||
"parent_raster_id": "parent-raster-17",
|
||||
"perceptual_image_hash": "6ada1b5aa46cc261",
|
||||
"processed_image_sha256": "3131313131313131313131313131313131313131313131313131313131313131",
|
||||
"raw_image_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
|
||||
"record_sha256": "330f67df045e915670260db85cd399b7abb0a5277ca857e2b9483274dfd561c9",
|
||||
"sample_id": "background-test-hard-negative",
|
||||
"source_family": "scene-17",
|
||||
"split": "background-test",
|
||||
"task": "object_detection",
|
||||
"temporal_family": "temporal-17"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-16",
|
||||
"acquisition_id": "acquisition-16",
|
||||
"bbox": [
|
||||
20000.0,
|
||||
70000.0,
|
||||
20100.0,
|
||||
70100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g16",
|
||||
"label_geometry_fingerprint": "004e762f898c4fad",
|
||||
"label_geometry_hash": "7070707070707070707070707070707070707070707070707070707070707070",
|
||||
"label_sha256": "5050505050505050505050505050505050505050505050505050505050505050",
|
||||
"native_feature_ids": [],
|
||||
"object_ids": [],
|
||||
"parent_raster_id": "parent-raster-16",
|
||||
"perceptual_image_hash": "be4078e1ef9fa296",
|
||||
"processed_image_sha256": "3030303030303030303030303030303030303030303030303030303030303030",
|
||||
"raw_image_sha256": "1010101010101010101010101010101010101010101010101010101010101010",
|
||||
"record_sha256": "3e8066639f58f5230faea2b2964c532148f9fc6d891a1d04a127c1fac1cdb826",
|
||||
"sample_id": "background-test-pure-empty",
|
||||
"source_family": "scene-16",
|
||||
"split": "background-test",
|
||||
"task": "object_detection",
|
||||
"temporal_family": "temporal-16"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-13",
|
||||
"acquisition_id": "acquisition-13",
|
||||
"bbox": [
|
||||
50000.0,
|
||||
50000.0,
|
||||
50100.0,
|
||||
50100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g13",
|
||||
"label_geometry_fingerprint": "6c5b5f9db0d73c31",
|
||||
"label_geometry_hash": "6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d",
|
||||
"label_sha256": "4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d",
|
||||
"native_feature_ids": [
|
||||
"native-13-1",
|
||||
"native-13-2"
|
||||
],
|
||||
"object_ids": [
|
||||
"o13",
|
||||
"o14"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-13",
|
||||
"perceptual_image_hash": "14e347040ef16eac",
|
||||
"processed_image_sha256": "2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d",
|
||||
"raw_image_sha256": "0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d",
|
||||
"record_sha256": "0550facfe80b78f11414cd578d3808fab503382f15c9e6e1f8fd4801ce3ab365",
|
||||
"sample_id": "change-test-wallonia-industrial",
|
||||
"source_family": "scene-13",
|
||||
"split": "test",
|
||||
"task": "change_detection",
|
||||
"temporal_family": "temporal-13"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-18",
|
||||
"acquisition_id": "acquisition-18",
|
||||
"bbox": [
|
||||
10000.0,
|
||||
90000.0,
|
||||
10100.0,
|
||||
90100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g18",
|
||||
"parent_raster_id": "parent-raster-18",
|
||||
"perceptual_image_hash": "15910b3a2d056648",
|
||||
"processed_image_sha256": "3232323232323232323232323232323232323232323232323232323232323232",
|
||||
"raw_image_sha256": "1212121212121212121212121212121212121212121212121212121212121212",
|
||||
"sample_id": "det-challenge-coast",
|
||||
"sealed": true,
|
||||
"source_family": "scene-18",
|
||||
"split": "challenge",
|
||||
"task": "object_detection",
|
||||
"temporal_family": "temporal-18",
|
||||
"withheld_fields": [
|
||||
"content_path_bindings",
|
||||
"governance_binding",
|
||||
"label_geometry_fingerprint",
|
||||
"label_geometry_hash",
|
||||
"label_geometry_path",
|
||||
"label_path",
|
||||
"label_sha256",
|
||||
"native_feature_ids",
|
||||
"object_ids",
|
||||
"record_sha256"
|
||||
]
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-09",
|
||||
"acquisition_id": "acquisition-09",
|
||||
"bbox": [
|
||||
10000.0,
|
||||
50000.0,
|
||||
10100.0,
|
||||
50100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g09",
|
||||
"label_geometry_fingerprint": "7ed221e11cc6828d",
|
||||
"label_geometry_hash": "6969696969696969696969696969696969696969696969696969696969696969",
|
||||
"label_sha256": "4949494949494949494949494949494949494949494949494949494949494949",
|
||||
"native_feature_ids": [
|
||||
"native-09-1",
|
||||
"native-09-2"
|
||||
],
|
||||
"object_ids": [
|
||||
"o07",
|
||||
"o08"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-09",
|
||||
"perceptual_image_hash": "d6d20ed34050b89e",
|
||||
"processed_image_sha256": "2929292929292929292929292929292929292929292929292929292929292929",
|
||||
"raw_image_sha256": "0909090909090909090909090909090909090909090909090909090909090909",
|
||||
"record_sha256": "a551408577ce3494c75160c1c4fe90fbbc79a9f37bd151d7df8095d1cd35bc8b",
|
||||
"sample_id": "det-test-flanders-urban",
|
||||
"source_family": "scene-09",
|
||||
"split": "test",
|
||||
"task": "object_detection",
|
||||
"temporal_family": "temporal-09"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-20",
|
||||
"acquisition_id": "acquisition-20",
|
||||
"bbox": [
|
||||
50000.0,
|
||||
90000.0,
|
||||
50100.0,
|
||||
90100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g20",
|
||||
"parent_raster_id": "parent-raster-20",
|
||||
"perceptual_image_hash": "85f82f2f8eb2c2f5",
|
||||
"processed_image_sha256": "3434343434343434343434343434343434343434343434343434343434343434",
|
||||
"raw_image_sha256": "1414141414141414141414141414141414141414141414141414141414141414",
|
||||
"sample_id": "raster-challenge-seasonal",
|
||||
"sealed": true,
|
||||
"source_family": "scene-20",
|
||||
"split": "challenge",
|
||||
"task": "raster_classification",
|
||||
"temporal_family": "temporal-20",
|
||||
"withheld_fields": [
|
||||
"content_path_bindings",
|
||||
"governance_binding",
|
||||
"label_geometry_fingerprint",
|
||||
"label_geometry_hash",
|
||||
"label_geometry_path",
|
||||
"label_path",
|
||||
"label_sha256",
|
||||
"native_feature_ids",
|
||||
"object_ids",
|
||||
"record_sha256"
|
||||
]
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-11",
|
||||
"acquisition_id": "acquisition-11",
|
||||
"bbox": [
|
||||
30000.0,
|
||||
50000.0,
|
||||
30100.0,
|
||||
50100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g11",
|
||||
"label_geometry_fingerprint": "f4269ac17b100571",
|
||||
"label_geometry_hash": "6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b",
|
||||
"label_sha256": "4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b",
|
||||
"native_feature_ids": [
|
||||
"native-11-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o10"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-11",
|
||||
"perceptual_image_hash": "4383fa06c328f326",
|
||||
"processed_image_sha256": "2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b",
|
||||
"raw_image_sha256": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b",
|
||||
"record_sha256": "d66076d5cb479670bc8716334f526ddcbab07a2aacab28d7a5d6893795618d56",
|
||||
"sample_id": "raster-test-brussels-urban",
|
||||
"source_family": "scene-11",
|
||||
"split": "test",
|
||||
"task": "raster_classification",
|
||||
"temporal_family": "temporal-11"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-19",
|
||||
"acquisition_id": "acquisition-19",
|
||||
"bbox": [
|
||||
30000.0,
|
||||
90000.0,
|
||||
30100.0,
|
||||
90100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g19",
|
||||
"parent_raster_id": "parent-raster-19",
|
||||
"perceptual_image_hash": "b5c6e201d687cf9e",
|
||||
"processed_image_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
|
||||
"raw_image_sha256": "1313131313131313131313131313131313131313131313131313131313131313",
|
||||
"sample_id": "seg-challenge-brussels",
|
||||
"sealed": true,
|
||||
"source_family": "scene-19",
|
||||
"split": "challenge",
|
||||
"task": "footprint_segmentation",
|
||||
"temporal_family": "temporal-19",
|
||||
"withheld_fields": [
|
||||
"content_path_bindings",
|
||||
"governance_binding",
|
||||
"label_geometry_fingerprint",
|
||||
"label_geometry_hash",
|
||||
"label_geometry_path",
|
||||
"label_path",
|
||||
"label_sha256",
|
||||
"native_feature_ids",
|
||||
"object_ids",
|
||||
"record_sha256"
|
||||
]
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-10",
|
||||
"acquisition_id": "acquisition-10",
|
||||
"bbox": [
|
||||
20000.0,
|
||||
50000.0,
|
||||
20100.0,
|
||||
50100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g10",
|
||||
"label_geometry_fingerprint": "e9fb3c93de82d0be",
|
||||
"label_geometry_hash": "6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a",
|
||||
"label_sha256": "4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a",
|
||||
"native_feature_ids": [
|
||||
"native-10-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o09"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-10",
|
||||
"perceptual_image_hash": "6c476d18e0539687",
|
||||
"processed_image_sha256": "2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a",
|
||||
"raw_image_sha256": "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a",
|
||||
"record_sha256": "7d0f9ee51cdf98e7f3acdc8117d48ce16d4d97538ba8179949a2c87dbbe5a526",
|
||||
"sample_id": "seg-test-wallonia-rural",
|
||||
"source_family": "scene-10",
|
||||
"split": "test",
|
||||
"task": "footprint_segmentation",
|
||||
"temporal_family": "temporal-10"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-14",
|
||||
"acquisition_id": "acquisition-14",
|
||||
"bbox": [
|
||||
60000.0,
|
||||
50000.0,
|
||||
60100.0,
|
||||
50100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g14",
|
||||
"label_geometry_fingerprint": "4ec9f1dd6c799ec7",
|
||||
"label_geometry_hash": "6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e",
|
||||
"label_sha256": "4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e",
|
||||
"native_feature_ids": [
|
||||
"native-14-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o15"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-14",
|
||||
"perceptual_image_hash": "c808940b0a325e9c",
|
||||
"processed_image_sha256": "2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e",
|
||||
"raw_image_sha256": "0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e",
|
||||
"record_sha256": "4a47d90297a8325bb8257d7ca76b3281df8312d385715745a1a1fb74ba89efcd",
|
||||
"sample_id": "terrain-test-flanders-rural",
|
||||
"source_family": "scene-14",
|
||||
"split": "test",
|
||||
"task": "terrain_interpretation",
|
||||
"temporal_family": "temporal-14"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-21",
|
||||
"acquisition_id": "acquisition-21",
|
||||
"bbox": [
|
||||
70000.0,
|
||||
90000.0,
|
||||
70100.0,
|
||||
90100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g21",
|
||||
"parent_raster_id": "parent-raster-21",
|
||||
"perceptual_image_hash": "c71055ef6baeee8b",
|
||||
"processed_image_sha256": "3535353535353535353535353535353535353535353535353535353535353535",
|
||||
"raw_image_sha256": "1515151515151515151515151515151515151515151515151515151515151515",
|
||||
"sample_id": "validation-challenge-corrupt",
|
||||
"sealed": true,
|
||||
"source_family": "scene-21",
|
||||
"split": "challenge",
|
||||
"task": "geospatial_data_validation",
|
||||
"temporal_family": "temporal-21",
|
||||
"withheld_fields": [
|
||||
"content_path_bindings",
|
||||
"governance_binding",
|
||||
"label_geometry_fingerprint",
|
||||
"label_geometry_hash",
|
||||
"label_geometry_path",
|
||||
"label_path",
|
||||
"label_sha256",
|
||||
"native_feature_ids",
|
||||
"object_ids",
|
||||
"record_sha256"
|
||||
]
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-15",
|
||||
"acquisition_id": "acquisition-15",
|
||||
"bbox": [
|
||||
70000.0,
|
||||
50000.0,
|
||||
70100.0,
|
||||
50100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g15",
|
||||
"label_geometry_fingerprint": "616691796f62dbbb",
|
||||
"label_geometry_hash": "6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f",
|
||||
"label_sha256": "4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f",
|
||||
"native_feature_ids": [
|
||||
"native-15-1"
|
||||
],
|
||||
"object_ids": [
|
||||
"o16"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-15",
|
||||
"perceptual_image_hash": "4365e6b7df6acf3d",
|
||||
"processed_image_sha256": "2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f",
|
||||
"raw_image_sha256": "0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f",
|
||||
"record_sha256": "0ff60414da05655b19c1717a27d8ba09aa8539a2b6c143e7a1a72b634fca397a",
|
||||
"sample_id": "validation-test-national",
|
||||
"source_family": "scene-15",
|
||||
"split": "test",
|
||||
"task": "geospatial_data_validation",
|
||||
"temporal_family": "temporal-15"
|
||||
},
|
||||
{
|
||||
"acquisition_date": "2025-01-12",
|
||||
"acquisition_id": "acquisition-12",
|
||||
"bbox": [
|
||||
40000.0,
|
||||
50000.0,
|
||||
40100.0,
|
||||
50100.0
|
||||
],
|
||||
"content_hash_verification": {
|
||||
"label_geometry_hash": "synthetic_fixture_bound",
|
||||
"label_sha256": "synthetic_fixture_bound",
|
||||
"processed_image_sha256": "synthetic_fixture_bound",
|
||||
"raw_image_sha256": "synthetic_fixture_bound"
|
||||
},
|
||||
"group_id": "g12",
|
||||
"label_geometry_fingerprint": "2db7d069691934ed",
|
||||
"label_geometry_hash": "6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c",
|
||||
"label_sha256": "4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c",
|
||||
"native_feature_ids": [
|
||||
"native-12-1",
|
||||
"native-12-2"
|
||||
],
|
||||
"object_ids": [
|
||||
"o11",
|
||||
"o12"
|
||||
],
|
||||
"parent_raster_id": "parent-raster-12",
|
||||
"perceptual_image_hash": "1236e1aa148c25d5",
|
||||
"processed_image_sha256": "2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c",
|
||||
"raw_image_sha256": "0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c",
|
||||
"record_sha256": "63f561268e24208db0dcb4321cd66624188f2f66016338dac504d52051dae34a",
|
||||
"sample_id": "vector-test-flanders-suburban",
|
||||
"source_family": "scene-12",
|
||||
"split": "test",
|
||||
"task": "vector_comparison",
|
||||
"temporal_family": "temporal-12"
|
||||
}
|
||||
],
|
||||
"schema_version": 1,
|
||||
"selection_use_allowed": false,
|
||||
"source_manifest_sha256": "1c75c875d4c297d046117bb9393ba5e648f13d3358ea92cb65374869c8f5fc01",
|
||||
"source_trust": {
|
||||
"mode": "synthetic_fixture",
|
||||
"policy_id": "geointel-p4-synthetic-fixture-v2",
|
||||
"production_accuracy_use_allowed": false
|
||||
},
|
||||
"training_access_allowed": false
|
||||
}
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
{
|
||||
"claim_boundary": "Reference implementation regression evidence; not production model accuracy.",
|
||||
"content_sha256": "aeb0f6510352f4e205e4fc4e8c423110ca72c2142016c7af846d758865f258c6",
|
||||
"implementation": "backend/app/services/qa_service.py via scripts/run_golden_qa_benchmark.py",
|
||||
"persistence": {
|
||||
"commit_count": 4,
|
||||
"metric_count": 24,
|
||||
"metric_keys": [
|
||||
"f1",
|
||||
"false_negative_count",
|
||||
"false_positive_count",
|
||||
"mean_iou",
|
||||
"precision",
|
||||
"recall"
|
||||
],
|
||||
"quality_check_count": 4
|
||||
},
|
||||
"scenario_count": 4,
|
||||
"scenarios": [
|
||||
{
|
||||
"benchmark_id": "golden-buildings-partial-match-v1",
|
||||
"description": "Two reference building polygons and two candidate building polygons: one candidate matches ref-1, one candidate is a false positive, and ref-2 is a false negative.",
|
||||
"fixtures": {
|
||||
"candidate": "fixtures/golden/predicted_buildings.geojson",
|
||||
"reference": "fixtures/golden/reference_buildings.geojson"
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.5,
|
||||
"false_negative_count": 1,
|
||||
"false_positive_count": 1,
|
||||
"mean_iou": 0.8339768339652592,
|
||||
"precision": 0.5,
|
||||
"recall": 0.5
|
||||
},
|
||||
"persistence": {
|
||||
"commit_count": 1,
|
||||
"metric_count": 6,
|
||||
"metric_keys": [
|
||||
"precision",
|
||||
"recall",
|
||||
"f1",
|
||||
"mean_iou",
|
||||
"false_positive_count",
|
||||
"false_negative_count"
|
||||
],
|
||||
"quality_check_count": 1
|
||||
},
|
||||
"result_counts": {
|
||||
"candidate_feature_count": 2,
|
||||
"matches": 1,
|
||||
"reference_feature_count": 2
|
||||
},
|
||||
"status": "passed"
|
||||
},
|
||||
{
|
||||
"benchmark_id": "golden-buildings-perfect-match-v1",
|
||||
"description": "One candidate building polygon exactly matches one reference polygon.",
|
||||
"fixtures": {
|
||||
"candidate": "fixtures/golden/predicted_buildings_perfect.geojson",
|
||||
"reference": "fixtures/golden/reference_buildings_perfect.geojson"
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"mean_iou": 1.0,
|
||||
"precision": 1.0,
|
||||
"recall": 1.0
|
||||
},
|
||||
"persistence": {
|
||||
"commit_count": 1,
|
||||
"metric_count": 6,
|
||||
"metric_keys": [
|
||||
"precision",
|
||||
"recall",
|
||||
"f1",
|
||||
"mean_iou",
|
||||
"false_positive_count",
|
||||
"false_negative_count"
|
||||
],
|
||||
"quality_check_count": 1
|
||||
},
|
||||
"result_counts": {
|
||||
"candidate_feature_count": 1,
|
||||
"matches": 1,
|
||||
"reference_feature_count": 1
|
||||
},
|
||||
"status": "passed"
|
||||
},
|
||||
{
|
||||
"benchmark_id": "golden-buildings-no-overlap-v1",
|
||||
"description": "One candidate polygon and one reference polygon do not overlap.",
|
||||
"fixtures": {
|
||||
"candidate": "fixtures/golden/predicted_buildings_no_overlap.geojson",
|
||||
"reference": "fixtures/golden/reference_buildings_no_overlap.geojson"
|
||||
},
|
||||
"metrics": {
|
||||
"f1": null,
|
||||
"false_negative_count": 1,
|
||||
"false_positive_count": 1,
|
||||
"mean_iou": null,
|
||||
"precision": 0.0,
|
||||
"recall": 0.0
|
||||
},
|
||||
"persistence": {
|
||||
"commit_count": 1,
|
||||
"metric_count": 6,
|
||||
"metric_keys": [
|
||||
"precision",
|
||||
"recall",
|
||||
"f1",
|
||||
"mean_iou",
|
||||
"false_positive_count",
|
||||
"false_negative_count"
|
||||
],
|
||||
"quality_check_count": 1
|
||||
},
|
||||
"result_counts": {
|
||||
"candidate_feature_count": 1,
|
||||
"matches": 0,
|
||||
"reference_feature_count": 1
|
||||
},
|
||||
"status": "passed"
|
||||
},
|
||||
{
|
||||
"benchmark_id": "golden-buildings-multipolygon-match-v1",
|
||||
"description": "One candidate MultiPolygon exactly matches one reference MultiPolygon.",
|
||||
"fixtures": {
|
||||
"candidate": "fixtures/golden/predicted_buildings_multipolygon.geojson",
|
||||
"reference": "fixtures/golden/reference_buildings_multipolygon.geojson"
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"false_negative_count": 0,
|
||||
"false_positive_count": 0,
|
||||
"mean_iou": 1.0,
|
||||
"precision": 1.0,
|
||||
"recall": 1.0
|
||||
},
|
||||
"persistence": {
|
||||
"commit_count": 1,
|
||||
"metric_count": 6,
|
||||
"metric_keys": [
|
||||
"precision",
|
||||
"recall",
|
||||
"f1",
|
||||
"mean_iou",
|
||||
"false_positive_count",
|
||||
"false_negative_count"
|
||||
],
|
||||
"quality_check_count": 1
|
||||
},
|
||||
"result_counts": {
|
||||
"candidate_feature_count": 1,
|
||||
"matches": 1,
|
||||
"reference_feature_count": 1
|
||||
},
|
||||
"status": "passed"
|
||||
}
|
||||
],
|
||||
"status": "passed",
|
||||
"version": 1
|
||||
}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
{
|
||||
"benchmark_manifest_sha256": "0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642",
|
||||
"critical_subgroup_policy": "Any required subgroup with insufficient support, missing metrics, a failed non-inferiority comparison or regression blocks promotion; averages cannot override it.",
|
||||
"decision": "At least one mandatory Phase 4 gate failed; Phase 5 remains blocked.",
|
||||
"gate_policy": "geointel-p4-evaluation-harness-v2",
|
||||
"invalid_gate_states": {},
|
||||
"local_gates": {
|
||||
"all_declared_evaluator_families_exercised": {
|
||||
"declared": [
|
||||
"change_detection",
|
||||
"footprint_segmentation",
|
||||
"geospatial_data_validation",
|
||||
"object_detection",
|
||||
"raster_classification",
|
||||
"terrain_interpretation",
|
||||
"vector_comparison"
|
||||
],
|
||||
"observed": [
|
||||
"change_detection",
|
||||
"footprint_segmentation",
|
||||
"geospatial_data_validation",
|
||||
"object_detection",
|
||||
"raster_classification",
|
||||
"terrain_interpretation",
|
||||
"vector_comparison"
|
||||
],
|
||||
"status": "pass"
|
||||
},
|
||||
"complete_raw_predictions_retained": {
|
||||
"status": "pass",
|
||||
"violating_samples": []
|
||||
},
|
||||
"implemented_capability_inventory": {
|
||||
"capability_count": 15,
|
||||
"status": "pass"
|
||||
},
|
||||
"manifest_training_firewall_contract": {
|
||||
"checks": {
|
||||
"clean_train_allowed": true,
|
||||
"non_train_role_blocked": true,
|
||||
"protected_path_blocked": true,
|
||||
"renamed_protected_lineage_blocked": true
|
||||
},
|
||||
"status": "pass"
|
||||
},
|
||||
"normative_split_roles_and_leakage": {
|
||||
"leakage_status": "pass",
|
||||
"observed_roles": [
|
||||
"background-test",
|
||||
"calibration",
|
||||
"challenge",
|
||||
"test",
|
||||
"train",
|
||||
"val"
|
||||
],
|
||||
"required_roles": [
|
||||
"background-test",
|
||||
"calibration",
|
||||
"challenge",
|
||||
"test",
|
||||
"train",
|
||||
"val"
|
||||
],
|
||||
"status": "pass"
|
||||
},
|
||||
"protected_operating_point_contract": {
|
||||
"evidence": "Protected cases carry pre-registered configurations. Fixed AP/risk-coverage diagnostics cannot select an operating point or feed back into training.",
|
||||
"status": "pass"
|
||||
},
|
||||
"reference_implementation_baseline": {
|
||||
"status": "pass"
|
||||
},
|
||||
"stratified_metric_contract": {
|
||||
"observed_overall_status": "not_evaluable",
|
||||
"status": "pass"
|
||||
},
|
||||
"undefined_metric_truth_table": {
|
||||
"sample_id": "background-test-pure-empty",
|
||||
"status": "pass"
|
||||
}
|
||||
},
|
||||
"local_harness_status": "pass",
|
||||
"missing_gate_names": {
|
||||
"local": [],
|
||||
"product": []
|
||||
},
|
||||
"numeric_model_release_targets": "not_frozen_without_reviewed_representative_incumbent_baseline",
|
||||
"phase_decision": "blocked",
|
||||
"product_benchmark_status": "fail",
|
||||
"product_gates": {
|
||||
"active_model_available_and_hash_verified": {
|
||||
"configured_path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt",
|
||||
"configured_sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1",
|
||||
"reason": "Configured active model is not locally accessible.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"authoritative_reference_portfolio_available": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"executed_product_incumbent_baseline": {
|
||||
"checked_artifacts": [],
|
||||
"derived_gates": {
|
||||
"authoritative_reference_portfolio_available": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"human_review_complete": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"protected_storage_isolation": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"representative_product_subgroup_support": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"split_independence": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
}
|
||||
},
|
||||
"expected_path": "artifacts/evidence/accuracy/P4/product-baseline-manifest.json",
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"human_review_complete": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"phase3_leakage_resolved": {
|
||||
"observed": "attention",
|
||||
"status": "fail"
|
||||
},
|
||||
"protected_storage_isolation": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"representative_product_subgroup_support": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
},
|
||||
"split_independence": {
|
||||
"reason": "No executed, hash-bound product incumbent baseline manifest is available.",
|
||||
"status": "not_evaluable"
|
||||
}
|
||||
},
|
||||
"promotion_allowed": false,
|
||||
"schema_version": 2,
|
||||
"status": "fail",
|
||||
"unexpected_gate_names": {
|
||||
"local": [],
|
||||
"product": []
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"checked_identity_fields": [
|
||||
"sample_id",
|
||||
"group_id",
|
||||
"source_family",
|
||||
"acquisition_date",
|
||||
"temporal_family",
|
||||
"object_ids",
|
||||
"native_feature_ids",
|
||||
"raw_image_sha256",
|
||||
"processed_image_sha256",
|
||||
"label_sha256",
|
||||
"record_sha256",
|
||||
"perceptual_image_hash",
|
||||
"label_geometry_hash",
|
||||
"label_geometry_fingerprint",
|
||||
"parent_raster_id",
|
||||
"acquisition_id",
|
||||
"bbox_distance"
|
||||
],
|
||||
"crs_validation": {
|
||||
"crs": "EPSG:31370",
|
||||
"distance_units": "m",
|
||||
"status": "pass"
|
||||
},
|
||||
"development_manifest_sha256": "64e4fc986c2390bedfbe100062d4b06e1098689f4d4a9a9da13a7f67357c0493",
|
||||
"finding_count": 0,
|
||||
"findings": [],
|
||||
"generator_version": "1.3.0",
|
||||
"independence_buffer_m": 2000.0,
|
||||
"inventory_total": 21,
|
||||
"label_geometry_hamming_threshold": 2,
|
||||
"perceptual_hamming_threshold": 4,
|
||||
"protected_manifest_sha256": "e57ca1529654ef1c2527e84122ff26ec3533e7c70658250983968fbc5d4af410",
|
||||
"schema_version": 1,
|
||||
"source_manifest_sha256": "1c75c875d4c297d046117bb9393ba5e648f13d3358ea92cb65374869c8f5fc01",
|
||||
"split_counts": {
|
||||
"background-test": 2,
|
||||
"calibration": 2,
|
||||
"challenge": 4,
|
||||
"test": 7,
|
||||
"train": 3,
|
||||
"val": 3
|
||||
},
|
||||
"status": "pass"
|
||||
}
|
||||
+8798
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,207 @@
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"metadata": {
|
||||
"context": "dense_urban_raster",
|
||||
"date": "2025-10-05",
|
||||
"difficulty": "hard",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Brussel",
|
||||
"object_size": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "brussels",
|
||||
"resolution_m": 1,
|
||||
"season": "autumn",
|
||||
"sensor": "synthetic-multispectral",
|
||||
"source": "fixture-thematic-raster",
|
||||
"tile_edge": true,
|
||||
"urbanity": "urban",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"metrics": {
|
||||
"accuracy": 0.8333333333333334,
|
||||
"accuracy_ci95_wilson": {
|
||||
"lower": 0.5519691377470266,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 12,
|
||||
"upper": 0.9530348578161463
|
||||
},
|
||||
"alignment_validated": true,
|
||||
"confusion_matrix": {
|
||||
"0": {
|
||||
"0": 4,
|
||||
"1": 0,
|
||||
"2": 0,
|
||||
"__prediction_nodata__": 0
|
||||
},
|
||||
"1": {
|
||||
"0": 0,
|
||||
"1": 4,
|
||||
"2": 1,
|
||||
"__prediction_nodata__": 0
|
||||
},
|
||||
"2": {
|
||||
"0": 0,
|
||||
"1": 1,
|
||||
"2": 2,
|
||||
"__prediction_nodata__": 0
|
||||
}
|
||||
},
|
||||
"correct_pixel_count": 10,
|
||||
"evaluated_reference_pixel_count": 12,
|
||||
"ignored_reference_pixel_count": 0,
|
||||
"macro_f1": 0.8222222222222223,
|
||||
"mean_iou": 0.7222222222222222,
|
||||
"per_class": {
|
||||
"0": {
|
||||
"f1": 1.0,
|
||||
"false_discovery_rate": 0.0,
|
||||
"false_negative": 0,
|
||||
"false_positive": 0,
|
||||
"iou": 1.0,
|
||||
"miss_rate": 0.0,
|
||||
"precision": 1.0,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.5101091635454027,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 4,
|
||||
"upper": 1.0
|
||||
},
|
||||
"prediction_count": 4,
|
||||
"recall": 1.0,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.5101091635454027,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 4,
|
||||
"upper": 1.0
|
||||
},
|
||||
"reference_count": 4,
|
||||
"true_positive": 4
|
||||
},
|
||||
"1": {
|
||||
"f1": 0.8000000000000002,
|
||||
"false_discovery_rate": 0.2,
|
||||
"false_negative": 1,
|
||||
"false_positive": 1,
|
||||
"iou": 0.6666666666666666,
|
||||
"miss_rate": 0.2,
|
||||
"precision": 0.8,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.37553462976252533,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 5,
|
||||
"upper": 0.9637758913675698
|
||||
},
|
||||
"prediction_count": 5,
|
||||
"recall": 0.8,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.37553462976252533,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 5,
|
||||
"upper": 0.9637758913675698
|
||||
},
|
||||
"reference_count": 5,
|
||||
"true_positive": 4
|
||||
},
|
||||
"2": {
|
||||
"f1": 0.6666666666666666,
|
||||
"false_discovery_rate": 0.3333333333333333,
|
||||
"false_negative": 1,
|
||||
"false_positive": 1,
|
||||
"iou": 0.5,
|
||||
"miss_rate": 0.3333333333333333,
|
||||
"precision": 0.6666666666666666,
|
||||
"precision_ci95_wilson": {
|
||||
"lower": 0.2076596008020477,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 3,
|
||||
"upper": 0.9385080552796037
|
||||
},
|
||||
"prediction_count": 3,
|
||||
"recall": 0.6666666666666666,
|
||||
"recall_ci95_wilson": {
|
||||
"lower": 0.2076596008020477,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 3,
|
||||
"upper": 0.9385080552796037
|
||||
},
|
||||
"reference_count": 3,
|
||||
"true_positive": 2
|
||||
}
|
||||
},
|
||||
"pixel_count": 12,
|
||||
"prediction_coverage": 1.0,
|
||||
"prediction_coverage_ci95_wilson": {
|
||||
"lower": 0.7575059933447592,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 12,
|
||||
"upper": 1.0
|
||||
},
|
||||
"prediction_outside_reference_count": 0,
|
||||
"prediction_present_pixel_count": 12
|
||||
},
|
||||
"sample_id": "raster-test-brussels-urban",
|
||||
"task": "raster_classification"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"context": "terrain_profile",
|
||||
"date": "2023-01-01",
|
||||
"difficulty": "normal",
|
||||
"label_review_state": "fixture_reviewed",
|
||||
"municipality": "Hasselt",
|
||||
"object_size": "not_applicable",
|
||||
"occlusion": "not_applicable",
|
||||
"ood": false,
|
||||
"region": "flanders",
|
||||
"resolution_m": 5,
|
||||
"season": "not_applicable",
|
||||
"sensor": "elevation-raster",
|
||||
"source": "fixture-dhmv",
|
||||
"tile_edge": false,
|
||||
"urbanity": "rural",
|
||||
"vegetation": "moderate"
|
||||
},
|
||||
"metrics": {
|
||||
"absolute_error_sum": 1.0999999999999996,
|
||||
"bias": 0.02499999999999991,
|
||||
"coverage": 0.8,
|
||||
"coverage_ci95_wilson": {
|
||||
"lower": 0.37553462976252533,
|
||||
"method": "Wilson score interval",
|
||||
"status": "computed",
|
||||
"support": 5,
|
||||
"upper": 0.9637758913675698
|
||||
},
|
||||
"error_distribution": {
|
||||
"count": 4,
|
||||
"max": 0.40000000000000036,
|
||||
"mean": 0.02499999999999991,
|
||||
"median": 0.0,
|
||||
"min": -0.3000000000000007
|
||||
},
|
||||
"error_sum": 0.09999999999999964,
|
||||
"evaluated_count": 4,
|
||||
"mae": 0.2749999999999999,
|
||||
"missing_count": 1,
|
||||
"reference_count": 5,
|
||||
"rmse": 0.2872281323269015,
|
||||
"squared_error_sum": 0.3300000000000001,
|
||||
"units": "m_TAW"
|
||||
},
|
||||
"sample_id": "terrain-test-flanders-rural",
|
||||
"task": "terrain_interpretation"
|
||||
}
|
||||
],
|
||||
"schema_version": 2,
|
||||
"status": "fixture_contract_only"
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"benchmark_manifest_sha256": "0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642",
|
||||
"case_count": 9,
|
||||
"evaluation_results_canonical_json_sha256": "55cdba9cce4dcdc88cee0829cd510d7e624a3472feb9d33f7e681621a792eba3",
|
||||
"evidence_path": "artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b",
|
||||
"evidence_run_id": "p4-2.0.1-9677d0ef37db82bcf39b",
|
||||
"failure_example_count": 14,
|
||||
"implemented_capability_count": 15,
|
||||
"local_harness_status": "pass",
|
||||
"phase4_done": false,
|
||||
"phase5_ready": false,
|
||||
"phase_decision": "blocked",
|
||||
"product_benchmark_status": "fail",
|
||||
"product_gate_evidence_sha256": "c9f5e3de3ab6d64911f8807b26caa661ccf911a5fd6b995107edda2004836e0c",
|
||||
"promotion_allowed": false,
|
||||
"reference_baseline_sha256": "aeb0f6510352f4e205e4fc4e8c423110ca72c2142016c7af846d758865f258c6",
|
||||
"schema_version": 2,
|
||||
"split_counts": {
|
||||
"background-test": 2,
|
||||
"calibration": 2,
|
||||
"challenge": 4,
|
||||
"test": 7,
|
||||
"train": 3,
|
||||
"val": 3
|
||||
},
|
||||
"status": "fail",
|
||||
"task_family_count": 7
|
||||
}
|
||||
Reference in New Issue
Block a user