From f41392a41538faa9cef00fcdd413aa4b6442f5df Mon Sep 17 00:00:00 2001 From: Jens Date: Sun, 2 Aug 2026 05:10:05 +0200 Subject: [PATCH] docs(accuracy): publish phase 4 benchmark evidence --- .gitignore | 6 +- .../acceptance-gates.json | 157 + .../aoi-metrics.json | 11 + .../baseline-raw-predictions.json | 1489 +++ .../benchmark-manifest.json | 105 + .../calibration-metrics.json | 274 + .../candidate-vs-incumbent.json | 12 + .../development-split-manifest.json | 334 + .../error-taxonomy.json | 48 + .../evaluation-contract.json | 300 + .../evidence-manifest.json | 129 + .../failure-gallery.json | 537 + .../generation-status.json | 8 + .../human-review-summary.json | 8 + .../input-manifest.json | 185 + .../latency-and-reliability.json | 6 + .../leakage-gate-report.json | 46 + .../metric-report.json | 10635 ++++++++++++++++ .../object-metrics.json | 806 ++ .../protected-split-manifest.json | 589 + .../reference-implementation-baseline.json | 163 + .../release-gate-report.json | 157 + .../split-and-leakage-audit.json | 46 + .../stratified-metrics.json | 8798 +++++++++++++ .../tile-metrics.json | 207 + .../workflow-summary.json | 28 + docs/CODEX_EXECUTION_LOG.md | 53 + docs/TODO.md | 28 + docs/accuracy-program/08-data-contracts.md | 5 + docs/accuracy-program/09-full-data-scan.md | 5 + .../10-evaluation-protocol.md | 175 + .../accuracy-program/11-baseline-benchmark.md | 124 + docs/accuracy-program/12-release-gates.md | 109 + docs/accuracy-program/status.json | 97 +- 34 files changed, 25672 insertions(+), 8 deletions(-) create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/acceptance-gates.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/aoi-metrics.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/baseline-raw-predictions.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/benchmark-manifest.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/calibration-metrics.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/candidate-vs-incumbent.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/development-split-manifest.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/error-taxonomy.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/evaluation-contract.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/evidence-manifest.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/failure-gallery.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/generation-status.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/human-review-summary.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/input-manifest.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/latency-and-reliability.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/leakage-gate-report.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/metric-report.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/object-metrics.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/protected-split-manifest.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/reference-implementation-baseline.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/release-gate-report.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/split-and-leakage-audit.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/stratified-metrics.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/tile-metrics.json create mode 100644 artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/workflow-summary.json create mode 100644 docs/accuracy-program/10-evaluation-protocol.md create mode 100644 docs/accuracy-program/11-baseline-benchmark.md create mode 100644 docs/accuracy-program/12-release-gates.md diff --git a/.gitignore b/.gitignore index 260f3229..269105ff 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,10 @@ build/ !/artifacts/evidence/accuracy/P3/** !/artifacts/evidence/accuracy/P4/ /artifacts/evidence/accuracy/P4/* -!/artifacts/evidence/accuracy/P4/reference-harness-v2/ -!/artifacts/evidence/accuracy/P4/reference-harness-v2/** +!/artifacts/evidence/accuracy/P4/runs/ +/artifacts/evidence/accuracy/P4/runs/* +!/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/ +!/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/** !/artifacts/evidence/accuracy/P2/ !/artifacts/evidence/accuracy/P2/** /.cache/ diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/acceptance-gates.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/acceptance-gates.json new file mode 100644 index 00000000..ca6bf907 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/acceptance-gates.json @@ -0,0 +1,157 @@ +{ + "benchmark_manifest_sha256": "0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642", + "critical_subgroup_policy": "Any required subgroup with insufficient support, missing metrics, a failed non-inferiority comparison or regression blocks promotion; averages cannot override it.", + "decision": "At least one mandatory Phase 4 gate failed; Phase 5 remains blocked.", + "gate_policy": "geointel-p4-evaluation-harness-v2", + "invalid_gate_states": {}, + "local_gates": { + "all_declared_evaluator_families_exercised": { + "declared": [ + "change_detection", + "footprint_segmentation", + "geospatial_data_validation", + "object_detection", + "raster_classification", + "terrain_interpretation", + "vector_comparison" + ], + "observed": [ + "change_detection", + "footprint_segmentation", + "geospatial_data_validation", + "object_detection", + "raster_classification", + "terrain_interpretation", + "vector_comparison" + ], + "status": "pass" + }, + "complete_raw_predictions_retained": { + "status": "pass", + "violating_samples": [] + }, + "implemented_capability_inventory": { + "capability_count": 15, + "status": "pass" + }, + "manifest_training_firewall_contract": { + "checks": { + "clean_train_allowed": true, + "non_train_role_blocked": true, + "protected_path_blocked": true, + "renamed_protected_lineage_blocked": true + }, + "status": "pass" + }, + "normative_split_roles_and_leakage": { + "leakage_status": "pass", + "observed_roles": [ + "background-test", + "calibration", + "challenge", + "test", + "train", + "val" + ], + "required_roles": [ + "background-test", + "calibration", + "challenge", + "test", + "train", + "val" + ], + "status": "pass" + }, + "protected_operating_point_contract": { + "evidence": "Protected cases carry pre-registered configurations. Fixed AP/risk-coverage diagnostics cannot select an operating point or feed back into training.", + "status": "pass" + }, + "reference_implementation_baseline": { + "status": "pass" + }, + "stratified_metric_contract": { + "observed_overall_status": "not_evaluable", + "status": "pass" + }, + "undefined_metric_truth_table": { + "sample_id": "background-test-pure-empty", + "status": "pass" + } + }, + "local_harness_status": "pass", + "missing_gate_names": { + "local": [], + "product": [] + }, + "numeric_model_release_targets": "not_frozen_without_reviewed_representative_incumbent_baseline", + "phase_decision": "blocked", + "product_benchmark_status": "fail", + "product_gates": { + "active_model_available_and_hash_verified": { + "configured_path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt", + "configured_sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1", + "reason": "Configured active model is not locally accessible.", + "status": "not_evaluable" + }, + "authoritative_reference_portfolio_available": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "executed_product_incumbent_baseline": { + "checked_artifacts": [], + "derived_gates": { + "authoritative_reference_portfolio_available": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "human_review_complete": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "protected_storage_isolation": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "representative_product_subgroup_support": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "split_independence": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + } + }, + "expected_path": "artifacts/evidence/accuracy/P4/product-baseline-manifest.json", + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "human_review_complete": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "phase3_leakage_resolved": { + "observed": "attention", + "status": "fail" + }, + "protected_storage_isolation": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "representative_product_subgroup_support": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "split_independence": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + } + }, + "promotion_allowed": false, + "schema_version": 2, + "status": "fail", + "unexpected_gate_names": { + "local": [], + "product": [] + } +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/aoi-metrics.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/aoi-metrics.json new file mode 100644 index 00000000..1855130a --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/aoi-metrics.json @@ -0,0 +1,11 @@ +{ + "reason": "Synthetic single-case fixtures do not provide independent product AOI clusters. AOI micro/macro and cluster-bootstrap evidence requires the protected product corpus.", + "required_future_outputs": [ + "per-AOI primary metrics", + "micro and macro aggregation", + "paired candidate-minus-incumbent deltas", + "cluster-bootstrap confidence intervals" + ], + "schema_version": 2, + "status": "not_evaluable" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/baseline-raw-predictions.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/baseline-raw-predictions.json new file mode 100644 index 00000000..68d3bceb --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/baseline-raw-predictions.json @@ -0,0 +1,1489 @@ +{ + "evaluator_version": "2.1.0", + "hash_specification": { + "algorithm": "SHA-256", + "canonical_json": "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), ensure_ascii=false and allow_nan=false", + "portfolio_canonical_json_sha256_input": "parsed complete portfolio", + "portfolio_file_sha256_input": "exact source-file bytes", + "results_canonical_json_sha256_input": "complete sample-id-ordered results array" + }, + "items": [ + { + "classes": [ + "building" + ], + "config": { + "confidence_threshold": 0.5, + "fixed_diagnostic_risk_thresholds": [ + 0, + 0.5, + 0.7, + 0.9 + ], + "match_iou": 0.5 + }, + "false_negatives": [], + "false_positives": [ + { + "bbox": [ + 20, + 20, + 35, + 35 + ], + "class": "building", + "confidence": 0.92, + "id": "p-hard-fp" + } + ], + "filter": { + "applied": true, + "field": "confidence", + "operator": ">=", + "threshold": 0.5 + }, + "hashes": { + "canonicalization": "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), ensure_ascii=false and allow_nan=false", + "case_input_canonical_json_sha256": "27595a6d16d96d05e0c8ee326dd03cbf060d68b86b7834bce600ce7d640c3a97", + "predictions_post_filter_canonical_json_sha256": "66e3b9311b85e2be0d968fbd59359a2d2147eec52034e7cda8bc84cb3e4139ff", + "predictions_pre_filter_canonical_json_sha256": "66e3b9311b85e2be0d968fbd59359a2d2147eec52034e7cda8bc84cb3e4139ff", + "references_canonical_json_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "input_lineage": { + "prediction": { + "derivation": "hand_authored_fixed_output", + "source_id": "synthetic:background-test-hard-negative:prediction", + "source_version": "1" + }, + "reference": { + "derivation": "hand_authored_contract_fixture", + "source_id": "synthetic:background-test-hard-negative:reference", + "source_version": "1" + } + }, + "input_sha256": "27595a6d16d96d05e0c8ee326dd03cbf060d68b86b7834bce600ce7d640c3a97", + "matches": [], + "metadata": { + "context": "coastal_hard_negative", + "date": "2025-07-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Oostende", + "object_size": "empty", + "occlusion": "none", + "ood": false, + "region": "flanders", + "resolution_m": 0.25, + "season": "summer", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": true, + "urbanity": "urban", + "vegetation": "moderate" + }, + "portfolio_lineage": { + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "declared": { + "origin": "repository_fixture", + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "version": "1" + }, + "portfolio_canonical_json_sha256": "1ac1bd77db125db48f9168406d51bfb723522f54a6c81dbe868d600c72f27b17", + "portfolio_file_sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "portfolio_id": "geointel-p4-reference-harness-v2", + "portfolio_kind": "synthetic_contract", + "portfolio_schema_version": 2, + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "split_roles": [ + "test", + "background-test" + ] + }, + "prediction_input_field": "predictions", + "predictions_post_filter": [ + { + "bbox": [ + 20, + 20, + 35, + 35 + ], + "class": "building", + "confidence": 0.92, + "id": "p-hard-fp" + } + ], + "predictions_pre_filter": [ + { + "bbox": [ + 20, + 20, + 35, + 35 + ], + "class": "building", + "confidence": 0.92, + "id": "p-hard-fp" + } + ], + "reference_input_field": "references", + "references": [], + "sample_id": "background-test-hard-negative", + "split": "background-test", + "task": "object_detection" + }, + { + "classes": [ + "building" + ], + "config": { + "confidence_threshold": 0.5, + "fixed_diagnostic_risk_thresholds": [ + 0, + 0.5, + 0.7, + 0.9 + ], + "match_iou": 0.5 + }, + "false_negatives": [], + "false_positives": [], + "filter": { + "applied": true, + "field": "confidence", + "operator": ">=", + "threshold": 0.5 + }, + "hashes": { + "canonicalization": "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), ensure_ascii=false and allow_nan=false", + "case_input_canonical_json_sha256": "0bfe26519b12db547f156905202badca6a5c5a9759f4d6eeb8c0526ddb846974", + "predictions_post_filter_canonical_json_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "predictions_pre_filter_canonical_json_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "references_canonical_json_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "input_lineage": { + "prediction": { + "derivation": "hand_authored_fixed_output", + "source_id": "synthetic:background-test-pure-empty:prediction", + "source_version": "1" + }, + "reference": { + "derivation": "hand_authored_contract_fixture", + "source_id": "synthetic:background-test-pure-empty:reference", + "source_version": "1" + } + }, + "input_sha256": "0bfe26519b12db547f156905202badca6a5c5a9759f4d6eeb8c0526ddb846974", + "matches": [], + "metadata": { + "context": "pure_background", + "date": "2025-02-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Brussel", + "object_size": "empty", + "occlusion": "none", + "ood": false, + "region": "brussels", + "resolution_m": 0.25, + "season": "winter", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": false, + "urbanity": "urban", + "vegetation": "low" + }, + "portfolio_lineage": { + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "declared": { + "origin": "repository_fixture", + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "version": "1" + }, + "portfolio_canonical_json_sha256": "1ac1bd77db125db48f9168406d51bfb723522f54a6c81dbe868d600c72f27b17", + "portfolio_file_sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "portfolio_id": "geointel-p4-reference-harness-v2", + "portfolio_kind": "synthetic_contract", + "portfolio_schema_version": 2, + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "split_roles": [ + "test", + "background-test" + ] + }, + "prediction_input_field": "predictions", + "predictions_post_filter": [], + "predictions_pre_filter": [], + "reference_input_field": "references", + "references": [], + "sample_id": "background-test-pure-empty", + "split": "background-test", + "task": "object_detection" + }, + { + "classes": [ + "added", + "removed" + ], + "config": { + "match_iou": 0.5 + }, + "false_negatives": [ + { + "bbox": [ + 10, + 0, + 15, + 5 + ], + "class": "removed", + "id": "r2" + } + ], + "false_positives": [ + { + "bbox": [ + 20, + 0, + 25, + 5 + ], + "class": "removed", + "confidence": 0.65, + "id": "p2" + } + ], + "filter": { + "applied": false + }, + "hashes": { + "canonicalization": "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), ensure_ascii=false and allow_nan=false", + "case_input_canonical_json_sha256": "324e224883a99770be116a6a74cae2502f95150b963fd6c92342c3111fe367d5", + "predictions_post_filter_canonical_json_sha256": "beb52d7c3cec145ad733a8adbe95105a6ff838d8f3cf35e335773b1755dc5c43", + "predictions_pre_filter_canonical_json_sha256": "beb52d7c3cec145ad733a8adbe95105a6ff838d8f3cf35e335773b1755dc5c43", + "references_canonical_json_sha256": "0018ab1745225a0dcf6d10ad1434a14bbc108f16d0f281e4d730d613941115d3" + }, + "input_lineage": { + "prediction": { + "derivation": "hand_authored_fixed_output", + "source_id": "synthetic:change-test-wallonia-industrial:prediction", + "source_version": "1" + }, + "reference": { + "derivation": "hand_authored_contract_fixture", + "source_id": "synthetic:change-test-wallonia-industrial:reference", + "source_version": "1" + } + }, + "input_sha256": "324e224883a99770be116a6a74cae2502f95150b963fd6c92342c3111fe367d5", + "matches": [ + { + "class": "added", + "confidence": 0.85, + "event": "added", + "overlap": 1.0, + "prediction_id": "p1", + "reference_id": "r1" + } + ], + "metadata": { + "context": "industrial_change", + "date": "2024-05-01/2025-05-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Liège", + "object_size": "mixed", + "occlusion": "none", + "ood": false, + "region": "wallonia", + "resolution_m": 0.5, + "season": "multi-date", + "sensor": "vector", + "source": "fixture-picc-temporal", + "tile_edge": false, + "urbanity": "industrial", + "vegetation": "low" + }, + "portfolio_lineage": { + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "declared": { + "origin": "repository_fixture", + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "version": "1" + }, + "portfolio_canonical_json_sha256": "1ac1bd77db125db48f9168406d51bfb723522f54a6c81dbe868d600c72f27b17", + "portfolio_file_sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "portfolio_id": "geointel-p4-reference-harness-v2", + "portfolio_kind": "synthetic_contract", + "portfolio_schema_version": 2, + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "split_roles": [ + "test", + "background-test" + ] + }, + "prediction_input_field": "predictions", + "predictions_post_filter": [ + { + "bbox": [ + 0, + 0, + 5, + 5 + ], + "class": "added", + "confidence": 0.85, + "id": "p1" + }, + { + "bbox": [ + 20, + 0, + 25, + 5 + ], + "class": "removed", + "confidence": 0.65, + "id": "p2" + } + ], + "predictions_pre_filter": [ + { + "bbox": [ + 0, + 0, + 5, + 5 + ], + "class": "added", + "confidence": 0.85, + "id": "p1" + }, + { + "bbox": [ + 20, + 0, + 25, + 5 + ], + "class": "removed", + "confidence": 0.65, + "id": "p2" + } + ], + "reference_input_field": "references", + "references": [ + { + "bbox": [ + 0, + 0, + 5, + 5 + ], + "class": "added", + "id": "r1" + }, + { + "bbox": [ + 10, + 0, + 15, + 5 + ], + "class": "removed", + "id": "r2" + } + ], + "sample_id": "change-test-wallonia-industrial", + "split": "test", + "task": "change_detection" + }, + { + "classes": [ + "building" + ], + "config": { + "confidence_threshold": 0.5, + "fixed_diagnostic_risk_thresholds": [ + 0, + 0.5, + 0.7, + 0.9 + ], + "match_iou": 0.5 + }, + "false_negatives": [], + "false_positives": [ + { + "bbox": [ + 85, + 85, + 95, + 95 + ], + "class": "building", + "confidence": 0.6, + "id": "p3" + } + ], + "filter": { + "applied": true, + "field": "confidence", + "operator": ">=", + "threshold": 0.5 + }, + "hashes": { + "canonicalization": "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), ensure_ascii=false and allow_nan=false", + "case_input_canonical_json_sha256": "9bcda4e622a057261be4bbb977064f6e4fd1ae9fa19289124e3ba4de5f829722", + "predictions_post_filter_canonical_json_sha256": "7c0f9832b4468f46d783b403494378e40a8f21dff3fb5d58545fa3ec06a31d0b", + "predictions_pre_filter_canonical_json_sha256": "8bee0a2cbd449de9a008b2ed9c9f0b08ca1f7b86b2ed8ac2171de7092089e00c", + "references_canonical_json_sha256": "238b628e55219d3fbd6e7b38d9ace75f535947e49e83191b8d46233cfcee45ff" + }, + "input_lineage": { + "prediction": { + "derivation": "hand_authored_fixed_output", + "source_id": "synthetic:det-test-flanders-urban:prediction", + "source_version": "1" + }, + "reference": { + "derivation": "hand_authored_contract_fixture", + "source_id": "synthetic:det-test-flanders-urban:reference", + "source_version": "1" + } + }, + "input_sha256": "9bcda4e622a057261be4bbb977064f6e4fd1ae9fa19289124e3ba4de5f829722", + "matches": [ + { + "class": "building", + "confidence": 0.9, + "overlap": 1.0, + "prediction_id": "p1", + "reference_id": "r1" + }, + { + "class": "building", + "confidence": 0.7, + "overlap": 0.8711111111111111, + "prediction_id": "p2", + "reference_id": "r2" + } + ], + "metadata": { + "context": "dense_urban", + "date": "2025-06-15", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Antwerpen", + "object_size": "mixed", + "occlusion": "partial", + "ood": false, + "region": "flanders", + "resolution_m": 0.25, + "season": "summer", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": false, + "urbanity": "urban", + "vegetation": "moderate" + }, + "portfolio_lineage": { + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "declared": { + "origin": "repository_fixture", + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "version": "1" + }, + "portfolio_canonical_json_sha256": "1ac1bd77db125db48f9168406d51bfb723522f54a6c81dbe868d600c72f27b17", + "portfolio_file_sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "portfolio_id": "geointel-p4-reference-harness-v2", + "portfolio_kind": "synthetic_contract", + "portfolio_schema_version": 2, + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "split_roles": [ + "test", + "background-test" + ] + }, + "prediction_input_field": "predictions", + "predictions_post_filter": [ + { + "bbox": [ + 10, + 10, + 30, + 30 + ], + "class": "building", + "confidence": 0.9, + "id": "p1" + }, + { + "bbox": [ + 52, + 52, + 80, + 80 + ], + "class": "building", + "confidence": 0.7, + "id": "p2" + }, + { + "bbox": [ + 85, + 85, + 95, + 95 + ], + "class": "building", + "confidence": 0.6, + "id": "p3" + } + ], + "predictions_pre_filter": [ + { + "bbox": [ + 10, + 10, + 30, + 30 + ], + "class": "building", + "confidence": 0.9, + "id": "p1" + }, + { + "bbox": [ + 52, + 52, + 80, + 80 + ], + "class": "building", + "confidence": 0.7, + "id": "p2" + }, + { + "bbox": [ + 85, + 85, + 95, + 95 + ], + "class": "building", + "confidence": 0.6, + "id": "p3" + }, + { + "bbox": [ + 2, + 80, + 7, + 86 + ], + "class": "building", + "confidence": 0.2, + "id": "p4-filtered" + } + ], + "reference_input_field": "references", + "references": [ + { + "bbox": [ + 10, + 10, + 30, + 30 + ], + "class": "building", + "id": "r1" + }, + { + "bbox": [ + 50, + 50, + 80, + 80 + ], + "class": "building", + "id": "r2" + } + ], + "sample_id": "det-test-flanders-urban", + "split": "test", + "task": "object_detection" + }, + { + "classes": [ + 0, + 1, + 2 + ], + "config": { + "class_order": [ + 0, + 1, + 2 + ], + "masked_pixels_excluded": true + }, + "false_negatives": [], + "false_positives": [ + { + "column": 3, + "pixel_index": 3, + "prediction": 2, + "reason": "class_mismatch", + "reference": 1, + "row": 0 + }, + { + "column": 1, + "pixel_index": 9, + "prediction": 1, + "reason": "class_mismatch", + "reference": 2, + "row": 2 + } + ], + "filter": { + "applied": false + }, + "hashes": { + "canonicalization": "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), ensure_ascii=false and allow_nan=false", + "case_input_canonical_json_sha256": "47640ac47be2509359172b307c164c2888ce4984b82eebf4edd450ec266beae9", + "predictions_post_filter_canonical_json_sha256": "9dead8589ca1c9f284a7df378a2fe3b817aa78c6d4595c0333cfc08d95f91297", + "predictions_pre_filter_canonical_json_sha256": "9dead8589ca1c9f284a7df378a2fe3b817aa78c6d4595c0333cfc08d95f91297", + "references_canonical_json_sha256": "52797b4d5180a470c1e651ea70af03c940a39ee8de82dcb341d23b12cfb1810e" + }, + "input_lineage": { + "prediction": { + "derivation": "hand_authored_fixed_output", + "source_id": "synthetic:raster-test-brussels-urban:prediction", + "source_version": "1" + }, + "reference": { + "derivation": "hand_authored_contract_fixture", + "source_id": "synthetic:raster-test-brussels-urban:reference", + "source_version": "1" + } + }, + "input_sha256": "47640ac47be2509359172b307c164c2888ce4984b82eebf4edd450ec266beae9", + "matches": [], + "metadata": { + "context": "dense_urban_raster", + "date": "2025-10-05", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Brussel", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "brussels", + "resolution_m": 1, + "season": "autumn", + "sensor": "synthetic-multispectral", + "source": "fixture-thematic-raster", + "tile_edge": true, + "urbanity": "urban", + "vegetation": "moderate" + }, + "portfolio_lineage": { + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "declared": { + "origin": "repository_fixture", + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "version": "1" + }, + "portfolio_canonical_json_sha256": "1ac1bd77db125db48f9168406d51bfb723522f54a6c81dbe868d600c72f27b17", + "portfolio_file_sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "portfolio_id": "geointel-p4-reference-harness-v2", + "portfolio_kind": "synthetic_contract", + "portfolio_schema_version": 2, + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "split_roles": [ + "test", + "background-test" + ] + }, + "prediction_input_field": "predictions", + "predictions_post_filter": [ + [ + 0, + 0, + 1, + 2 + ], + [ + 0, + 1, + 1, + 2 + ], + [ + 2, + 1, + 1, + 0 + ] + ], + "predictions_pre_filter": [ + [ + 0, + 0, + 1, + 2 + ], + [ + 0, + 1, + 1, + 2 + ], + [ + 2, + 1, + 1, + 0 + ] + ], + "reference_input_field": "references", + "references": [ + [ + 0, + 0, + 1, + 1 + ], + [ + 0, + 1, + 1, + 2 + ], + [ + 2, + 2, + 1, + 0 + ] + ], + "sample_id": "raster-test-brussels-urban", + "split": "test", + "task": "raster_classification" + }, + { + "classes": [ + "building" + ], + "config": { + "boundary_tolerance_m": 1, + "match_iou": 0.5 + }, + "false_negatives": [], + "false_positives": [], + "filter": { + "applied": false + }, + "hashes": { + "canonicalization": "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), ensure_ascii=false and allow_nan=false", + "case_input_canonical_json_sha256": "6c63b5b8c2943b5f393569f8520f031ebfe988b621ea2e6894c6d446928ca6c4", + "predictions_post_filter_canonical_json_sha256": "dfed6b0f5ac2b7280c7729414a19951b8d074ce00eb83c7d5eaa49e7bdefe135", + "predictions_pre_filter_canonical_json_sha256": "dfed6b0f5ac2b7280c7729414a19951b8d074ce00eb83c7d5eaa49e7bdefe135", + "references_canonical_json_sha256": "c5bd8154f877c9d7c5691e6c63d77657ce83ca0a5b1d8fadadccb6ab0a5ff4a9" + }, + "input_lineage": { + "prediction": { + "derivation": "hand_authored_fixed_output", + "source_id": "synthetic:seg-test-wallonia-rural:prediction", + "source_version": "1" + }, + "reference": { + "derivation": "hand_authored_contract_fixture", + "source_id": "synthetic:seg-test-wallonia-rural:reference", + "source_version": "1" + } + }, + "input_sha256": "6c63b5b8c2943b5f393569f8520f031ebfe988b621ea2e6894c6d446928ca6c4", + "matches": [ + { + "boundary_f1": 1.0, + "centroid_distance_m": 0.5, + "class": "building", + "confidence": 0.8, + "dice": 0.8881578947368421, + "overlap": 0.7988165680473372, + "prediction_id": "p1", + "reference_id": "r1", + "relative_area_error": -0.1 + } + ], + "metadata": { + "context": "rural_buildings", + "date": "2025-04-20", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Namur", + "object_size": "large", + "occlusion": "none", + "ood": false, + "region": "wallonia", + "resolution_m": 0.5, + "season": "spring", + "sensor": "synthetic-rgb", + "source": "fixture-picc", + "tile_edge": false, + "urbanity": "rural", + "vegetation": "low" + }, + "portfolio_lineage": { + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "declared": { + "origin": "repository_fixture", + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "version": "1" + }, + "portfolio_canonical_json_sha256": "1ac1bd77db125db48f9168406d51bfb723522f54a6c81dbe868d600c72f27b17", + "portfolio_file_sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "portfolio_id": "geointel-p4-reference-harness-v2", + "portfolio_kind": "synthetic_contract", + "portfolio_schema_version": 2, + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "split_roles": [ + "test", + "background-test" + ] + }, + "prediction_input_field": "predictions", + "predictions_post_filter": [ + { + "class": "building", + "confidence": 0.8, + "id": "p1", + "polygon": [ + [ + 0.5, + 0.5 + ], + [ + 9.5, + 0.5 + ], + [ + 9.5, + 8.5 + ], + [ + 0.5, + 8.5 + ], + [ + 0.5, + 0.5 + ] + ] + } + ], + "predictions_pre_filter": [ + { + "class": "building", + "confidence": 0.8, + "id": "p1", + "polygon": [ + [ + 0.5, + 0.5 + ], + [ + 9.5, + 0.5 + ], + [ + 9.5, + 8.5 + ], + [ + 0.5, + 8.5 + ], + [ + 0.5, + 0.5 + ] + ] + } + ], + "reference_input_field": "references", + "references": [ + { + "class": "building", + "id": "r1", + "polygon": [ + [ + 0, + 0 + ], + [ + 10, + 0 + ], + [ + 10, + 8 + ], + [ + 0, + 8 + ], + [ + 0, + 0 + ] + ] + } + ], + "sample_id": "seg-test-wallonia-rural", + "split": "test", + "task": "footprint_segmentation" + }, + { + "classes": null, + "config": { + "missing_prediction_policy": "exclude_and_reduce_coverage" + }, + "false_negatives": [ + { + "missing_index": 3 + } + ], + "false_positives": [], + "filter": { + "applied": false + }, + "hashes": { + "canonicalization": "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), ensure_ascii=false and allow_nan=false", + "case_input_canonical_json_sha256": "d9e2637e15491546f8eaad2dc140fe2af78f213dd9899bc0f683234880a5e0c8", + "predictions_post_filter_canonical_json_sha256": "74d1fc53e85cc8ba6ca6cd8fb81a998ada3e3dd0e7d8ef5967f9ec8cad722768", + "predictions_pre_filter_canonical_json_sha256": "74d1fc53e85cc8ba6ca6cd8fb81a998ada3e3dd0e7d8ef5967f9ec8cad722768", + "references_canonical_json_sha256": "592601848bf07aeef289349574a01b11c006e801f47a29ddd584113b146f7390" + }, + "input_lineage": { + "prediction": { + "derivation": "hand_authored_fixed_output", + "source_id": "synthetic:terrain-test-flanders-rural:prediction", + "source_version": "1" + }, + "reference": { + "derivation": "hand_authored_contract_fixture", + "source_id": "synthetic:terrain-test-flanders-rural:reference", + "source_version": "1" + } + }, + "input_sha256": "d9e2637e15491546f8eaad2dc140fe2af78f213dd9899bc0f683234880a5e0c8", + "matches": [], + "metadata": { + "context": "terrain_profile", + "date": "2023-01-01", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Hasselt", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "flanders", + "resolution_m": 5, + "season": "not_applicable", + "sensor": "elevation-raster", + "source": "fixture-dhmv", + "tile_edge": false, + "urbanity": "rural", + "vegetation": "moderate" + }, + "portfolio_lineage": { + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "declared": { + "origin": "repository_fixture", + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "version": "1" + }, + "portfolio_canonical_json_sha256": "1ac1bd77db125db48f9168406d51bfb723522f54a6c81dbe868d600c72f27b17", + "portfolio_file_sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "portfolio_id": "geointel-p4-reference-harness-v2", + "portfolio_kind": "synthetic_contract", + "portfolio_schema_version": 2, + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "split_roles": [ + "test", + "background-test" + ] + }, + "prediction_input_field": "predictions", + "predictions_post_filter": [ + 12.2, + 12.8, + 14.4, + null, + 15.7 + ], + "predictions_pre_filter": [ + 12.2, + 12.8, + 14.4, + null, + 15.7 + ], + "reference_input_field": "references", + "references": [ + 12, + 13, + 14, + 15, + 16 + ], + "sample_id": "terrain-test-flanders-rural", + "split": "test", + "task": "terrain_interpretation" + }, + { + "classes": null, + "config": { + "critical_severities": [ + "blocker", + "critical" + ] + }, + "false_negatives": [ + { + "anomaly": "D-MISSING-PROVENANCE", + "severity": "critical" + } + ], + "false_positives": [ + { + "anomaly": "D-RESOLUTION", + "severity": "major" + } + ], + "filter": { + "applied": false + }, + "hashes": { + "canonicalization": "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), ensure_ascii=false and allow_nan=false", + "case_input_canonical_json_sha256": "cd191708e28435027749f573412592dd7ce2c4cd1e08656f6306382691a9d2fb", + "predictions_post_filter_canonical_json_sha256": "a9f7113f70c1b41d8927ba6018a8e22fbeb15499f0833ad16dc9be3b1d567371", + "predictions_pre_filter_canonical_json_sha256": "a9f7113f70c1b41d8927ba6018a8e22fbeb15499f0833ad16dc9be3b1d567371", + "references_canonical_json_sha256": "1d19e53c82b4815685a23921146ce35c0105f403774b712c1c41ee3ca4aa0672" + }, + "input_lineage": { + "prediction": { + "derivation": "hand_authored_fixed_output", + "source_id": "synthetic:validation-test-national:prediction", + "source_version": "1" + }, + "reference": { + "derivation": "hand_authored_contract_fixture", + "source_id": "synthetic:validation-test-national:reference", + "source_version": "1" + } + }, + "input_sha256": "cd191708e28435027749f573412592dd7ce2c4cd1e08656f6306382691a9d2fb", + "matches": [ + { + "anomaly": "D-CRS-WRONG", + "expected_severity": "blocker", + "observed_severity": "blocker" + }, + { + "anomaly": "D-INVALID-GEOMETRY", + "expected_severity": "critical", + "observed_severity": "critical" + } + ], + "metadata": { + "context": "data_contract", + "date": "2025-01-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "not_applicable", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "national", + "resolution_m": null, + "season": "not_applicable", + "sensor": "mixed", + "source": "fixture-contracts", + "tile_edge": false, + "urbanity": "mixed", + "vegetation": "mixed" + }, + "portfolio_lineage": { + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "declared": { + "origin": "repository_fixture", + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "version": "1" + }, + "portfolio_canonical_json_sha256": "1ac1bd77db125db48f9168406d51bfb723522f54a6c81dbe868d600c72f27b17", + "portfolio_file_sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "portfolio_id": "geointel-p4-reference-harness-v2", + "portfolio_kind": "synthetic_contract", + "portfolio_schema_version": 2, + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "split_roles": [ + "test", + "background-test" + ] + }, + "prediction_input_field": "observed_anomalies", + "predictions_post_filter": [ + { + "code": "D-CRS-WRONG", + "severity": "blocker" + }, + { + "code": "D-INVALID-GEOMETRY", + "severity": "critical" + }, + { + "code": "D-RESOLUTION", + "severity": "major" + } + ], + "predictions_pre_filter": [ + { + "code": "D-CRS-WRONG", + "severity": "blocker" + }, + { + "code": "D-INVALID-GEOMETRY", + "severity": "critical" + }, + { + "code": "D-RESOLUTION", + "severity": "major" + } + ], + "reference_input_field": "expected_anomalies", + "references": [ + { + "code": "D-CRS-WRONG", + "severity": "blocker" + }, + { + "code": "D-INVALID-GEOMETRY", + "severity": "critical" + }, + { + "code": "D-MISSING-PROVENANCE", + "severity": "critical" + } + ], + "sample_id": "validation-test-national", + "split": "test", + "task": "geospatial_data_validation" + }, + { + "classes": [ + "building" + ], + "config": { + "match_iou": 0.5 + }, + "false_negatives": [ + { + "class": "building", + "id": "r2", + "polygon": [ + [ + 10, + 0 + ], + [ + 14, + 0 + ], + [ + 14, + 4 + ], + [ + 10, + 4 + ], + [ + 10, + 0 + ] + ] + } + ], + "false_positives": [ + { + "class": "building", + "id": "p2", + "polygon": [ + [ + 20, + 0 + ], + [ + 24, + 0 + ], + [ + 24, + 4 + ], + [ + 20, + 4 + ], + [ + 20, + 0 + ] + ] + } + ], + "filter": { + "applied": false + }, + "hashes": { + "canonicalization": "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), ensure_ascii=false and allow_nan=false", + "case_input_canonical_json_sha256": "e3b554b9b91a89214b9ed7703f4fb3463607024f35a33b934c94ce8c783d405a", + "predictions_post_filter_canonical_json_sha256": "b2a08e0928cd1d93e4548f95a1a40bdf404891bdaa9b876cce0f684edcce3fed", + "predictions_pre_filter_canonical_json_sha256": "b2a08e0928cd1d93e4548f95a1a40bdf404891bdaa9b876cce0f684edcce3fed", + "references_canonical_json_sha256": "06b0c3de29764b1ccc34e1094e7d5aaadace27fa8f4dc4d6022f01db441ad663" + }, + "input_lineage": { + "prediction": { + "derivation": "hand_authored_fixed_output", + "source_id": "synthetic:vector-test-flanders-suburban:prediction", + "source_version": "1" + }, + "reference": { + "derivation": "hand_authored_contract_fixture", + "source_id": "synthetic:vector-test-flanders-suburban:reference", + "source_version": "1" + } + }, + "input_sha256": "e3b554b9b91a89214b9ed7703f4fb3463607024f35a33b934c94ce8c783d405a", + "matches": [ + { + "class": "building", + "confidence": null, + "overlap": 1.0, + "prediction_id": "p1", + "reference_id": "r1" + } + ], + "metadata": { + "context": "suburban_buildings", + "date": "2025-01-15", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Mol", + "object_size": "mixed", + "occlusion": "not_applicable", + "ood": false, + "region": "flanders", + "resolution_m": 0.1, + "season": "not_applicable", + "sensor": "vector", + "source": "fixture-grb", + "tile_edge": false, + "urbanity": "suburban", + "vegetation": "low" + }, + "portfolio_lineage": { + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "declared": { + "origin": "repository_fixture", + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "version": "1" + }, + "portfolio_canonical_json_sha256": "1ac1bd77db125db48f9168406d51bfb723522f54a6c81dbe868d600c72f27b17", + "portfolio_file_sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "portfolio_id": "geointel-p4-reference-harness-v2", + "portfolio_kind": "synthetic_contract", + "portfolio_schema_version": 2, + "source_path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "split_roles": [ + "test", + "background-test" + ] + }, + "prediction_input_field": "predictions", + "predictions_post_filter": [ + { + "class": "building", + "id": "p1", + "polygon": [ + [ + 0, + 0 + ], + [ + 4, + 0 + ], + [ + 4, + 4 + ], + [ + 0, + 4 + ], + [ + 0, + 0 + ] + ] + }, + { + "class": "building", + "id": "p2", + "polygon": [ + [ + 20, + 0 + ], + [ + 24, + 0 + ], + [ + 24, + 4 + ], + [ + 20, + 4 + ], + [ + 20, + 0 + ] + ] + } + ], + "predictions_pre_filter": [ + { + "class": "building", + "id": "p1", + "polygon": [ + [ + 0, + 0 + ], + [ + 4, + 0 + ], + [ + 4, + 4 + ], + [ + 0, + 4 + ], + [ + 0, + 0 + ] + ] + }, + { + "class": "building", + "id": "p2", + "polygon": [ + [ + 20, + 0 + ], + [ + 24, + 0 + ], + [ + 24, + 4 + ], + [ + 20, + 4 + ], + [ + 20, + 0 + ] + ] + } + ], + "reference_input_field": "references", + "references": [ + { + "class": "building", + "id": "r1", + "polygon": [ + [ + 0, + 0 + ], + [ + 4, + 0 + ], + [ + 4, + 4 + ], + [ + 0, + 4 + ], + [ + 0, + 0 + ] + ] + }, + { + "class": "building", + "id": "r2", + "polygon": [ + [ + 10, + 0 + ], + [ + 14, + 0 + ], + [ + 14, + 4 + ], + [ + 10, + 4 + ], + [ + 10, + 0 + ] + ] + } + ], + "sample_id": "vector-test-flanders-suburban", + "split": "test", + "task": "vector_comparison" + } + ], + "items_canonical_json_sha256": "14c27341951d8d250578013d7ae3c5fe6c585a6ea84752410ab786fc2d5f805a", + "portfolio_file_sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "schema_version": 2 +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/benchmark-manifest.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/benchmark-manifest.json new file mode 100644 index 00000000..56d2fe6a --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/benchmark-manifest.json @@ -0,0 +1,105 @@ +{ + "benchmark_id": "geointel-p4-reference-harness-v2", + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "code": [ + { + "path": "scripts/accuracy_phase4_evaluator.py", + "sha256": "d2e40f7978d3ff4ce7025eaaaefb06ccc76c269d6874f8b0f528739d535a1da1", + "size_bytes": 93353 + }, + { + "path": "scripts/generate_accuracy_phase4_splits.py", + "sha256": "d12842922bcd99b1672176df15a9cef250128e7d7fc662ab15e16f1c084ba570", + "size_bytes": 75923 + }, + { + "path": "scripts/run_accuracy_phase4_benchmark.py", + "sha256": "003ecb4babc7f56a6d13e9e1c08dda902b9b26203cc7d53476d4d5e083b8b85b", + "size_bytes": 123717 + }, + { + "path": "scripts/run_golden_qa_benchmark.py", + "sha256": "2c8a19fd92db8f07f491b8f7b34d9736384f81e02a6b83019fac71b101ebdaa0", + "size_bytes": 10856 + }, + { + "path": "backend/app/services/qa_service.py", + "sha256": "2d4f645410fbf2972e55dddbf5a85db5368b2b1932792cd522ec51c483339244", + "size_bytes": 15206 + } + ], + "evaluation_results_canonical_json_sha256": "55cdba9cce4dcdc88cee0829cd510d7e624a3472feb9d33f7e681621a792eba3", + "evaluator_version": "2.1.0", + "evidence_run_id": "p4-2.0.1-9677d0ef37db82bcf39b", + "inference_and_selection": { + "background_test_used_for_selection": false, + "challenge_labels_available": false, + "production_model_inference_executed": false, + "raw_predictions_retained": true, + "synthetic_reference_harness": true, + "test_used_for_selection": false, + "threshold_source": "pre_registered_configuration_only" + }, + "input_manifest": { + "path": "input-manifest.json", + "sha256": "44f1c81d76baf4dc965afc2cd109ac165ae8fe0308f202018ea21bac786bcd0b" + }, + "inputs": { + "golden_qa_manifest": { + "path": "fixtures/golden/golden_qa_benchmarks.json", + "sha256": "d60576ff7048ca3a048dc166e8977f3a31e9f8fa352be5ad222a017260e179e2", + "size_bytes": 2758 + }, + "phase3_full_scan": { + "path": "artifacts/evidence/accuracy/P3/full-scan-manifest.json", + "sha256": "28c1022d125f30bfb4aa362b57d8a90cf8c88270b2a9ac6582b86b6a44a08038", + "size_bytes": 755527 + }, + "phase3_leakage": { + "path": "artifacts/evidence/accuracy/P3/leakage-report.json", + "sha256": "8e1b7a24a840549d442bc03132ee1f21c6d50f8714cbb8d33430391a3769cd4e", + "size_bytes": 580 + }, + "protected_cases": { + "path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "size_bytes": 20091 + }, + "split_source": { + "path": "fixtures/accuracy/p4/split-source-manifest.json", + "sha256": "27ec60406ac57a02f1335013ee344a43d4e0093e7ee1329ec22876da623f5bfc", + "size_bytes": 17331 + } + }, + "manifest_sha256": "0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642", + "product_baseline": { + "artifacts": [], + "manifest": null, + "product_gate_evidence_sha256": "c9f5e3de3ab6d64911f8807b26caa661ccf911a5fd6b995107edda2004836e0c", + "requested_path": "artifacts/evidence/accuracy/P4/product-baseline-manifest.json", + "validation_status": "not_evaluable" + }, + "product_gate_evidence_sha256": "c9f5e3de3ab6d64911f8807b26caa661ccf911a5fd6b995107edda2004836e0c", + "reference_baseline_sha256": "aeb0f6510352f4e205e4fc4e8c423110ca72c2142016c7af846d758865f258c6", + "repository_commit": "70fb4b94e5cb7c248beec5a936ce186f38cc183c", + "runtime": { + "cuda_used_for_reference_harness": false, + "dependencies": { + "numpy": "2.2.2", + "pyproj": "3.7.2", + "shapely": "2.1.2" + }, + "execution_device": "CPU deterministic evaluator arithmetic; no production model inference", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.13.2", + "python_implementation": "CPython" + }, + "schema_version": 2, + "split_generator_version": "1.3.0", + "split_manifests": { + "development_sha256": "64e4fc986c2390bedfbe100062d4b06e1098689f4d4a9a9da13a7f67357c0493", + "leakage_status": "pass", + "protected_sha256": "e57ca1529654ef1c2527e84122ff26ec3533e7c70658250983968fbc5d4af410" + }, + "workflow_version": "2.0.1" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/calibration-metrics.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/calibration-metrics.json new file mode 100644 index 00000000..1cb1027b --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/calibration-metrics.json @@ -0,0 +1,274 @@ +{ + "items": [ + { + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "coverage_risk": [ + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.0, + "total_prediction_count": 1 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.5, + "total_prediction_count": 1 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.7, + "total_prediction_count": 1 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.9, + "total_prediction_count": 1 + } + ], + "sample_id": "background-test-hard-negative", + "task": "object_detection" + }, + { + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "coverage_risk": [ + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.0, + "total_prediction_count": 0 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.5, + "total_prediction_count": 0 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.7, + "total_prediction_count": 0 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.9, + "total_prediction_count": 0 + } + ], + "sample_id": "background-test-pure-empty", + "task": "object_detection" + }, + { + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "coverage_risk": [ + { + "false_negative_count": 0, + "false_positive_count": 2, + "matched_reference_count": 2, + "reference_count": 2, + "reference_coverage": 1.0, + "retained_prediction_count": 4, + "retained_prediction_coverage": 1.0, + "risk": 0.5, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.0, + "total_prediction_count": 4 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 2, + "reference_count": 2, + "reference_coverage": 1.0, + "retained_prediction_count": 3, + "retained_prediction_coverage": 0.75, + "risk": 0.3333333333333333, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.5, + "total_prediction_count": 4 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 2, + "reference_count": 2, + "reference_coverage": 1.0, + "retained_prediction_count": 2, + "retained_prediction_coverage": 0.5, + "risk": 0.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.7, + "total_prediction_count": 4 + }, + { + "false_negative_count": 1, + "false_positive_count": 0, + "matched_reference_count": 1, + "reference_count": 2, + "reference_coverage": 0.5, + "retained_prediction_count": 1, + "retained_prediction_coverage": 0.25, + "risk": 0.5, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.9, + "total_prediction_count": 4 + } + ], + "sample_id": "det-test-flanders-urban", + "task": "object_detection" + } + ], + "note": "Fixed diagnostic bins and risk thresholds test metric arithmetic; they do not select or change any operating point.", + "schema_version": 2, + "selection_allowed": false, + "status": "fixture_diagnostic_only" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/candidate-vs-incumbent.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/candidate-vs-incumbent.json new file mode 100644 index 00000000..e66be8eb --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/candidate-vs-incumbent.json @@ -0,0 +1,12 @@ +{ + "future_gate_contract": { + "aggregate_improvement_may_mask_subgroup_regression": false, + "global_and_critical_subgroups_required": true, + "missing_or_insufficient_support": "not_evaluable", + "numeric_margin": "to_be_frozen_before_protected_access", + "unit": "paired independent AOI" + }, + "reason": "Phase 4 has no valid real incumbent product baseline and no pre-registered candidate; synthetic fixture values cannot define non-inferiority.", + "schema_version": 2, + "status": "not_evaluable" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/development-split-manifest.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/development-split-manifest.json new file mode 100644 index 00000000..166e8fcc --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/development-split-manifest.json @@ -0,0 +1,334 @@ +{ + "allowed_splits": [ + "calibration", + "train", + "val" + ], + "assignment_algorithm": "group-before-split-deficit-balancer-v1", + "assignment_mode": "preassigned", + "assignment_seed": "geointel-p4-group-split-v1", + "claim_boundary": "Synthetic contract fixtures for evaluator regression only; never production accuracy evidence.", + "crs": "EPSG:31370", + "dataset_version": "geointel-p4-harness-fixture-v2", + "generator_version": "1.3.0", + "independence_buffer_m": 2000.0, + "label_geometry_hamming_threshold": 2, + "mandatory_split_roles": [ + "train", + "val", + "calibration", + "test", + "background-test", + "challenge" + ], + "manifest_role": "development_and_calibration", + "manifest_sha256": "64e4fc986c2390bedfbe100062d4b06e1098689f4d4a9a9da13a7f67357c0493", + "perceptual_hamming_threshold": 4, + "samples": [ + { + "acquisition_date": "2025-01-07", + "acquisition_id": "acquisition-07", + "bbox": [ + 70000.0, + 30000.0, + 70100.0, + 30100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g07", + "label_geometry_fingerprint": "dba609f34cb9826b", + "label_geometry_hash": "6767676767676767676767676767676767676767676767676767676767676767", + "label_sha256": "4747474747474747474747474747474747474747474747474747474747474747", + "native_feature_ids": [ + "native-07-1" + ], + "object_ids": [ + "o21" + ], + "parent_raster_id": "parent-raster-07", + "perceptual_image_hash": "c4316514fed98f6b", + "processed_image_sha256": "2727272727272727272727272727272727272727272727272727272727272727", + "raw_image_sha256": "0707070707070707070707070707070707070707070707070707070707070707", + "record_sha256": "3f04e62eb97c0a067a4742084a9665607beab43a5c0392a89c77bc10cb472136", + "sample_id": "calibration-det-a", + "source_family": "scene-07", + "split": "calibration", + "task": "object_detection", + "temporal_family": "temporal-07" + }, + { + "acquisition_date": "2025-01-08", + "acquisition_id": "acquisition-08", + "bbox": [ + 90000.0, + 30000.0, + 90100.0, + 30100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g08", + "label_geometry_fingerprint": "627df87eea48aec0", + "label_geometry_hash": "6868686868686868686868686868686868686868686868686868686868686868", + "label_sha256": "4848484848484848484848484848484848484848484848484848484848484848", + "native_feature_ids": [ + "native-08-1" + ], + "object_ids": [ + "o22" + ], + "parent_raster_id": "parent-raster-08", + "perceptual_image_hash": "518ba3145333c08c", + "processed_image_sha256": "2828282828282828282828282828282828282828282828282828282828282828", + "raw_image_sha256": "0808080808080808080808080808080808080808080808080808080808080808", + "record_sha256": "d3cec9ed2d1a4c7380958a960ac24d1dbc5a729ccb81cc0f22016ee348137a07", + "sample_id": "calibration-seg-b", + "source_family": "scene-08", + "split": "calibration", + "task": "footprint_segmentation", + "temporal_family": "temporal-08" + }, + { + "acquisition_date": "2025-01-01", + "acquisition_id": "acquisition-01", + "bbox": [ + 10000.0, + 10000.0, + 10100.0, + 10100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g01", + "label_geometry_fingerprint": "c0f62e16b4033ee9", + "label_geometry_hash": "6161616161616161616161616161616161616161616161616161616161616161", + "label_sha256": "4141414141414141414141414141414141414141414141414141414141414141", + "native_feature_ids": [ + "native-01-1" + ], + "object_ids": [ + "o01" + ], + "parent_raster_id": "parent-raster-01", + "perceptual_image_hash": "dcef24412401d4f9", + "processed_image_sha256": "2121212121212121212121212121212121212121212121212121212121212121", + "raw_image_sha256": "0101010101010101010101010101010101010101010101010101010101010101", + "record_sha256": "16142e63a8e93d20d1fcfb66cfd8aeeebc6cbe7784f5ce8c403cd4b2f6e5dcac", + "sample_id": "det-train-a", + "source_family": "scene-01", + "split": "train", + "task": "object_detection", + "temporal_family": "temporal-01" + }, + { + "acquisition_date": "2025-01-03", + "acquisition_id": "acquisition-03", + "bbox": [ + 30000.0, + 10000.0, + 30100.0, + 10100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g03", + "label_geometry_fingerprint": "30820f2c124a3ff0", + "label_geometry_hash": "6363636363636363636363636363636363636363636363636363636363636363", + "label_sha256": "4343434343434343434343434343434343434343434343434343434343434343", + "native_feature_ids": [ + "native-03-1" + ], + "object_ids": [ + "o03" + ], + "parent_raster_id": "parent-raster-03", + "perceptual_image_hash": "75e36e008b4a6702", + "processed_image_sha256": "2323232323232323232323232323232323232323232323232323232323232323", + "raw_image_sha256": "0303030303030303030303030303030303030303030303030303030303030303", + "record_sha256": "60c78f7f4cb086681fa38d3a2e160824003cb646890faf6ddde142eab585adc8", + "sample_id": "raster-train-a", + "source_family": "scene-03", + "split": "train", + "task": "raster_classification", + "temporal_family": "temporal-03" + }, + { + "acquisition_date": "2025-01-02", + "acquisition_id": "acquisition-02", + "bbox": [ + 20000.0, + 10000.0, + 20100.0, + 10100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g02", + "label_geometry_fingerprint": "666a0fc2d26265ee", + "label_geometry_hash": "6262626262626262626262626262626262626262626262626262626262626262", + "label_sha256": "4242424242424242424242424242424242424242424242424242424242424242", + "native_feature_ids": [ + "native-02-1" + ], + "object_ids": [ + "o02" + ], + "parent_raster_id": "parent-raster-02", + "perceptual_image_hash": "030bcdd3bc87412e", + "processed_image_sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "raw_image_sha256": "0202020202020202020202020202020202020202020202020202020202020202", + "record_sha256": "b92bff2d0e039441711100b95178a6bebab0052db1b7e35259dcbcb227d50448", + "sample_id": "seg-train-a", + "source_family": "scene-02", + "split": "train", + "task": "footprint_segmentation", + "temporal_family": "temporal-02" + }, + { + "acquisition_date": "2025-01-05", + "acquisition_id": "acquisition-05", + "bbox": [ + 30000.0, + 30000.0, + 30100.0, + 30100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g05", + "label_geometry_fingerprint": "14ceff3640a013eb", + "label_geometry_hash": "6565656565656565656565656565656565656565656565656565656565656565", + "label_sha256": "4545454545454545454545454545454545454545454545454545454545454545", + "native_feature_ids": [ + "native-05-1" + ], + "object_ids": [ + "o05" + ], + "parent_raster_id": "parent-raster-05", + "perceptual_image_hash": "828b99671c3f2f39", + "processed_image_sha256": "2525252525252525252525252525252525252525252525252525252525252525", + "raw_image_sha256": "0505050505050505050505050505050505050505050505050505050505050505", + "record_sha256": "bc142eb7eb7ed86621984ff89184b1b79cfaa13e1011832c8b8175312761ea91", + "sample_id": "val-change-b", + "source_family": "scene-05", + "split": "val", + "task": "change_detection", + "temporal_family": "temporal-05" + }, + { + "acquisition_date": "2025-01-06", + "acquisition_id": "acquisition-06", + "bbox": [ + 50000.0, + 30000.0, + 50100.0, + 30100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g06", + "label_geometry_fingerprint": "a1d9bbb93ba6d6da", + "label_geometry_hash": "6666666666666666666666666666666666666666666666666666666666666666", + "label_sha256": "4646464646464646464646464646464646464646464646464646464646464646", + "native_feature_ids": [ + "native-06-1" + ], + "object_ids": [ + "o06" + ], + "parent_raster_id": "parent-raster-06", + "perceptual_image_hash": "11c11105b0d10635", + "processed_image_sha256": "2626262626262626262626262626262626262626262626262626262626262626", + "raw_image_sha256": "0606060606060606060606060606060606060606060606060606060606060606", + "record_sha256": "a861ffb75dba0e50e4082232620521465acecdbc0cf1f6dee9b7509d9c0d480e", + "sample_id": "val-terrain-c", + "source_family": "scene-06", + "split": "val", + "task": "terrain_interpretation", + "temporal_family": "temporal-06" + }, + { + "acquisition_date": "2025-01-04", + "acquisition_id": "acquisition-04", + "bbox": [ + 10000.0, + 30000.0, + 10100.0, + 30100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g04", + "label_geometry_fingerprint": "a559b46cba64e501", + "label_geometry_hash": "6464646464646464646464646464646464646464646464646464646464646464", + "label_sha256": "4444444444444444444444444444444444444444444444444444444444444444", + "native_feature_ids": [ + "native-04-1" + ], + "object_ids": [ + "o04" + ], + "parent_raster_id": "parent-raster-04", + "perceptual_image_hash": "2e3b331cc01536c9", + "processed_image_sha256": "2424242424242424242424242424242424242424242424242424242424242424", + "raw_image_sha256": "0404040404040404040404040404040404040404040404040404040404040404", + "record_sha256": "e2a931aa255519d0bd87655d92132164923b369a31e12583c8aa499610044a30", + "sample_id": "val-vector-a", + "source_family": "scene-04", + "split": "val", + "task": "vector_comparison", + "temporal_family": "temporal-04" + } + ], + "schema_version": 1, + "selection_access_allowed_by_split": { + "calibration": true, + "train": false, + "val": true + }, + "source_manifest_sha256": "1c75c875d4c297d046117bb9393ba5e648f13d3358ea92cb65374869c8f5fc01", + "source_trust": { + "mode": "synthetic_fixture", + "policy_id": "geointel-p4-synthetic-fixture-v2", + "production_accuracy_use_allowed": false + }, + "training_access_allowed_by_split": { + "calibration": false, + "train": true, + "val": false + } +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/error-taxonomy.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/error-taxonomy.json new file mode 100644 index 00000000..d8a9f814 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/error-taxonomy.json @@ -0,0 +1,48 @@ +{ + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "observed_codes": [ + { + "error_code": "D-VALIDATION-FN", + "kind": "validation_false_negative" + }, + { + "error_code": "D-VALIDATION-FP", + "kind": "validation_false_positive" + }, + { + "error_code": "M-AREA-BIAS", + "kind": "area_bias" + }, + { + "error_code": "M-CLASS", + "kind": "raster_misclassification" + }, + { + "error_code": "M-FN-MISSED", + "kind": "event_false_negative" + }, + { + "error_code": "M-FN-MISSED", + "kind": "false_negative" + }, + { + "error_code": "M-FP-CONFUSER", + "kind": "event_false_positive" + }, + { + "error_code": "M-FP-CONFUSER", + "kind": "false_positive" + }, + { + "error_code": "M-MISCALIBRATED", + "kind": "miscalibrated" + }, + { + "error_code": "P-PARTIAL", + "kind": "terrain_missing" + } + ], + "observed_failure_count": 14, + "schema_version": 2, + "source": "docs/accuracy-program/05-metric-framework.md section 4" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/evaluation-contract.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/evaluation-contract.json new file mode 100644 index 00000000..a499c3cc --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/evaluation-contract.json @@ -0,0 +1,300 @@ +{ + "benchmark_id": "geointel-p4-reference-harness-v2", + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "evaluator_families": [ + "change_detection", + "footprint_segmentation", + "geospatial_data_validation", + "object_detection", + "raster_classification", + "terrain_interpretation", + "vector_comparison" + ], + "evaluator_version": "2.1.0", + "gate_states": [ + "fail", + "not_evaluable", + "pass" + ], + "implemented_capabilities": [ + { + "capability_id": "model_object_detection", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "model_inference_pipeline", + "implementation_paths": [ + "backend/app/services/detection_service.py" + ], + "suitable_metrics": [ + "precision", + "recall", + "F1", + "AP50", + "AP50-95", + "IoU", + "ECE", + "Brier", + "coverage-risk" + ], + "task": "object_detection" + }, + { + "capability_id": "building_proposal_filtering", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "not_separately_benchmarked", + "implementation_kind": "supporting_candidate_classifier", + "implementation_paths": [ + "scripts/train_building_proposal_classifier.py", + "scripts/evaluate_belgium_building_candidate.py" + ], + "suitable_metrics": [ + "candidate precision", + "candidate recall", + "F1", + "calibration" + ], + "task": "object_detection" + }, + { + "capability_id": "footprint_segmentation", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "model_inference_pipeline", + "implementation_paths": [ + "backend/app/services/segmentation_service.py", + "backend/app/services/segmentation_adapter.py" + ], + "suitable_metrics": [ + "object precision", + "object recall", + "F1", + "IoU", + "Dice", + "boundary F1", + "centroid distance", + "area error", + "topology" + ], + "task": "footprint_segmentation" + }, + { + "capability_id": "detection_and_vector_qa", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "deterministic_geospatial_comparison", + "implementation_paths": [ + "backend/app/services/qa_service.py", + "backend/app/services/detection_qa_service.py", + "backend/app/services/quality_check_service.py" + ], + "suitable_metrics": [ + "precision", + "recall", + "F1", + "IoU", + "topology", + "coverage" + ], + "task": "vector_comparison" + }, + { + "capability_id": "vector_clip_buffer_intersect", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "covered_by_family_not_separately_benchmarked", + "implementation_kind": "deterministic_vector_processing", + "implementation_paths": [ + "backend/app/services/vector_operations_service.py", + "backend/app/services/vector_feature_service.py" + ], + "suitable_metrics": [ + "geometry validity", + "CRS correctness", + "area conservation", + "feature counts", + "topology" + ], + "task": "vector_comparison" + }, + { + "capability_id": "temporal_vector_change", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "deterministic_change_detection", + "implementation_paths": [ + "backend/app/services/change_detection_service.py" + ], + "suitable_metrics": [ + "event precision", + "event recall", + "event F1", + "IoU" + ], + "task": "change_detection" + }, + { + "capability_id": "thematic_raster_interpretation", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_metric_contract_only_no_generic_learned_classifier_claim", + "implementation_kind": "deterministic_source_interpretation", + "implementation_paths": [ + "backend/app/services/thematic_raster_analysis_service.py" + ], + "suitable_metrics": [ + "pixel accuracy", + "per-class F1", + "per-class IoU", + "mean IoU" + ], + "task": "raster_classification" + }, + { + "capability_id": "raster_clip_reproject_indices", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "covered_by_family_not_separately_benchmarked", + "implementation_kind": "deterministic_raster_processing", + "implementation_paths": [ + "backend/app/services/raster_service.py", + "backend/app/services/raster_operations_service.py" + ], + "suitable_metrics": [ + "CRS/transform preservation", + "pixel alignment", + "nodata", + "numeric tolerance" + ], + "task": "raster_classification" + }, + { + "capability_id": "raster_partition_mosaic", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "not_separately_benchmarked", + "implementation_kind": "deterministic_raster_partitioning", + "implementation_paths": [ + "backend/app/services/raster_partition_analysis_service.py" + ], + "suitable_metrics": [ + "seam equality", + "coverage completeness", + "resolution consistency" + ], + "task": "raster_classification" + }, + { + "capability_id": "terrain_height_interpretation", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "deterministic_continuous_raster_analysis", + "implementation_paths": [ + "backend/app/services/terrain_analysis_service.py", + "backend/app/services/spw_terrain_service.py" + ], + "suitable_metrics": [ + "MAE", + "RMSE", + "bias", + "coverage", + "unit integrity" + ], + "task": "terrain_interpretation" + }, + { + "capability_id": "flood_hazard_interpretation", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "covered_by_family_not_separately_benchmarked", + "implementation_kind": "deterministic_scenario_raster_analysis", + "implementation_paths": [ + "backend/app/services/flood_hazard_analysis_service.py" + ], + "suitable_metrics": [ + "depth MAE/RMSE", + "hazard-class IoU", + "coverage", + "scenario identity" + ], + "task": "terrain_interpretation" + }, + { + "capability_id": "bathymetry_interpretation", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "covered_by_family_not_separately_benchmarked", + "implementation_kind": "deterministic_vertical_reference_analysis", + "implementation_paths": [ + "backend/app/services/bathymetry_raster_analysis_service.py", + "backend/app/services/mdk_bathymetry_probe_service.py" + ], + "suitable_metrics": [ + "MAE", + "RMSE", + "bias", + "coverage", + "vertical-datum integrity" + ], + "task": "terrain_interpretation" + }, + { + "capability_id": "data_contract_validation_and_scan", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "deterministic_validation", + "implementation_paths": [ + "backend/app/services/data_contract_validation.py", + "scripts/run_accuracy_phase3_full_data_scan.py" + ], + "suitable_metrics": [ + "anomaly precision", + "anomaly recall", + "anomaly F1", + "critical misses" + ], + "task": "geospatial_data_validation" + }, + { + "capability_id": "aoi_partition_orchestration", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "not_separately_benchmarked", + "implementation_kind": "deterministic_orchestration", + "implementation_paths": [ + "backend/app/services/aoi_operation_service.py", + "backend/app/services/aoi_operation_executor.py", + "backend/app/services/aoi_operation_worker.py" + ], + "suitable_metrics": [ + "partition completeness", + "overlap/gap", + "idempotency", + "resume correctness" + ], + "task": "geospatial_data_validation" + }, + { + "capability_id": "geo_assistant_orchestration", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "no_independent_accuracy_score_underlying_tool_results_are_authoritative", + "implementation_kind": "tool_orchestration_interface", + "implementation_paths": [ + "backend/app/services/geo_assistant_service.py" + ], + "suitable_metrics": [ + "tool-selection correctness", + "grounding", + "unsupported-claim rate" + ], + "task": "geospatial_data_validation" + } + ], + "metric_contract": { + "path": "docs/accuracy-program/05-metric-framework.md", + "sha256": "18abfc21964b7d98848d8ae513afce242184980c9517c9184d40efd9165ea00d", + "size_bytes": 28053 + }, + "protected_policy": { + "diagnostic_curves_select_operating_point": false, + "operating_point_selection_allowed": false, + "test_feedback_allowed": false, + "threshold_selection_source": "pre_registered_configuration_only" + }, + "schema_version": 2, + "split_generator_version": "1.3.0", + "undefined_value_policy": "Undefined denominators are null with numerator, denominator and support; they are never coerced to a perfect score.", + "workflow_version": "2.0.1" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/evidence-manifest.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/evidence-manifest.json new file mode 100644 index 00000000..5e219031 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/evidence-manifest.json @@ -0,0 +1,129 @@ +{ + "artifact_count": 24, + "artifacts": [ + { + "path": "acceptance-gates.json", + "sha256": "e5f1c9c43c8da22acfa5486e641c010a278431f90634f91335b039ab7bdb50a4", + "size_bytes": 5517 + }, + { + "path": "aoi-metrics.json", + "sha256": "2f5424c4a624316d0c8e71696440080d535ec0682831a80d664694f7cc198e94", + "size_bytes": 430 + }, + { + "path": "baseline-raw-predictions.json", + "sha256": "d03ec665fcb3efb9546c1e5a42888cd2775307e3c7ba90d225389caca7e1140f", + "size_bytes": 42170 + }, + { + "path": "benchmark-manifest.json", + "sha256": "868dd7eb2dfa8344e417879ed3f5dd7c7d9e5673add7e3e2294c5cbde4b75d57", + "size_bytes": 4212 + }, + { + "path": "calibration-metrics.json", + "sha256": "ad54f924512529deba6d869736ca3f72dde8651227943c0b9e8695912e7a0c3d", + "size_bytes": 8858 + }, + { + "path": "candidate-vs-incumbent.json", + "sha256": "7ee6fb5722b43fa0dc0a8e966547c954c62c7401b10e6f727e21d332b34f077f", + "size_bytes": 517 + }, + { + "path": "development-split-manifest.json", + "sha256": "636b1d38695ade99ee3896a2d870fcaf0635168e8e25657353ac51a7bfba2248", + "size_bytes": 12424 + }, + { + "path": "error-taxonomy.json", + "sha256": "acf2804e482dfdca1df05ef2c6e40701b82877466761b880b314e46699f470dc", + "size_bytes": 1116 + }, + { + "path": "evaluation-contract.json", + "sha256": "797598cf3060cffb30f352ed022459c3be2e656c19b190c38b66fb99dbe54bd2", + "size_bytes": 11057 + }, + { + "path": "failure-gallery.json", + "sha256": "eaf19cbad22a44fc2b6420ff0591536a61560815f472bf7fb54ed0179bd93986", + "size_bytes": 14852 + }, + { + "path": "generation-status.json", + "sha256": "3311ae1ddf622f94c1a2e596b24454e0c033428b5d767d8cc3c67ee4e33a1a63", + "size_bytes": 374 + }, + { + "path": "human-review-summary.json", + "sha256": "1ff22096bccb03ca5188d53bfe5f5d4a8ac937fdb43265c20b676f9211d1abbd", + "size_bytes": 201 + }, + { + "path": "input-manifest.json", + "sha256": "44f1c81d76baf4dc965afc2cd109ac165ae8fe0308f202018ea21bac786bcd0b", + "size_bytes": 6092 + }, + { + "path": "latency-and-reliability.json", + "sha256": "7c89bc0562b99565f6b1c54eac562f129192481ac61571ab48fb69a1f8553f95", + "size_bytes": 263 + }, + { + "path": "leakage-gate-report.json", + "sha256": "ea532fc95b4e3be30299b6ec1939a42166c8107214b4264208c1a10c2abc0ff8", + "size_bytes": 1221 + }, + { + "path": "metric-report.json", + "sha256": "6c92f31c1b0363d031e03968a50985e4e27075dc33ef03e9ce5e6a7fbb30c4f0", + "size_bytes": 403544 + }, + { + "path": "object-metrics.json", + "sha256": "30cb52b1ee18a75bb68304ec6b88f54455a53e68052392232dec370ac3a68a10", + "size_bytes": 24826 + }, + { + "path": "protected-split-manifest.json", + "sha256": "f294af0b2957af7de6ec533e4495d9d0d75da86cc0d29e265ed6998104829c64", + "size_bytes": 20986 + }, + { + "path": "reference-implementation-baseline.json", + "sha256": "1c8fd9970d91449842922ba38f733da9d99142ce5f2a03736d14f9f61ad849a6", + "size_bytes": 4710 + }, + { + "path": "release-gate-report.json", + "sha256": "e5f1c9c43c8da22acfa5486e641c010a278431f90634f91335b039ab7bdb50a4", + "size_bytes": 5517 + }, + { + "path": "split-and-leakage-audit.json", + "sha256": "ea532fc95b4e3be30299b6ec1939a42166c8107214b4264208c1a10c2abc0ff8", + "size_bytes": 1221 + }, + { + "path": "stratified-metrics.json", + "sha256": "66f7b7ed639cef400e52cada5d2274f8c185616b4beb6b44bcc89a8aa9929048", + "size_bytes": 326587 + }, + { + "path": "tile-metrics.json", + "sha256": "0920c5d0b2076ed1ebcc06144f4cf786c3a18d1d67f9e388314f7f9ccb857256", + "size_bytes": 6344 + }, + { + "path": "workflow-summary.json", + "sha256": "bcb4469fcaf4c360ed716037484117edf4e93c6b79bbdcf7f75cd3f79ca093a6", + "size_bytes": 1035 + } + ], + "benchmark_id": "geointel-p4-reference-harness-v2", + "claim_boundary": "Immutable local reference-harness evidence only; product accuracy and release remain blocked while product gates are not evaluable or fail.", + "phase": "P4", + "schema_version": 2 +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/failure-gallery.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/failure-gallery.json new file mode 100644 index 00000000..62225eee --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/failure-gallery.json @@ -0,0 +1,537 @@ +{ + "failure_count": 14, + "items": [ + { + "contexts": [], + "error_code": "P-PARTIAL", + "evidence": { + "missing_index": 3 + }, + "failure_id": "1397377fd37cd400f01a", + "kind": "terrain_missing", + "metadata": { + "context": "terrain_profile", + "date": "2023-01-01", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Hasselt", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "flanders", + "resolution_m": 5, + "season": "not_applicable", + "sensor": "elevation-raster", + "source": "fixture-dhmv", + "tile_edge": false, + "urbanity": "rural", + "vegetation": "moderate" + }, + "sample_id": "terrain-test-flanders-rural", + "secondary_error_codes": [], + "task": "terrain_interpretation" + }, + { + "contexts": [], + "error_code": "M-FN-MISSED", + "evidence": { + "bbox": [ + 10, + 0, + 15, + 5 + ], + "class": "removed", + "id": "r2" + }, + "failure_id": "28a0be1f9d4960036f3a", + "kind": "event_false_negative", + "metadata": { + "context": "industrial_change", + "date": "2024-05-01/2025-05-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Liège", + "object_size": "mixed", + "occlusion": "none", + "ood": false, + "region": "wallonia", + "resolution_m": 0.5, + "season": "multi-date", + "sensor": "vector", + "source": "fixture-picc-temporal", + "tile_edge": false, + "urbanity": "industrial", + "vegetation": "low" + }, + "sample_id": "change-test-wallonia-industrial", + "secondary_error_codes": [], + "task": "change_detection" + }, + { + "contexts": [], + "error_code": "M-AREA-BIAS", + "evidence": { + "prediction_id": "p1", + "reference_id": "r1", + "relative_area_error": -0.1 + }, + "failure_id": "2d53b69d95b17bf49580", + "kind": "area_bias", + "metadata": { + "context": "rural_buildings", + "date": "2025-04-20", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Namur", + "object_size": "large", + "occlusion": "none", + "ood": false, + "region": "wallonia", + "resolution_m": 0.5, + "season": "spring", + "sensor": "synthetic-rgb", + "source": "fixture-picc", + "tile_edge": false, + "urbanity": "rural", + "vegetation": "low" + }, + "sample_id": "seg-test-wallonia-rural", + "secondary_error_codes": [], + "task": "footprint_segmentation" + }, + { + "contexts": [ + "tile_edge" + ], + "error_code": "M-CLASS", + "evidence": { + "column": 1, + "pixel_index": 9, + "prediction": 1, + "reason": "class_mismatch", + "reference": 2, + "row": 2 + }, + "failure_id": "2fabc83865f9e1f49efc", + "kind": "raster_misclassification", + "metadata": { + "context": "dense_urban_raster", + "date": "2025-10-05", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Brussel", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "brussels", + "resolution_m": 1, + "season": "autumn", + "sensor": "synthetic-multispectral", + "source": "fixture-thematic-raster", + "tile_edge": true, + "urbanity": "urban", + "vegetation": "moderate" + }, + "sample_id": "raster-test-brussels-urban", + "secondary_error_codes": [], + "task": "raster_classification" + }, + { + "contexts": [ + "high_confidence", + "tile_edge" + ], + "error_code": "M-FP-CONFUSER", + "evidence": { + "bbox": [ + 20, + 20, + 35, + 35 + ], + "class": "building", + "confidence": 0.92, + "id": "p-hard-fp" + }, + "failure_id": "39eae35c6660b1ac1bcd", + "kind": "false_positive", + "metadata": { + "context": "coastal_hard_negative", + "date": "2025-07-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Oostende", + "object_size": "empty", + "occlusion": "none", + "ood": false, + "region": "flanders", + "resolution_m": 0.25, + "season": "summer", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": true, + "urbanity": "urban", + "vegetation": "moderate" + }, + "sample_id": "background-test-hard-negative", + "secondary_error_codes": [ + "M-MISCALIBRATED" + ], + "task": "object_detection" + }, + { + "contexts": [], + "error_code": "D-VALIDATION-FN", + "evidence": { + "anomaly": "D-MISSING-PROVENANCE", + "severity": "critical" + }, + "failure_id": "40e33e21c399cc657f26", + "kind": "validation_false_negative", + "metadata": { + "context": "data_contract", + "date": "2025-01-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "not_applicable", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "national", + "resolution_m": null, + "season": "not_applicable", + "sensor": "mixed", + "source": "fixture-contracts", + "tile_edge": false, + "urbanity": "mixed", + "vegetation": "mixed" + }, + "sample_id": "validation-test-national", + "secondary_error_codes": [], + "task": "geospatial_data_validation" + }, + { + "contexts": [], + "error_code": "M-FP-CONFUSER", + "evidence": { + "bbox": [ + 20, + 0, + 25, + 5 + ], + "class": "removed", + "confidence": 0.65, + "id": "p2" + }, + "failure_id": "49560fb2dcb347c4cff5", + "kind": "event_false_positive", + "metadata": { + "context": "industrial_change", + "date": "2024-05-01/2025-05-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Liège", + "object_size": "mixed", + "occlusion": "none", + "ood": false, + "region": "wallonia", + "resolution_m": 0.5, + "season": "multi-date", + "sensor": "vector", + "source": "fixture-picc-temporal", + "tile_edge": false, + "urbanity": "industrial", + "vegetation": "low" + }, + "sample_id": "change-test-wallonia-industrial", + "secondary_error_codes": [], + "task": "change_detection" + }, + { + "contexts": [], + "error_code": "M-MISCALIBRATED", + "evidence": { + "brier": 0.15333333333333332, + "claim_boundary": "diagnostic deviation only; not a representative population-calibration claim", + "ece": 0.13333333333333325, + "prediction_support": 3 + }, + "failure_id": "8d4113e9688207b68f98", + "kind": "miscalibrated", + "metadata": { + "context": "dense_urban", + "date": "2025-06-15", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Antwerpen", + "object_size": "mixed", + "occlusion": "partial", + "ood": false, + "region": "flanders", + "resolution_m": 0.25, + "season": "summer", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": false, + "urbanity": "urban", + "vegetation": "moderate" + }, + "sample_id": "det-test-flanders-urban", + "secondary_error_codes": [], + "task": "object_detection" + }, + { + "contexts": [ + "tile_edge" + ], + "error_code": "M-CLASS", + "evidence": { + "column": 3, + "pixel_index": 3, + "prediction": 2, + "reason": "class_mismatch", + "reference": 1, + "row": 0 + }, + "failure_id": "a7c91430a7766b02755e", + "kind": "raster_misclassification", + "metadata": { + "context": "dense_urban_raster", + "date": "2025-10-05", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Brussel", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "brussels", + "resolution_m": 1, + "season": "autumn", + "sensor": "synthetic-multispectral", + "source": "fixture-thematic-raster", + "tile_edge": true, + "urbanity": "urban", + "vegetation": "moderate" + }, + "sample_id": "raster-test-brussels-urban", + "secondary_error_codes": [], + "task": "raster_classification" + }, + { + "contexts": [], + "error_code": "M-FP-CONFUSER", + "evidence": { + "bbox": [ + 85, + 85, + 95, + 95 + ], + "class": "building", + "confidence": 0.6, + "id": "p3" + }, + "failure_id": "b7e154a0f6ddb9d8653f", + "kind": "false_positive", + "metadata": { + "context": "dense_urban", + "date": "2025-06-15", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Antwerpen", + "object_size": "mixed", + "occlusion": "partial", + "ood": false, + "region": "flanders", + "resolution_m": 0.25, + "season": "summer", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": false, + "urbanity": "urban", + "vegetation": "moderate" + }, + "sample_id": "det-test-flanders-urban", + "secondary_error_codes": [], + "task": "object_detection" + }, + { + "contexts": [], + "error_code": "M-FP-CONFUSER", + "evidence": { + "class": "building", + "id": "p2", + "polygon": [ + [ + 20, + 0 + ], + [ + 24, + 0 + ], + [ + 24, + 4 + ], + [ + 20, + 4 + ], + [ + 20, + 0 + ] + ] + }, + "failure_id": "bf43633497be5101cddb", + "kind": "false_positive", + "metadata": { + "context": "suburban_buildings", + "date": "2025-01-15", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Mol", + "object_size": "mixed", + "occlusion": "not_applicable", + "ood": false, + "region": "flanders", + "resolution_m": 0.1, + "season": "not_applicable", + "sensor": "vector", + "source": "fixture-grb", + "tile_edge": false, + "urbanity": "suburban", + "vegetation": "low" + }, + "sample_id": "vector-test-flanders-suburban", + "secondary_error_codes": [], + "task": "vector_comparison" + }, + { + "contexts": [], + "error_code": "M-FN-MISSED", + "evidence": { + "class": "building", + "id": "r2", + "polygon": [ + [ + 10, + 0 + ], + [ + 14, + 0 + ], + [ + 14, + 4 + ], + [ + 10, + 4 + ], + [ + 10, + 0 + ] + ] + }, + "failure_id": "c9a80d5bb196dc7113e2", + "kind": "false_negative", + "metadata": { + "context": "suburban_buildings", + "date": "2025-01-15", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Mol", + "object_size": "mixed", + "occlusion": "not_applicable", + "ood": false, + "region": "flanders", + "resolution_m": 0.1, + "season": "not_applicable", + "sensor": "vector", + "source": "fixture-grb", + "tile_edge": false, + "urbanity": "suburban", + "vegetation": "low" + }, + "sample_id": "vector-test-flanders-suburban", + "secondary_error_codes": [], + "task": "vector_comparison" + }, + { + "contexts": [], + "error_code": "D-VALIDATION-FP", + "evidence": { + "anomaly": "D-RESOLUTION", + "severity": "major" + }, + "failure_id": "e00a10875bea399c62c3", + "kind": "validation_false_positive", + "metadata": { + "context": "data_contract", + "date": "2025-01-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "not_applicable", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "national", + "resolution_m": null, + "season": "not_applicable", + "sensor": "mixed", + "source": "fixture-contracts", + "tile_edge": false, + "urbanity": "mixed", + "vegetation": "mixed" + }, + "sample_id": "validation-test-national", + "secondary_error_codes": [], + "task": "geospatial_data_validation" + }, + { + "contexts": [ + "tile_edge" + ], + "error_code": "M-MISCALIBRATED", + "evidence": { + "brier": 0.8464, + "claim_boundary": "diagnostic deviation only; not a representative population-calibration claim", + "ece": 0.92, + "prediction_support": 1 + }, + "failure_id": "edae8689bdec601bca2b", + "kind": "miscalibrated", + "metadata": { + "context": "coastal_hard_negative", + "date": "2025-07-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Oostende", + "object_size": "empty", + "occlusion": "none", + "ood": false, + "region": "flanders", + "resolution_m": 0.25, + "season": "summer", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": true, + "urbanity": "urban", + "vegetation": "moderate" + }, + "sample_id": "background-test-hard-negative", + "secondary_error_codes": [], + "task": "object_detection" + } + ], + "items_canonical_json_sha256": "2002025055588195db73e20017882e412e988f3c55dedc89a1fd0c1dcd7a7e3e", + "rendering_status": "machine_readable_examples_retained; a visual production gallery requires controlled access to protected imagery", + "schema_version": 2, + "taxonomy": "docs/accuracy-program/05-metric-framework.md section 4" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/generation-status.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/generation-status.json new file mode 100644 index 00000000..335e0379 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/generation-status.json @@ -0,0 +1,8 @@ +{ + "development_manifest_sha256": "64e4fc986c2390bedfbe100062d4b06e1098689f4d4a9a9da13a7f67357c0493", + "generator_version": "1.3.0", + "protected_manifest_sha256": "e57ca1529654ef1c2527e84122ff26ec3533e7c70658250983968fbc5d4af410", + "schema_version": 1, + "source_manifest_sha256": "1c75c875d4c297d046117bb9393ba5e648f13d3358ea92cb65374869c8f5fc01", + "status": "pass" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/human-review-summary.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/human-review-summary.json new file mode 100644 index 00000000..5da4c94a --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/human-review-summary.json @@ -0,0 +1,8 @@ +{ + "ai_review_is_human_signoff": false, + "required": null, + "reviewed": null, + "schema_version": 2, + "source": "readiness-snapshot.json bound to Phase-1/3 evidence", + "status": "not_evaluable" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/input-manifest.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/input-manifest.json new file mode 100644 index 00000000..44a92a94 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/input-manifest.json @@ -0,0 +1,185 @@ +{ + "code": [ + { + "path": "scripts/accuracy_phase4_evaluator.py", + "sha256": "d2e40f7978d3ff4ce7025eaaaefb06ccc76c269d6874f8b0f528739d535a1da1", + "size_bytes": 93353 + }, + { + "path": "scripts/generate_accuracy_phase4_splits.py", + "sha256": "d12842922bcd99b1672176df15a9cef250128e7d7fc662ab15e16f1c084ba570", + "size_bytes": 75923 + }, + { + "path": "scripts/run_accuracy_phase4_benchmark.py", + "sha256": "003ecb4babc7f56a6d13e9e1c08dda902b9b26203cc7d53476d4d5e083b8b85b", + "size_bytes": 123717 + }, + { + "path": "scripts/run_golden_qa_benchmark.py", + "sha256": "2c8a19fd92db8f07f491b8f7b34d9736384f81e02a6b83019fac71b101ebdaa0", + "size_bytes": 10856 + }, + { + "path": "backend/app/services/qa_service.py", + "sha256": "2d4f645410fbf2972e55dddbf5a85db5368b2b1932792cd522ec51c483339244", + "size_bytes": 15206 + } + ], + "inputs": [ + { + "path": "fixtures/accuracy/p4/split-source-manifest.json", + "sha256": "27ec60406ac57a02f1335013ee344a43d4e0093e7ee1329ec22876da623f5bfc", + "size_bytes": 17331 + }, + { + "path": "fixtures/accuracy/p4/protected-baseline-cases.json", + "sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "size_bytes": 20091 + }, + { + "path": "fixtures/golden/golden_qa_benchmarks.json", + "sha256": "d60576ff7048ca3a048dc166e8977f3a31e9f8fa352be5ad222a017260e179e2", + "size_bytes": 2758 + }, + { + "path": "docs/accuracy-program/05-metric-framework.md", + "sha256": "18abfc21964b7d98848d8ae513afce242184980c9517c9184d40efd9165ea00d", + "size_bytes": 28053 + }, + { + "path": "docs/accuracy-program/07-source-authority-matrix.md", + "sha256": "42fc59ec012b3d4eff5674ba347b11fb418a42b0178a12c9de16eac873ef6a9c", + "size_bytes": 30807 + }, + { + "path": "artifacts/evidence/accuracy/P3/full-scan-manifest.json", + "sha256": "28c1022d125f30bfb4aa362b57d8a90cf8c88270b2a9ac6582b86b6a44a08038", + "size_bytes": 755527 + }, + { + "path": "artifacts/evidence/accuracy/P3/leakage-report.json", + "sha256": "8e1b7a24a840549d442bc03132ee1f21c6d50f8714cbb8d33430391a3769cd4e", + "size_bytes": 580 + } + ], + "model_execution": { + "configured_active_model": { + "model_id": "yolo-configured", + "model_version": "", + "nationally_validated": false, + "path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt", + "sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1", + "size_bytes": 22516074, + "validated_area_names": [ + "Mol", + "Kempen" + ] + }, + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "product_baseline": { + "artifacts": [], + "manifest": null, + "product_gate_evidence_sha256": "c9f5e3de3ab6d64911f8807b26caa661ccf911a5fd6b995107edda2004836e0c", + "requested_path": "artifacts/evidence/accuracy/P4/product-baseline-manifest.json", + "validation_status": "not_evaluable" + }, + "readiness_snapshot": { + "active_model": { + "model_id": "yolo-configured", + "model_version": "", + "nationally_validated": false, + "path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt", + "sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1", + "size_bytes": 22516074, + "validated_area_names": [ + "Mol", + "Kempen" + ] + }, + "authority_requirements": [ + { + "primary": "grb", + "task": "building_validation", + "zone": "flanders" + }, + { + "primary": "picc", + "task": "building_validation", + "zone": "wallonia" + }, + { + "primary": "urbis", + "task": "building_validation", + "zone": "brussels" + }, + { + "primary": "dhmv", + "task": "terrain_height", + "zone": "flanders" + }, + { + "primary": "spw_terrain", + "task": "terrain_height", + "zone": "wallonia" + }, + { + "primary": "mdk", + "task": "north_sea_bathymetry", + "zone": "belgian_north_sea" + }, + { + "contextual": "sentinel-2", + "primary": "official_orthophoto", + "task": "imagery_corroboration", + "zone": "belgium" + } + ], + "phase3_leakage_status": "attention", + "phase3_scan": { + "content_hash": null, + "grb_consistency": { + "reason": "No authoritative GRB snapshot was accessible in the configured local roots; no derived result was marked as GRB ground truth.", + "status": "unavailable" + }, + "scan_id": "p3-46ee3f8d3a2dc52b" + }, + "schema_version": 1, + "source_paths": { + "accuracy_status_projection": { + "path": "docs/accuracy-program/status.json", + "selected_json_pointers": [ + "/runtime/active_model" + ], + "sha256": "441dc0302156d8a1293fc330318bed5a39a6e6c558ee52dc9ada4965a5a516f6" + }, + "phase3_full_scan": { + "path": "artifacts/evidence/accuracy/P3/full-scan-manifest.json", + "sha256": "28c1022d125f30bfb4aa362b57d8a90cf8c88270b2a9ac6582b86b6a44a08038", + "size_bytes": 755527 + }, + "phase3_leakage": { + "path": "artifacts/evidence/accuracy/P3/leakage-report.json", + "sha256": "8e1b7a24a840549d442bc03132ee1f21c6d50f8714cbb8d33430391a3769cd4e", + "size_bytes": 580 + } + } + }, + "readiness_snapshot_sha256": "5fab098e77e8537d95d83adeda5ec081106fea760b4b476d3517079c33d1cd9f", + "repository_commit": "70fb4b94e5cb7c248beec5a936ce186f38cc183c", + "runtime": { + "cuda_used_for_reference_harness": false, + "dependencies": { + "numpy": "2.2.2", + "pyproj": "3.7.2", + "shapely": "2.1.2" + }, + "execution_device": "CPU deterministic evaluator arithmetic; no production model inference", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.13.2", + "python_implementation": "CPython" + }, + "schema_version": 2 +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/latency-and-reliability.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/latency-and-reliability.json new file mode 100644 index 00000000..1009ed87 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/latency-and-reliability.json @@ -0,0 +1,6 @@ +{ + "model_inference_executed": false, + "reason": "The reference harness performs deterministic evaluator arithmetic only. GPU latency, VRAM, throughput and failure-rate gates require the real active model.", + "schema_version": 2, + "status": "not_evaluable" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/leakage-gate-report.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/leakage-gate-report.json new file mode 100644 index 00000000..69be6695 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/leakage-gate-report.json @@ -0,0 +1,46 @@ +{ + "checked_identity_fields": [ + "sample_id", + "group_id", + "source_family", + "acquisition_date", + "temporal_family", + "object_ids", + "native_feature_ids", + "raw_image_sha256", + "processed_image_sha256", + "label_sha256", + "record_sha256", + "perceptual_image_hash", + "label_geometry_hash", + "label_geometry_fingerprint", + "parent_raster_id", + "acquisition_id", + "bbox_distance" + ], + "crs_validation": { + "crs": "EPSG:31370", + "distance_units": "m", + "status": "pass" + }, + "development_manifest_sha256": "64e4fc986c2390bedfbe100062d4b06e1098689f4d4a9a9da13a7f67357c0493", + "finding_count": 0, + "findings": [], + "generator_version": "1.3.0", + "independence_buffer_m": 2000.0, + "inventory_total": 21, + "label_geometry_hamming_threshold": 2, + "perceptual_hamming_threshold": 4, + "protected_manifest_sha256": "e57ca1529654ef1c2527e84122ff26ec3533e7c70658250983968fbc5d4af410", + "schema_version": 1, + "source_manifest_sha256": "1c75c875d4c297d046117bb9393ba5e648f13d3358ea92cb65374869c8f5fc01", + "split_counts": { + "background-test": 2, + "calibration": 2, + "challenge": 4, + "test": 7, + "train": 3, + "val": 3 + }, + "status": "pass" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/metric-report.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/metric-report.json new file mode 100644 index 00000000..ccb3da3e --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/metric-report.json @@ -0,0 +1,10635 @@ +{ + "case_count": 9, + "claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.", + "evaluated_task_families": [ + "change_detection", + "footprint_segmentation", + "geospatial_data_validation", + "object_detection", + "raster_classification", + "terrain_interpretation", + "vector_comparison" + ], + "evaluator_version": "2.1.0", + "failure_taxonomy": { + "claim_boundary": "diagnostic classification; not a release decision", + "contexts": { + "high_confidence": "false positive meets the highest frozen diagnostic confidence threshold", + "out_of_distribution": "case metadata explicitly marks OOD context", + "tile_edge": "case metadata explicitly marks tile-edge context" + }, + "primary_codes": { + "D-VALIDATION-FN": "reference anomaly or severity not exactly reported", + "D-VALIDATION-FP": "anomaly or severity reported without an exact reference match", + "M-AREA-BIAS": "matched footprint has a measurable relative area bias", + "M-BOUNDARY": "matched footprint has a measurable boundary deviation", + "M-CLASS": "raster class differs from the valid reference class", + "M-FN-MISSED": "unmatched reference object or event", + "M-FP-CONFUSER": "unmatched object or event prediction", + "M-MISCALIBRATED": "confidence diagnostic deviates from observed correctness", + "M-OOD": "error observed in an explicitly declared out-of-distribution case", + "P-PARTIAL": "required prediction value is unavailable" + } + }, + "full_results_canonical_json_sha256": "55cdba9cce4dcdc88cee0829cd510d7e624a3472feb9d33f7e681621a792eba3", + "hash_specification": { + "algorithm": "SHA-256", + "canonical_json": "UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), ensure_ascii=false and allow_nan=false", + "portfolio_canonical_json_sha256_input": "parsed complete portfolio", + "portfolio_file_sha256_input": "exact source-file bytes", + "results_canonical_json_sha256_input": "complete sample-id-ordered results array" + }, + "metric_results_canonical_json_sha256": "cdaf2460bf51231d8f547402ab9c464a72fa9226aa69380e313609f9fb2e3b2e", + "portfolio_canonical_json_sha256": "1ac1bd77db125db48f9168406d51bfb723522f54a6c81dbe868d600c72f27b17", + "portfolio_file_sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473", + "portfolio_id": "geointel-p4-reference-harness-v2", + "portfolio_kind": "synthetic_contract", + "portfolio_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "object_detection": { + "case_support": 3, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "protected_policy": { + "diagnostic_curves_select_operating_point": false, + "operating_point_selection_allowed": false, + "test_feedback_allowed": false, + "threshold_selection_source": "pre_registered_configuration_only" + }, + "results": [ + { + "metadata": { + "context": "coastal_hard_negative", + "date": "2025-07-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Oostende", + "object_size": "empty", + "occlusion": "none", + "ood": false, + "region": "flanders", + "resolution_m": 0.25, + "season": "summer", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": true, + "urbanity": "urban", + "vegetation": "moderate" + }, + "metrics": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "coverage_risk": [ + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.0, + "total_prediction_count": 1 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.5, + "total_prediction_count": 1 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.7, + "total_prediction_count": 1 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.9, + "total_prediction_count": 1 + } + ], + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "match_iou": 0.5, + "matched_iou": { + "count": 0, + "max": null, + "mean": null, + "median": null, + "min": null + }, + "miss_rate": null, + "operating_confidence": 0.5, + "per_class": { + "building": { + "ap50": null, + "ap50_95": null, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "miss_rate": null, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "sample_id": "background-test-hard-negative", + "task": "object_detection" + }, + { + "metadata": { + "context": "pure_background", + "date": "2025-02-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Brussel", + "object_size": "empty", + "occlusion": "none", + "ood": false, + "region": "brussels", + "resolution_m": 0.25, + "season": "winter", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": false, + "urbanity": "urban", + "vegetation": "low" + }, + "metrics": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "coverage_risk": [ + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.0, + "total_prediction_count": 0 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.5, + "total_prediction_count": 0 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.7, + "total_prediction_count": 0 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.9, + "total_prediction_count": 0 + } + ], + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "match_iou": 0.5, + "matched_iou": { + "count": 0, + "max": null, + "mean": null, + "median": null, + "min": null + }, + "miss_rate": null, + "operating_confidence": 0.5, + "per_class": { + "building": { + "ap50": null, + "ap50_95": null, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "miss_rate": null, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "sample_id": "background-test-pure-empty", + "task": "object_detection" + }, + { + "metadata": { + "context": "industrial_change", + "date": "2024-05-01/2025-05-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Liège", + "object_size": "mixed", + "occlusion": "none", + "ood": false, + "region": "wallonia", + "resolution_m": 0.5, + "season": "multi-date", + "sensor": "vector", + "source": "fixture-picc-temporal", + "tile_edge": false, + "urbanity": "industrial", + "vegetation": "low" + }, + "metrics": { + "event_metrics": { + "added": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "removed": { + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 1.0, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "recall": 0.0, + "recall_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "reference_count": 1, + "true_positive": 0 + } + }, + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "sample_id": "change-test-wallonia-industrial", + "task": "change_detection" + }, + { + "metadata": { + "context": "dense_urban", + "date": "2025-06-15", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Antwerpen", + "object_size": "mixed", + "occlusion": "partial", + "ood": false, + "region": "flanders", + "resolution_m": 0.25, + "season": "summer", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": false, + "urbanity": "urban", + "vegetation": "moderate" + }, + "metrics": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "coverage_risk": [ + { + "false_negative_count": 0, + "false_positive_count": 2, + "matched_reference_count": 2, + "reference_count": 2, + "reference_coverage": 1.0, + "retained_prediction_count": 4, + "retained_prediction_coverage": 1.0, + "risk": 0.5, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.0, + "total_prediction_count": 4 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 2, + "reference_count": 2, + "reference_coverage": 1.0, + "retained_prediction_count": 3, + "retained_prediction_coverage": 0.75, + "risk": 0.3333333333333333, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.5, + "total_prediction_count": 4 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 2, + "reference_count": 2, + "reference_coverage": 1.0, + "retained_prediction_count": 2, + "retained_prediction_coverage": 0.5, + "risk": 0.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.7, + "total_prediction_count": 4 + }, + { + "false_negative_count": 1, + "false_positive_count": 0, + "matched_reference_count": 1, + "reference_count": 2, + "reference_coverage": 0.5, + "retained_prediction_count": 1, + "retained_prediction_coverage": 0.25, + "risk": 0.5, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.9, + "total_prediction_count": 4 + } + ], + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "match_iou": 0.5, + "matched_iou": { + "count": 2, + "max": 1.0, + "mean": 0.9355555555555555, + "median": 0.9355555555555555, + "min": 0.8711111111111111 + }, + "miss_rate": 0.0, + "operating_confidence": 0.5, + "per_class": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "miss_rate": 0.0, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "sample_id": "det-test-flanders-urban", + "task": "object_detection" + }, + { + "metadata": { + "context": "dense_urban_raster", + "date": "2025-10-05", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Brussel", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "brussels", + "resolution_m": 1, + "season": "autumn", + "sensor": "synthetic-multispectral", + "source": "fixture-thematic-raster", + "tile_edge": true, + "urbanity": "urban", + "vegetation": "moderate" + }, + "metrics": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "alignment_validated": true, + "confusion_matrix": { + "0": { + "0": 4, + "1": 0, + "2": 0, + "__prediction_nodata__": 0 + }, + "1": { + "0": 0, + "1": 4, + "2": 1, + "__prediction_nodata__": 0 + }, + "2": { + "0": 0, + "1": 1, + "2": 2, + "__prediction_nodata__": 0 + } + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "ignored_reference_pixel_count": 0, + "macro_f1": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "pixel_count": 12, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_outside_reference_count": 0, + "prediction_present_pixel_count": 12 + }, + "sample_id": "raster-test-brussels-urban", + "task": "raster_classification" + }, + { + "metadata": { + "context": "rural_buildings", + "date": "2025-04-20", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Namur", + "object_size": "large", + "occlusion": "none", + "ood": false, + "region": "wallonia", + "resolution_m": 0.5, + "season": "spring", + "sensor": "synthetic-rgb", + "source": "fixture-picc", + "tile_edge": false, + "urbanity": "rural", + "vegetation": "low" + }, + "metrics": { + "centroid_distance_m": { + "count": 1, + "max": 0.5, + "mean": 0.5, + "median": 0.5, + "min": 0.5 + }, + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "mean_boundary_f1": 1.0, + "mean_dice": 0.8881578947368421, + "mean_iou": 0.7988165680473372, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "relative_area_error": { + "count": 1, + "max": -0.1, + "mean": -0.1, + "median": -0.1, + "min": -0.1 + }, + "spatial_context_validated": true, + "topologically_invalid_predictions": 0, + "topologically_valid_predictions": 1, + "true_positive": 1 + }, + "sample_id": "seg-test-wallonia-rural", + "task": "footprint_segmentation" + }, + { + "metadata": { + "context": "terrain_profile", + "date": "2023-01-01", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Hasselt", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "flanders", + "resolution_m": 5, + "season": "not_applicable", + "sensor": "elevation-raster", + "source": "fixture-dhmv", + "tile_edge": false, + "urbanity": "rural", + "vegetation": "moderate" + }, + "metrics": { + "absolute_error_sum": 1.0999999999999996, + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "error_distribution": { + "count": 4, + "max": 0.40000000000000036, + "mean": 0.02499999999999991, + "median": 0.0, + "min": -0.3000000000000007 + }, + "error_sum": 0.09999999999999964, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "missing_count": 1, + "reference_count": 5, + "rmse": 0.2872281323269015, + "squared_error_sum": 0.3300000000000001, + "units": "m_TAW" + }, + "sample_id": "terrain-test-flanders-rural", + "task": "terrain_interpretation" + }, + { + "metadata": { + "context": "data_contract", + "date": "2025-01-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "not_applicable", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "national", + "resolution_m": null, + "season": "not_applicable", + "sensor": "mixed", + "source": "fixture-contracts", + "tile_edge": false, + "urbanity": "mixed", + "vegetation": "mixed" + }, + "metrics": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "misses_by_severity": { + "blocker": 0, + "critical": 1, + "informational": 0, + "major": 0, + "minor": 0 + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "severity_mismatch_count": 0, + "severity_mismatches": [], + "true_positive": 2 + }, + "sample_id": "validation-test-national", + "task": "geospatial_data_validation" + }, + { + "metadata": { + "context": "suburban_buildings", + "date": "2025-01-15", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Mol", + "object_size": "mixed", + "occlusion": "not_applicable", + "ood": false, + "region": "flanders", + "resolution_m": 0.1, + "season": "not_applicable", + "sensor": "vector", + "source": "fixture-grb", + "tile_edge": false, + "urbanity": "suburban", + "vegetation": "low" + }, + "metrics": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "mean_iou": 1.0, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "spatial_context_validated": true, + "topologically_valid": true, + "true_positive": 1 + }, + "sample_id": "vector-test-flanders-suburban", + "task": "vector_comparison" + } + ], + "results_canonical_json_sha256": "55cdba9cce4dcdc88cee0829cd510d7e624a3472feb9d33f7e681621a792eba3", + "schema_version": 2, + "selection_policy": "No data-dependent threshold, model or gate selection is performed from protected cases.", + "split_roles": [ + "test", + "background-test" + ], + "subgroups": { + "dimensions": { + "context": { + "strata": { + "coastal_hard_negative": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": 0.0, + "precision_case_support": 1, + "recall": null, + "recall_case_support": 0, + "status": "computed" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 1, + "reference_count": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 1, + "predictions": 1, + "ranking_predictions": 1, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "data_contract": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "dense_urban": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "prediction_count": 4, + "reference_count": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 3, + "predictions": 3, + "ranking_predictions": 4, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.8 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "dense_urban_raster": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + }, + "industrial_change": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "pure_background": { + "case_support": 1, + "failure_count": 0, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "rural_buildings": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "suburban_buildings": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "terrain_profile": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "date": { + "strata": { + "2023-01-01": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + }, + "2024-05-01/2025-05-01": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "2025-01-01": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "2025-01-15": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "2025-02-01": { + "case_support": 1, + "failure_count": 0, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "2025-04-20": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "2025-06-15": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "prediction_count": 4, + "reference_count": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 3, + "predictions": 3, + "ranking_predictions": 4, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.8 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "2025-07-01": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": 0.0, + "precision_case_support": 1, + "recall": null, + "recall_case_support": 0, + "status": "computed" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 1, + "reference_count": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 1, + "predictions": 1, + "ranking_predictions": 1, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "2025-10-05": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "difficulty": { + "strata": { + "hard": { + "case_support": 6, + "failure_count": 10, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "object_detection": { + "case_support": 3, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "geospatial_data_validation", + "object_detection", + "raster_classification" + ] + }, + "normal": { + "case_support": 3, + "failure_count": 4, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation", + "terrain_interpretation", + "vector_comparison" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "municipality": { + "strata": { + "Antwerpen": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "prediction_count": 4, + "reference_count": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 3, + "predictions": 3, + "ranking_predictions": 4, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.8 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "Brussel": { + "case_support": 2, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection", + "raster_classification" + ] + }, + "Hasselt": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + }, + "Liège": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "Mol": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "Namur": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "Oostende": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": 0.0, + "precision_case_support": 1, + "recall": null, + "recall_case_support": 0, + "status": "computed" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 1, + "reference_count": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 1, + "predictions": 1, + "ranking_predictions": 1, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "not_applicable": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "object_size": { + "strata": { + "empty": { + "case_support": 2, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 2, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": 0.0, + "precision_case_support": 1, + "recall": null, + "recall_case_support": 0, + "status": "computed" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 1, + "reference_count": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 1, + "predictions": 1, + "ranking_predictions": 1, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "large": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "mixed": { + "case_support": 3, + "failure_count": 6, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "object_detection": { + "case_support": 1, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "prediction_count": 4, + "reference_count": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 3, + "predictions": 3, + "ranking_predictions": 4, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.8 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "object_detection", + "vector_comparison" + ] + }, + "not_applicable": { + "case_support": 3, + "failure_count": 5, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation", + "raster_classification", + "terrain_interpretation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "occlusion": { + "strata": { + "none": { + "case_support": 4, + "failure_count": 5, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "object_detection": { + "case_support": 2, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": 0.0, + "precision_case_support": 1, + "recall": null, + "recall_case_support": 0, + "status": "computed" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 1, + "reference_count": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 1, + "predictions": 1, + "ranking_predictions": 1, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "footprint_segmentation", + "object_detection" + ] + }, + "not_applicable": { + "case_support": 4, + "failure_count": 7, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation", + "raster_classification", + "terrain_interpretation", + "vector_comparison" + ] + }, + "partial": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "prediction_count": 4, + "reference_count": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 3, + "predictions": 3, + "ranking_predictions": 4, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.8 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "region": { + "strata": { + "brussels": { + "case_support": 2, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection", + "raster_classification" + ] + }, + "flanders": { + "case_support": 4, + "failure_count": 7, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 2, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection", + "terrain_interpretation", + "vector_comparison" + ] + }, + "national": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "wallonia": { + "case_support": 2, + "failure_count": 3, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "footprint_segmentation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "resolution_m": { + "strata": { + "0.1": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "0.25": { + "case_support": 3, + "failure_count": 4, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 3, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "0.5": { + "case_support": 2, + "failure_count": 3, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "footprint_segmentation" + ] + }, + "1": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + }, + "5": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + }, + "__not_applicable__": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "season": { + "strata": { + "autumn": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + }, + "multi-date": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "not_applicable": { + "case_support": 3, + "failure_count": 5, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation", + "terrain_interpretation", + "vector_comparison" + ] + }, + "spring": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "summer": { + "case_support": 2, + "failure_count": 4, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 2, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "winter": { + "case_support": 1, + "failure_count": 0, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "sensor": { + "strata": { + "elevation-raster": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + }, + "mixed": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "synthetic-multispectral": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + }, + "synthetic-rgb": { + "case_support": 4, + "failure_count": 5, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "object_detection": { + "case_support": 3, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation", + "object_detection" + ] + }, + "vector": { + "case_support": 2, + "failure_count": 4, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "vector_comparison" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "source": { + "strata": { + "fixture-contracts": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "fixture-dhmv": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + }, + "fixture-grb": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "fixture-orthophoto": { + "case_support": 3, + "failure_count": 4, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 3, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "fixture-picc": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "fixture-picc-temporal": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "fixture-thematic-raster": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "urbanity": { + "strata": { + "industrial": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "mixed": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "rural": { + "case_support": 2, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation", + "terrain_interpretation" + ] + }, + "suburban": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "urban": { + "case_support": 4, + "failure_count": 6, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 3, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection", + "raster_classification" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "vegetation": { + "strata": { + "low": { + "case_support": 4, + "failure_count": 5, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "footprint_segmentation", + "object_detection", + "vector_comparison" + ] + }, + "mixed": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "moderate": { + "case_support": 4, + "failure_count": 7, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 2, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection", + "raster_classification", + "terrain_interpretation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + } + }, + "minimum_case_support": 5, + "overall_status": "not_evaluable", + "schema_version": 2, + "support_unit": "independent benchmark cases; pixel/object counts are reported separately and do not replace case support" + }, + "task_count": 7, + "task_inventory": [ + { + "capability_id": "model_object_detection", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "model_inference_pipeline", + "implementation_paths": [ + "backend/app/services/detection_service.py" + ], + "suitable_metrics": [ + "precision", + "recall", + "F1", + "AP50", + "AP50-95", + "IoU", + "ECE", + "Brier", + "coverage-risk" + ], + "task": "object_detection" + }, + { + "capability_id": "building_proposal_filtering", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "not_separately_benchmarked", + "implementation_kind": "supporting_candidate_classifier", + "implementation_paths": [ + "scripts/train_building_proposal_classifier.py", + "scripts/evaluate_belgium_building_candidate.py" + ], + "suitable_metrics": [ + "candidate precision", + "candidate recall", + "F1", + "calibration" + ], + "task": "object_detection" + }, + { + "capability_id": "footprint_segmentation", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "model_inference_pipeline", + "implementation_paths": [ + "backend/app/services/segmentation_service.py", + "backend/app/services/segmentation_adapter.py" + ], + "suitable_metrics": [ + "object precision", + "object recall", + "F1", + "IoU", + "Dice", + "boundary F1", + "centroid distance", + "area error", + "topology" + ], + "task": "footprint_segmentation" + }, + { + "capability_id": "detection_and_vector_qa", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "deterministic_geospatial_comparison", + "implementation_paths": [ + "backend/app/services/qa_service.py", + "backend/app/services/detection_qa_service.py", + "backend/app/services/quality_check_service.py" + ], + "suitable_metrics": [ + "precision", + "recall", + "F1", + "IoU", + "topology", + "coverage" + ], + "task": "vector_comparison" + }, + { + "capability_id": "vector_clip_buffer_intersect", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "covered_by_family_not_separately_benchmarked", + "implementation_kind": "deterministic_vector_processing", + "implementation_paths": [ + "backend/app/services/vector_operations_service.py", + "backend/app/services/vector_feature_service.py" + ], + "suitable_metrics": [ + "geometry validity", + "CRS correctness", + "area conservation", + "feature counts", + "topology" + ], + "task": "vector_comparison" + }, + { + "capability_id": "temporal_vector_change", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "deterministic_change_detection", + "implementation_paths": [ + "backend/app/services/change_detection_service.py" + ], + "suitable_metrics": [ + "event precision", + "event recall", + "event F1", + "IoU" + ], + "task": "change_detection" + }, + { + "capability_id": "thematic_raster_interpretation", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_metric_contract_only_no_generic_learned_classifier_claim", + "implementation_kind": "deterministic_source_interpretation", + "implementation_paths": [ + "backend/app/services/thematic_raster_analysis_service.py" + ], + "suitable_metrics": [ + "pixel accuracy", + "per-class F1", + "per-class IoU", + "mean IoU" + ], + "task": "raster_classification" + }, + { + "capability_id": "raster_clip_reproject_indices", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "covered_by_family_not_separately_benchmarked", + "implementation_kind": "deterministic_raster_processing", + "implementation_paths": [ + "backend/app/services/raster_service.py", + "backend/app/services/raster_operations_service.py" + ], + "suitable_metrics": [ + "CRS/transform preservation", + "pixel alignment", + "nodata", + "numeric tolerance" + ], + "task": "raster_classification" + }, + { + "capability_id": "raster_partition_mosaic", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "not_separately_benchmarked", + "implementation_kind": "deterministic_raster_partitioning", + "implementation_paths": [ + "backend/app/services/raster_partition_analysis_service.py" + ], + "suitable_metrics": [ + "seam equality", + "coverage completeness", + "resolution consistency" + ], + "task": "raster_classification" + }, + { + "capability_id": "terrain_height_interpretation", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "deterministic_continuous_raster_analysis", + "implementation_paths": [ + "backend/app/services/terrain_analysis_service.py", + "backend/app/services/spw_terrain_service.py" + ], + "suitable_metrics": [ + "MAE", + "RMSE", + "bias", + "coverage", + "unit integrity" + ], + "task": "terrain_interpretation" + }, + { + "capability_id": "flood_hazard_interpretation", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "covered_by_family_not_separately_benchmarked", + "implementation_kind": "deterministic_scenario_raster_analysis", + "implementation_paths": [ + "backend/app/services/flood_hazard_analysis_service.py" + ], + "suitable_metrics": [ + "depth MAE/RMSE", + "hazard-class IoU", + "coverage", + "scenario identity" + ], + "task": "terrain_interpretation" + }, + { + "capability_id": "bathymetry_interpretation", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "covered_by_family_not_separately_benchmarked", + "implementation_kind": "deterministic_vertical_reference_analysis", + "implementation_paths": [ + "backend/app/services/bathymetry_raster_analysis_service.py", + "backend/app/services/mdk_bathymetry_probe_service.py" + ], + "suitable_metrics": [ + "MAE", + "RMSE", + "bias", + "coverage", + "vertical-datum integrity" + ], + "task": "terrain_interpretation" + }, + { + "capability_id": "data_contract_validation_and_scan", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "synthetic_contract_case_only", + "implementation_kind": "deterministic_validation", + "implementation_paths": [ + "backend/app/services/data_contract_validation.py", + "scripts/run_accuracy_phase3_full_data_scan.py" + ], + "suitable_metrics": [ + "anomaly precision", + "anomaly recall", + "anomaly F1", + "critical misses" + ], + "task": "geospatial_data_validation" + }, + { + "capability_id": "aoi_partition_orchestration", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "not_separately_benchmarked", + "implementation_kind": "deterministic_orchestration", + "implementation_paths": [ + "backend/app/services/aoi_operation_service.py", + "backend/app/services/aoi_operation_executor.py", + "backend/app/services/aoi_operation_worker.py" + ], + "suitable_metrics": [ + "partition completeness", + "overlap/gap", + "idempotency", + "resume correctness" + ], + "task": "geospatial_data_validation" + }, + { + "capability_id": "geo_assistant_orchestration", + "claim_boundary": "A family mapping is not evidence that this capability has a separate representative product benchmark.", + "evaluation_status": "no_independent_accuracy_score_underlying_tool_results_are_authoritative", + "implementation_kind": "tool_orchestration_interface", + "implementation_paths": [ + "backend/app/services/geo_assistant_service.py" + ], + "suitable_metrics": [ + "tool-selection correctness", + "grounding", + "unsupported-claim rate" + ], + "task": "geospatial_data_validation" + } + ] +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/object-metrics.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/object-metrics.json new file mode 100644 index 00000000..ef967c77 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/object-metrics.json @@ -0,0 +1,806 @@ +{ + "results": [ + { + "metadata": { + "context": "coastal_hard_negative", + "date": "2025-07-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Oostende", + "object_size": "empty", + "occlusion": "none", + "ood": false, + "region": "flanders", + "resolution_m": 0.25, + "season": "summer", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": true, + "urbanity": "urban", + "vegetation": "moderate" + }, + "metrics": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "coverage_risk": [ + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.0, + "total_prediction_count": 1 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.5, + "total_prediction_count": 1 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.7, + "total_prediction_count": 1 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 1, + "retained_prediction_coverage": 1.0, + "risk": 1.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.9, + "total_prediction_count": 1 + } + ], + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "match_iou": 0.5, + "matched_iou": { + "count": 0, + "max": null, + "mean": null, + "median": null, + "min": null + }, + "miss_rate": null, + "operating_confidence": 0.5, + "per_class": { + "building": { + "ap50": null, + "ap50_95": null, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "miss_rate": null, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "sample_id": "background-test-hard-negative", + "task": "object_detection" + }, + { + "metadata": { + "context": "pure_background", + "date": "2025-02-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Brussel", + "object_size": "empty", + "occlusion": "none", + "ood": false, + "region": "brussels", + "resolution_m": 0.25, + "season": "winter", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": false, + "urbanity": "urban", + "vegetation": "low" + }, + "metrics": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "coverage_risk": [ + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.0, + "total_prediction_count": 0 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.5, + "total_prediction_count": 0 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.7, + "total_prediction_count": 0 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 0, + "reference_count": 0, + "reference_coverage": null, + "retained_prediction_count": 0, + "retained_prediction_coverage": null, + "risk": null, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.9, + "total_prediction_count": 0 + } + ], + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "match_iou": 0.5, + "matched_iou": { + "count": 0, + "max": null, + "mean": null, + "median": null, + "min": null + }, + "miss_rate": null, + "operating_confidence": 0.5, + "per_class": { + "building": { + "ap50": null, + "ap50_95": null, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "miss_rate": null, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "sample_id": "background-test-pure-empty", + "task": "object_detection" + }, + { + "metadata": { + "context": "industrial_change", + "date": "2024-05-01/2025-05-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Liège", + "object_size": "mixed", + "occlusion": "none", + "ood": false, + "region": "wallonia", + "resolution_m": 0.5, + "season": "multi-date", + "sensor": "vector", + "source": "fixture-picc-temporal", + "tile_edge": false, + "urbanity": "industrial", + "vegetation": "low" + }, + "metrics": { + "event_metrics": { + "added": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "removed": { + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 1.0, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "recall": 0.0, + "recall_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "reference_count": 1, + "true_positive": 0 + } + }, + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "sample_id": "change-test-wallonia-industrial", + "task": "change_detection" + }, + { + "metadata": { + "context": "dense_urban", + "date": "2025-06-15", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Antwerpen", + "object_size": "mixed", + "occlusion": "partial", + "ood": false, + "region": "flanders", + "resolution_m": 0.25, + "season": "summer", + "sensor": "synthetic-rgb", + "source": "fixture-orthophoto", + "tile_edge": false, + "urbanity": "urban", + "vegetation": "moderate" + }, + "metrics": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "coverage_risk": [ + { + "false_negative_count": 0, + "false_positive_count": 2, + "matched_reference_count": 2, + "reference_count": 2, + "reference_coverage": 1.0, + "retained_prediction_count": 4, + "retained_prediction_coverage": 1.0, + "risk": 0.5, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.0, + "total_prediction_count": 4 + }, + { + "false_negative_count": 0, + "false_positive_count": 1, + "matched_reference_count": 2, + "reference_count": 2, + "reference_coverage": 1.0, + "retained_prediction_count": 3, + "retained_prediction_coverage": 0.75, + "risk": 0.3333333333333333, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.5, + "total_prediction_count": 4 + }, + { + "false_negative_count": 0, + "false_positive_count": 0, + "matched_reference_count": 2, + "reference_count": 2, + "reference_coverage": 1.0, + "retained_prediction_count": 2, + "retained_prediction_coverage": 0.5, + "risk": 0.0, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.7, + "total_prediction_count": 4 + }, + { + "false_negative_count": 1, + "false_positive_count": 0, + "matched_reference_count": 1, + "reference_count": 2, + "reference_coverage": 0.5, + "retained_prediction_count": 1, + "retained_prediction_coverage": 0.25, + "risk": 0.5, + "risk_definition": "(false_positive + false_negative) / (true_positive + false_positive + false_negative)", + "threshold": 0.9, + "total_prediction_count": 4 + } + ], + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "match_iou": 0.5, + "matched_iou": { + "count": 2, + "max": 1.0, + "mean": 0.9355555555555555, + "median": 0.9355555555555555, + "min": 0.8711111111111111 + }, + "miss_rate": 0.0, + "operating_confidence": 0.5, + "per_class": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "miss_rate": 0.0, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "sample_id": "det-test-flanders-urban", + "task": "object_detection" + }, + { + "metadata": { + "context": "rural_buildings", + "date": "2025-04-20", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Namur", + "object_size": "large", + "occlusion": "none", + "ood": false, + "region": "wallonia", + "resolution_m": 0.5, + "season": "spring", + "sensor": "synthetic-rgb", + "source": "fixture-picc", + "tile_edge": false, + "urbanity": "rural", + "vegetation": "low" + }, + "metrics": { + "centroid_distance_m": { + "count": 1, + "max": 0.5, + "mean": 0.5, + "median": 0.5, + "min": 0.5 + }, + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "mean_boundary_f1": 1.0, + "mean_dice": 0.8881578947368421, + "mean_iou": 0.7988165680473372, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "relative_area_error": { + "count": 1, + "max": -0.1, + "mean": -0.1, + "median": -0.1, + "min": -0.1 + }, + "spatial_context_validated": true, + "topologically_invalid_predictions": 0, + "topologically_valid_predictions": 1, + "true_positive": 1 + }, + "sample_id": "seg-test-wallonia-rural", + "task": "footprint_segmentation" + }, + { + "metadata": { + "context": "data_contract", + "date": "2025-01-01", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "not_applicable", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "national", + "resolution_m": null, + "season": "not_applicable", + "sensor": "mixed", + "source": "fixture-contracts", + "tile_edge": false, + "urbanity": "mixed", + "vegetation": "mixed" + }, + "metrics": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "misses_by_severity": { + "blocker": 0, + "critical": 1, + "informational": 0, + "major": 0, + "minor": 0 + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "severity_mismatch_count": 0, + "severity_mismatches": [], + "true_positive": 2 + }, + "sample_id": "validation-test-national", + "task": "geospatial_data_validation" + }, + { + "metadata": { + "context": "suburban_buildings", + "date": "2025-01-15", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Mol", + "object_size": "mixed", + "occlusion": "not_applicable", + "ood": false, + "region": "flanders", + "resolution_m": 0.1, + "season": "not_applicable", + "sensor": "vector", + "source": "fixture-grb", + "tile_edge": false, + "urbanity": "suburban", + "vegetation": "low" + }, + "metrics": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "mean_iou": 1.0, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "spatial_context_validated": true, + "topologically_valid": true, + "true_positive": 1 + }, + "sample_id": "vector-test-flanders-suburban", + "task": "vector_comparison" + } + ], + "schema_version": 2, + "status": "fixture_contract_only" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/protected-split-manifest.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/protected-split-manifest.json new file mode 100644 index 00000000..258b2459 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/protected-split-manifest.json @@ -0,0 +1,589 @@ +{ + "access_policy": "Only the frozen release evaluator may consume test/background-test labels; challenge labels remain external/sealed.", + "allowed_splits": [ + "background-test", + "challenge", + "test" + ], + "assignment_algorithm": "group-before-split-deficit-balancer-v1", + "assignment_mode": "preassigned", + "assignment_seed": "geointel-p4-group-split-v1", + "claim_boundary": "Synthetic contract fixtures for evaluator regression only; never production accuracy evidence.", + "crs": "EPSG:31370", + "dataset_version": "geointel-p4-harness-fixture-v2", + "generator_version": "1.3.0", + "implemented_tasks": [ + "change_detection", + "footprint_segmentation", + "geospatial_data_validation", + "object_detection", + "raster_classification", + "terrain_interpretation", + "vector_comparison" + ], + "independence_buffer_m": 2000.0, + "label_geometry_hamming_threshold": 2, + "labels_available_by_split": { + "background-test": "frozen_evaluator_only", + "challenge": "sealed_external", + "test": "frozen_evaluator_only" + }, + "mandatory_split_roles": [ + "train", + "val", + "calibration", + "test", + "background-test", + "challenge" + ], + "manifest_role": "protected_release_only", + "manifest_sha256": "e57ca1529654ef1c2527e84122ff26ec3533e7c70658250983968fbc5d4af410", + "perceptual_hamming_threshold": 4, + "protected_task_coverage": [ + { + "background_test_count": 0, + "exemption_reason": null, + "sealed_challenge_count": 0, + "status": "covered", + "task": "change_detection", + "test_count": 1 + }, + { + "background_test_count": 0, + "exemption_reason": null, + "sealed_challenge_count": 1, + "status": "covered", + "task": "footprint_segmentation", + "test_count": 1 + }, + { + "background_test_count": 0, + "exemption_reason": null, + "sealed_challenge_count": 1, + "status": "covered", + "task": "geospatial_data_validation", + "test_count": 1 + }, + { + "background_test_count": 2, + "exemption_reason": null, + "sealed_challenge_count": 1, + "status": "covered", + "task": "object_detection", + "test_count": 1 + }, + { + "background_test_count": 0, + "exemption_reason": null, + "sealed_challenge_count": 1, + "status": "covered", + "task": "raster_classification", + "test_count": 1 + }, + { + "background_test_count": 0, + "exemption_reason": null, + "sealed_challenge_count": 0, + "status": "covered", + "task": "terrain_interpretation", + "test_count": 1 + }, + { + "background_test_count": 0, + "exemption_reason": null, + "sealed_challenge_count": 0, + "status": "covered", + "task": "vector_comparison", + "test_count": 1 + } + ], + "samples": [ + { + "acquisition_date": "2025-01-17", + "acquisition_id": "acquisition-17", + "bbox": [ + 60000.0, + 70000.0, + 60100.0, + 70100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g17", + "label_geometry_fingerprint": "2b405bcc9c95acfb", + "label_geometry_hash": "7171717171717171717171717171717171717171717171717171717171717171", + "label_sha256": "5151515151515151515151515151515151515151515151515151515151515151", + "native_feature_ids": [], + "object_ids": [], + "parent_raster_id": "parent-raster-17", + "perceptual_image_hash": "6ada1b5aa46cc261", + "processed_image_sha256": "3131313131313131313131313131313131313131313131313131313131313131", + "raw_image_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "record_sha256": "330f67df045e915670260db85cd399b7abb0a5277ca857e2b9483274dfd561c9", + "sample_id": "background-test-hard-negative", + "source_family": "scene-17", + "split": "background-test", + "task": "object_detection", + "temporal_family": "temporal-17" + }, + { + "acquisition_date": "2025-01-16", + "acquisition_id": "acquisition-16", + "bbox": [ + 20000.0, + 70000.0, + 20100.0, + 70100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g16", + "label_geometry_fingerprint": "004e762f898c4fad", + "label_geometry_hash": "7070707070707070707070707070707070707070707070707070707070707070", + "label_sha256": "5050505050505050505050505050505050505050505050505050505050505050", + "native_feature_ids": [], + "object_ids": [], + "parent_raster_id": "parent-raster-16", + "perceptual_image_hash": "be4078e1ef9fa296", + "processed_image_sha256": "3030303030303030303030303030303030303030303030303030303030303030", + "raw_image_sha256": "1010101010101010101010101010101010101010101010101010101010101010", + "record_sha256": "3e8066639f58f5230faea2b2964c532148f9fc6d891a1d04a127c1fac1cdb826", + "sample_id": "background-test-pure-empty", + "source_family": "scene-16", + "split": "background-test", + "task": "object_detection", + "temporal_family": "temporal-16" + }, + { + "acquisition_date": "2025-01-13", + "acquisition_id": "acquisition-13", + "bbox": [ + 50000.0, + 50000.0, + 50100.0, + 50100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g13", + "label_geometry_fingerprint": "6c5b5f9db0d73c31", + "label_geometry_hash": "6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d", + "label_sha256": "4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d", + "native_feature_ids": [ + "native-13-1", + "native-13-2" + ], + "object_ids": [ + "o13", + "o14" + ], + "parent_raster_id": "parent-raster-13", + "perceptual_image_hash": "14e347040ef16eac", + "processed_image_sha256": "2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d", + "raw_image_sha256": "0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d", + "record_sha256": "0550facfe80b78f11414cd578d3808fab503382f15c9e6e1f8fd4801ce3ab365", + "sample_id": "change-test-wallonia-industrial", + "source_family": "scene-13", + "split": "test", + "task": "change_detection", + "temporal_family": "temporal-13" + }, + { + "acquisition_date": "2025-01-18", + "acquisition_id": "acquisition-18", + "bbox": [ + 10000.0, + 90000.0, + 10100.0, + 90100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g18", + "parent_raster_id": "parent-raster-18", + "perceptual_image_hash": "15910b3a2d056648", + "processed_image_sha256": "3232323232323232323232323232323232323232323232323232323232323232", + "raw_image_sha256": "1212121212121212121212121212121212121212121212121212121212121212", + "sample_id": "det-challenge-coast", + "sealed": true, + "source_family": "scene-18", + "split": "challenge", + "task": "object_detection", + "temporal_family": "temporal-18", + "withheld_fields": [ + "content_path_bindings", + "governance_binding", + "label_geometry_fingerprint", + "label_geometry_hash", + "label_geometry_path", + "label_path", + "label_sha256", + "native_feature_ids", + "object_ids", + "record_sha256" + ] + }, + { + "acquisition_date": "2025-01-09", + "acquisition_id": "acquisition-09", + "bbox": [ + 10000.0, + 50000.0, + 10100.0, + 50100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g09", + "label_geometry_fingerprint": "7ed221e11cc6828d", + "label_geometry_hash": "6969696969696969696969696969696969696969696969696969696969696969", + "label_sha256": "4949494949494949494949494949494949494949494949494949494949494949", + "native_feature_ids": [ + "native-09-1", + "native-09-2" + ], + "object_ids": [ + "o07", + "o08" + ], + "parent_raster_id": "parent-raster-09", + "perceptual_image_hash": "d6d20ed34050b89e", + "processed_image_sha256": "2929292929292929292929292929292929292929292929292929292929292929", + "raw_image_sha256": "0909090909090909090909090909090909090909090909090909090909090909", + "record_sha256": "a551408577ce3494c75160c1c4fe90fbbc79a9f37bd151d7df8095d1cd35bc8b", + "sample_id": "det-test-flanders-urban", + "source_family": "scene-09", + "split": "test", + "task": "object_detection", + "temporal_family": "temporal-09" + }, + { + "acquisition_date": "2025-01-20", + "acquisition_id": "acquisition-20", + "bbox": [ + 50000.0, + 90000.0, + 50100.0, + 90100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g20", + "parent_raster_id": "parent-raster-20", + "perceptual_image_hash": "85f82f2f8eb2c2f5", + "processed_image_sha256": "3434343434343434343434343434343434343434343434343434343434343434", + "raw_image_sha256": "1414141414141414141414141414141414141414141414141414141414141414", + "sample_id": "raster-challenge-seasonal", + "sealed": true, + "source_family": "scene-20", + "split": "challenge", + "task": "raster_classification", + "temporal_family": "temporal-20", + "withheld_fields": [ + "content_path_bindings", + "governance_binding", + "label_geometry_fingerprint", + "label_geometry_hash", + "label_geometry_path", + "label_path", + "label_sha256", + "native_feature_ids", + "object_ids", + "record_sha256" + ] + }, + { + "acquisition_date": "2025-01-11", + "acquisition_id": "acquisition-11", + "bbox": [ + 30000.0, + 50000.0, + 30100.0, + 50100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g11", + "label_geometry_fingerprint": "f4269ac17b100571", + "label_geometry_hash": "6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b", + "label_sha256": "4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b", + "native_feature_ids": [ + "native-11-1" + ], + "object_ids": [ + "o10" + ], + "parent_raster_id": "parent-raster-11", + "perceptual_image_hash": "4383fa06c328f326", + "processed_image_sha256": "2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b", + "raw_image_sha256": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "record_sha256": "d66076d5cb479670bc8716334f526ddcbab07a2aacab28d7a5d6893795618d56", + "sample_id": "raster-test-brussels-urban", + "source_family": "scene-11", + "split": "test", + "task": "raster_classification", + "temporal_family": "temporal-11" + }, + { + "acquisition_date": "2025-01-19", + "acquisition_id": "acquisition-19", + "bbox": [ + 30000.0, + 90000.0, + 30100.0, + 90100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g19", + "parent_raster_id": "parent-raster-19", + "perceptual_image_hash": "b5c6e201d687cf9e", + "processed_image_sha256": "3333333333333333333333333333333333333333333333333333333333333333", + "raw_image_sha256": "1313131313131313131313131313131313131313131313131313131313131313", + "sample_id": "seg-challenge-brussels", + "sealed": true, + "source_family": "scene-19", + "split": "challenge", + "task": "footprint_segmentation", + "temporal_family": "temporal-19", + "withheld_fields": [ + "content_path_bindings", + "governance_binding", + "label_geometry_fingerprint", + "label_geometry_hash", + "label_geometry_path", + "label_path", + "label_sha256", + "native_feature_ids", + "object_ids", + "record_sha256" + ] + }, + { + "acquisition_date": "2025-01-10", + "acquisition_id": "acquisition-10", + "bbox": [ + 20000.0, + 50000.0, + 20100.0, + 50100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g10", + "label_geometry_fingerprint": "e9fb3c93de82d0be", + "label_geometry_hash": "6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a", + "label_sha256": "4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a", + "native_feature_ids": [ + "native-10-1" + ], + "object_ids": [ + "o09" + ], + "parent_raster_id": "parent-raster-10", + "perceptual_image_hash": "6c476d18e0539687", + "processed_image_sha256": "2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a", + "raw_image_sha256": "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + "record_sha256": "7d0f9ee51cdf98e7f3acdc8117d48ce16d4d97538ba8179949a2c87dbbe5a526", + "sample_id": "seg-test-wallonia-rural", + "source_family": "scene-10", + "split": "test", + "task": "footprint_segmentation", + "temporal_family": "temporal-10" + }, + { + "acquisition_date": "2025-01-14", + "acquisition_id": "acquisition-14", + "bbox": [ + 60000.0, + 50000.0, + 60100.0, + 50100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g14", + "label_geometry_fingerprint": "4ec9f1dd6c799ec7", + "label_geometry_hash": "6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e", + "label_sha256": "4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e", + "native_feature_ids": [ + "native-14-1" + ], + "object_ids": [ + "o15" + ], + "parent_raster_id": "parent-raster-14", + "perceptual_image_hash": "c808940b0a325e9c", + "processed_image_sha256": "2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e", + "raw_image_sha256": "0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e", + "record_sha256": "4a47d90297a8325bb8257d7ca76b3281df8312d385715745a1a1fb74ba89efcd", + "sample_id": "terrain-test-flanders-rural", + "source_family": "scene-14", + "split": "test", + "task": "terrain_interpretation", + "temporal_family": "temporal-14" + }, + { + "acquisition_date": "2025-01-21", + "acquisition_id": "acquisition-21", + "bbox": [ + 70000.0, + 90000.0, + 70100.0, + 90100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g21", + "parent_raster_id": "parent-raster-21", + "perceptual_image_hash": "c71055ef6baeee8b", + "processed_image_sha256": "3535353535353535353535353535353535353535353535353535353535353535", + "raw_image_sha256": "1515151515151515151515151515151515151515151515151515151515151515", + "sample_id": "validation-challenge-corrupt", + "sealed": true, + "source_family": "scene-21", + "split": "challenge", + "task": "geospatial_data_validation", + "temporal_family": "temporal-21", + "withheld_fields": [ + "content_path_bindings", + "governance_binding", + "label_geometry_fingerprint", + "label_geometry_hash", + "label_geometry_path", + "label_path", + "label_sha256", + "native_feature_ids", + "object_ids", + "record_sha256" + ] + }, + { + "acquisition_date": "2025-01-15", + "acquisition_id": "acquisition-15", + "bbox": [ + 70000.0, + 50000.0, + 70100.0, + 50100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g15", + "label_geometry_fingerprint": "616691796f62dbbb", + "label_geometry_hash": "6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f", + "label_sha256": "4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f", + "native_feature_ids": [ + "native-15-1" + ], + "object_ids": [ + "o16" + ], + "parent_raster_id": "parent-raster-15", + "perceptual_image_hash": "4365e6b7df6acf3d", + "processed_image_sha256": "2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f", + "raw_image_sha256": "0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f", + "record_sha256": "0ff60414da05655b19c1717a27d8ba09aa8539a2b6c143e7a1a72b634fca397a", + "sample_id": "validation-test-national", + "source_family": "scene-15", + "split": "test", + "task": "geospatial_data_validation", + "temporal_family": "temporal-15" + }, + { + "acquisition_date": "2025-01-12", + "acquisition_id": "acquisition-12", + "bbox": [ + 40000.0, + 50000.0, + 40100.0, + 50100.0 + ], + "content_hash_verification": { + "label_geometry_hash": "synthetic_fixture_bound", + "label_sha256": "synthetic_fixture_bound", + "processed_image_sha256": "synthetic_fixture_bound", + "raw_image_sha256": "synthetic_fixture_bound" + }, + "group_id": "g12", + "label_geometry_fingerprint": "2db7d069691934ed", + "label_geometry_hash": "6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c", + "label_sha256": "4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c", + "native_feature_ids": [ + "native-12-1", + "native-12-2" + ], + "object_ids": [ + "o11", + "o12" + ], + "parent_raster_id": "parent-raster-12", + "perceptual_image_hash": "1236e1aa148c25d5", + "processed_image_sha256": "2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c", + "raw_image_sha256": "0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c", + "record_sha256": "63f561268e24208db0dcb4321cd66624188f2f66016338dac504d52051dae34a", + "sample_id": "vector-test-flanders-suburban", + "source_family": "scene-12", + "split": "test", + "task": "vector_comparison", + "temporal_family": "temporal-12" + } + ], + "schema_version": 1, + "selection_use_allowed": false, + "source_manifest_sha256": "1c75c875d4c297d046117bb9393ba5e648f13d3358ea92cb65374869c8f5fc01", + "source_trust": { + "mode": "synthetic_fixture", + "policy_id": "geointel-p4-synthetic-fixture-v2", + "production_accuracy_use_allowed": false + }, + "training_access_allowed": false +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/reference-implementation-baseline.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/reference-implementation-baseline.json new file mode 100644 index 00000000..84076ed9 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/reference-implementation-baseline.json @@ -0,0 +1,163 @@ +{ + "claim_boundary": "Reference implementation regression evidence; not production model accuracy.", + "content_sha256": "aeb0f6510352f4e205e4fc4e8c423110ca72c2142016c7af846d758865f258c6", + "implementation": "backend/app/services/qa_service.py via scripts/run_golden_qa_benchmark.py", + "persistence": { + "commit_count": 4, + "metric_count": 24, + "metric_keys": [ + "f1", + "false_negative_count", + "false_positive_count", + "mean_iou", + "precision", + "recall" + ], + "quality_check_count": 4 + }, + "scenario_count": 4, + "scenarios": [ + { + "benchmark_id": "golden-buildings-partial-match-v1", + "description": "Two reference building polygons and two candidate building polygons: one candidate matches ref-1, one candidate is a false positive, and ref-2 is a false negative.", + "fixtures": { + "candidate": "fixtures/golden/predicted_buildings.geojson", + "reference": "fixtures/golden/reference_buildings.geojson" + }, + "metrics": { + "f1": 0.5, + "false_negative_count": 1, + "false_positive_count": 1, + "mean_iou": 0.8339768339652592, + "precision": 0.5, + "recall": 0.5 + }, + "persistence": { + "commit_count": 1, + "metric_count": 6, + "metric_keys": [ + "precision", + "recall", + "f1", + "mean_iou", + "false_positive_count", + "false_negative_count" + ], + "quality_check_count": 1 + }, + "result_counts": { + "candidate_feature_count": 2, + "matches": 1, + "reference_feature_count": 2 + }, + "status": "passed" + }, + { + "benchmark_id": "golden-buildings-perfect-match-v1", + "description": "One candidate building polygon exactly matches one reference polygon.", + "fixtures": { + "candidate": "fixtures/golden/predicted_buildings_perfect.geojson", + "reference": "fixtures/golden/reference_buildings_perfect.geojson" + }, + "metrics": { + "f1": 1.0, + "false_negative_count": 0, + "false_positive_count": 0, + "mean_iou": 1.0, + "precision": 1.0, + "recall": 1.0 + }, + "persistence": { + "commit_count": 1, + "metric_count": 6, + "metric_keys": [ + "precision", + "recall", + "f1", + "mean_iou", + "false_positive_count", + "false_negative_count" + ], + "quality_check_count": 1 + }, + "result_counts": { + "candidate_feature_count": 1, + "matches": 1, + "reference_feature_count": 1 + }, + "status": "passed" + }, + { + "benchmark_id": "golden-buildings-no-overlap-v1", + "description": "One candidate polygon and one reference polygon do not overlap.", + "fixtures": { + "candidate": "fixtures/golden/predicted_buildings_no_overlap.geojson", + "reference": "fixtures/golden/reference_buildings_no_overlap.geojson" + }, + "metrics": { + "f1": null, + "false_negative_count": 1, + "false_positive_count": 1, + "mean_iou": null, + "precision": 0.0, + "recall": 0.0 + }, + "persistence": { + "commit_count": 1, + "metric_count": 6, + "metric_keys": [ + "precision", + "recall", + "f1", + "mean_iou", + "false_positive_count", + "false_negative_count" + ], + "quality_check_count": 1 + }, + "result_counts": { + "candidate_feature_count": 1, + "matches": 0, + "reference_feature_count": 1 + }, + "status": "passed" + }, + { + "benchmark_id": "golden-buildings-multipolygon-match-v1", + "description": "One candidate MultiPolygon exactly matches one reference MultiPolygon.", + "fixtures": { + "candidate": "fixtures/golden/predicted_buildings_multipolygon.geojson", + "reference": "fixtures/golden/reference_buildings_multipolygon.geojson" + }, + "metrics": { + "f1": 1.0, + "false_negative_count": 0, + "false_positive_count": 0, + "mean_iou": 1.0, + "precision": 1.0, + "recall": 1.0 + }, + "persistence": { + "commit_count": 1, + "metric_count": 6, + "metric_keys": [ + "precision", + "recall", + "f1", + "mean_iou", + "false_positive_count", + "false_negative_count" + ], + "quality_check_count": 1 + }, + "result_counts": { + "candidate_feature_count": 1, + "matches": 1, + "reference_feature_count": 1 + }, + "status": "passed" + } + ], + "status": "passed", + "version": 1 +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/release-gate-report.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/release-gate-report.json new file mode 100644 index 00000000..ca6bf907 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/release-gate-report.json @@ -0,0 +1,157 @@ +{ + "benchmark_manifest_sha256": "0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642", + "critical_subgroup_policy": "Any required subgroup with insufficient support, missing metrics, a failed non-inferiority comparison or regression blocks promotion; averages cannot override it.", + "decision": "At least one mandatory Phase 4 gate failed; Phase 5 remains blocked.", + "gate_policy": "geointel-p4-evaluation-harness-v2", + "invalid_gate_states": {}, + "local_gates": { + "all_declared_evaluator_families_exercised": { + "declared": [ + "change_detection", + "footprint_segmentation", + "geospatial_data_validation", + "object_detection", + "raster_classification", + "terrain_interpretation", + "vector_comparison" + ], + "observed": [ + "change_detection", + "footprint_segmentation", + "geospatial_data_validation", + "object_detection", + "raster_classification", + "terrain_interpretation", + "vector_comparison" + ], + "status": "pass" + }, + "complete_raw_predictions_retained": { + "status": "pass", + "violating_samples": [] + }, + "implemented_capability_inventory": { + "capability_count": 15, + "status": "pass" + }, + "manifest_training_firewall_contract": { + "checks": { + "clean_train_allowed": true, + "non_train_role_blocked": true, + "protected_path_blocked": true, + "renamed_protected_lineage_blocked": true + }, + "status": "pass" + }, + "normative_split_roles_and_leakage": { + "leakage_status": "pass", + "observed_roles": [ + "background-test", + "calibration", + "challenge", + "test", + "train", + "val" + ], + "required_roles": [ + "background-test", + "calibration", + "challenge", + "test", + "train", + "val" + ], + "status": "pass" + }, + "protected_operating_point_contract": { + "evidence": "Protected cases carry pre-registered configurations. Fixed AP/risk-coverage diagnostics cannot select an operating point or feed back into training.", + "status": "pass" + }, + "reference_implementation_baseline": { + "status": "pass" + }, + "stratified_metric_contract": { + "observed_overall_status": "not_evaluable", + "status": "pass" + }, + "undefined_metric_truth_table": { + "sample_id": "background-test-pure-empty", + "status": "pass" + } + }, + "local_harness_status": "pass", + "missing_gate_names": { + "local": [], + "product": [] + }, + "numeric_model_release_targets": "not_frozen_without_reviewed_representative_incumbent_baseline", + "phase_decision": "blocked", + "product_benchmark_status": "fail", + "product_gates": { + "active_model_available_and_hash_verified": { + "configured_path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt", + "configured_sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1", + "reason": "Configured active model is not locally accessible.", + "status": "not_evaluable" + }, + "authoritative_reference_portfolio_available": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "executed_product_incumbent_baseline": { + "checked_artifacts": [], + "derived_gates": { + "authoritative_reference_portfolio_available": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "human_review_complete": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "protected_storage_isolation": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "representative_product_subgroup_support": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "split_independence": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + } + }, + "expected_path": "artifacts/evidence/accuracy/P4/product-baseline-manifest.json", + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "human_review_complete": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "phase3_leakage_resolved": { + "observed": "attention", + "status": "fail" + }, + "protected_storage_isolation": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "representative_product_subgroup_support": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + }, + "split_independence": { + "reason": "No executed, hash-bound product incumbent baseline manifest is available.", + "status": "not_evaluable" + } + }, + "promotion_allowed": false, + "schema_version": 2, + "status": "fail", + "unexpected_gate_names": { + "local": [], + "product": [] + } +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/split-and-leakage-audit.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/split-and-leakage-audit.json new file mode 100644 index 00000000..69be6695 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/split-and-leakage-audit.json @@ -0,0 +1,46 @@ +{ + "checked_identity_fields": [ + "sample_id", + "group_id", + "source_family", + "acquisition_date", + "temporal_family", + "object_ids", + "native_feature_ids", + "raw_image_sha256", + "processed_image_sha256", + "label_sha256", + "record_sha256", + "perceptual_image_hash", + "label_geometry_hash", + "label_geometry_fingerprint", + "parent_raster_id", + "acquisition_id", + "bbox_distance" + ], + "crs_validation": { + "crs": "EPSG:31370", + "distance_units": "m", + "status": "pass" + }, + "development_manifest_sha256": "64e4fc986c2390bedfbe100062d4b06e1098689f4d4a9a9da13a7f67357c0493", + "finding_count": 0, + "findings": [], + "generator_version": "1.3.0", + "independence_buffer_m": 2000.0, + "inventory_total": 21, + "label_geometry_hamming_threshold": 2, + "perceptual_hamming_threshold": 4, + "protected_manifest_sha256": "e57ca1529654ef1c2527e84122ff26ec3533e7c70658250983968fbc5d4af410", + "schema_version": 1, + "source_manifest_sha256": "1c75c875d4c297d046117bb9393ba5e648f13d3358ea92cb65374869c8f5fc01", + "split_counts": { + "background-test": 2, + "calibration": 2, + "challenge": 4, + "test": 7, + "train": 3, + "val": 3 + }, + "status": "pass" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/stratified-metrics.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/stratified-metrics.json new file mode 100644 index 00000000..02b63292 --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/stratified-metrics.json @@ -0,0 +1,8798 @@ +{ + "dimensions": { + "context": { + "strata": { + "coastal_hard_negative": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": 0.0, + "precision_case_support": 1, + "recall": null, + "recall_case_support": 0, + "status": "computed" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 1, + "reference_count": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 1, + "predictions": 1, + "ranking_predictions": 1, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "data_contract": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "dense_urban": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "prediction_count": 4, + "reference_count": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 3, + "predictions": 3, + "ranking_predictions": 4, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.8 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "dense_urban_raster": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + }, + "industrial_change": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "pure_background": { + "case_support": 1, + "failure_count": 0, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "rural_buildings": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "suburban_buildings": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "terrain_profile": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "date": { + "strata": { + "2023-01-01": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + }, + "2024-05-01/2025-05-01": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "2025-01-01": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "2025-01-15": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "2025-02-01": { + "case_support": 1, + "failure_count": 0, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "2025-04-20": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "2025-06-15": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "prediction_count": 4, + "reference_count": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 3, + "predictions": 3, + "ranking_predictions": 4, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.8 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "2025-07-01": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": 0.0, + "precision_case_support": 1, + "recall": null, + "recall_case_support": 0, + "status": "computed" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 1, + "reference_count": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 1, + "predictions": 1, + "ranking_predictions": 1, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "2025-10-05": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "difficulty": { + "strata": { + "hard": { + "case_support": 6, + "failure_count": 10, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "object_detection": { + "case_support": 3, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "geospatial_data_validation", + "object_detection", + "raster_classification" + ] + }, + "normal": { + "case_support": 3, + "failure_count": 4, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation", + "terrain_interpretation", + "vector_comparison" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "municipality": { + "strata": { + "Antwerpen": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "prediction_count": 4, + "reference_count": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 3, + "predictions": 3, + "ranking_predictions": 4, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.8 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "Brussel": { + "case_support": 2, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection", + "raster_classification" + ] + }, + "Hasselt": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + }, + "Liège": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "Mol": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "Namur": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "Oostende": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": 0.0, + "precision_case_support": 1, + "recall": null, + "recall_case_support": 0, + "status": "computed" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 1, + "reference_count": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 1, + "predictions": 1, + "ranking_predictions": 1, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "not_applicable": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "object_size": { + "strata": { + "empty": { + "case_support": 2, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 2, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": 0.0, + "precision_case_support": 1, + "recall": null, + "recall_case_support": 0, + "status": "computed" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 1, + "reference_count": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 1, + "predictions": 1, + "ranking_predictions": 1, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "large": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "mixed": { + "case_support": 3, + "failure_count": 6, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "object_detection": { + "case_support": 1, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "prediction_count": 4, + "reference_count": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 3, + "predictions": 3, + "ranking_predictions": 4, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.8 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "object_detection", + "vector_comparison" + ] + }, + "not_applicable": { + "case_support": 3, + "failure_count": 5, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation", + "raster_classification", + "terrain_interpretation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "occlusion": { + "strata": { + "none": { + "case_support": 4, + "failure_count": 5, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "object_detection": { + "case_support": 2, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": 0.0, + "precision_case_support": 1, + "recall": null, + "recall_case_support": 0, + "status": "computed" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.6, + "mean_confidence": null, + "upper": 0.8 + }, + { + "accuracy": 0, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.92, + "upper": 1.0 + } + ], + "brier": 0.8464, + "ece": 0.92, + "status": "computed" + }, + "f1": null, + "false_discovery_rate": 1.0, + "false_negative": 0, + "false_positive": 1, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 1, + "reference_count": 0 + } + }, + "precision": 0.0, + "precision_ci95_wilson": { + "lower": 0.0, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 0.7934506856227626 + }, + "prediction_count": 1, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 1, + "predictions": 1, + "ranking_predictions": 1, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "footprint_segmentation", + "object_detection" + ] + }, + "not_applicable": { + "case_support": 4, + "failure_count": 7, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation", + "raster_classification", + "terrain_interpretation", + "vector_comparison" + ] + }, + "partial": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 1, + "count": 1, + "lower": 0.8, + "mean_confidence": 0.9, + "upper": 1.0 + } + ], + "brier": 0.15333333333333332, + "ece": 0.13333333333333325, + "status": "computed" + }, + "f1": 0.8, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 0, + "false_positive": 1, + "map50": 1.0, + "map50_95": 0.900990099009901, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 1.0, + "ap50_95": 0.900990099009901, + "prediction_count": 4, + "reference_count": 2 + } + }, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 3, + "predictions": 3, + "ranking_predictions": 4, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.8 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "region": { + "strata": { + "brussels": { + "case_support": 2, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection", + "raster_classification" + ] + }, + "flanders": { + "case_support": 4, + "failure_count": 7, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 2, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection", + "terrain_interpretation", + "vector_comparison" + ] + }, + "national": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "wallonia": { + "case_support": 2, + "failure_count": 3, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "footprint_segmentation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "resolution_m": { + "strata": { + "0.1": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "0.25": { + "case_support": 3, + "failure_count": 4, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 3, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "0.5": { + "case_support": 2, + "failure_count": 3, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "footprint_segmentation" + ] + }, + "1": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + }, + "5": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + }, + "__not_applicable__": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "season": { + "strata": { + "autumn": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + }, + "multi-date": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "not_applicable": { + "case_support": 3, + "failure_count": 5, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation", + "terrain_interpretation", + "vector_comparison" + ] + }, + "spring": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "summer": { + "case_support": 2, + "failure_count": 4, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 2, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "winter": { + "case_support": 1, + "failure_count": 0, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "sensor": { + "strata": { + "elevation-raster": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + }, + "mixed": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "synthetic-multispectral": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + }, + "synthetic-rgb": { + "case_support": 4, + "failure_count": 5, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "object_detection": { + "case_support": 3, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation", + "object_detection" + ] + }, + "vector": { + "case_support": 2, + "failure_count": 4, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "vector_comparison" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "source": { + "strata": { + "fixture-contracts": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "fixture-dhmv": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "terrain_interpretation" + ] + }, + "fixture-grb": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "fixture-orthophoto": { + "case_support": 3, + "failure_count": 4, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 3, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection" + ] + }, + "fixture-picc": { + "case_support": 1, + "failure_count": 1, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation" + ] + }, + "fixture-picc-temporal": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "fixture-thematic-raster": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "raster_classification" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "urbanity": { + "strata": { + "industrial": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection" + ] + }, + "mixed": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "rural": { + "case_support": 2, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "footprint_segmentation", + "terrain_interpretation" + ] + }, + "suburban": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "vector_comparison" + ] + }, + "urban": { + "case_support": 4, + "failure_count": 6, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 3, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection", + "raster_classification" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + }, + "vegetation": { + "strata": { + "low": { + "case_support": 4, + "failure_count": 5, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "change_detection": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "footprint_segmentation": { + "case_support": 1, + "macro": { + "f1": 1.0, + "f1_case_support": 1, + "mean_boundary_f1": 1.0, + "mean_boundary_f1_case_support": 1, + "mean_dice": 0.8881578947368421, + "mean_dice_case_support": 1, + "mean_iou": 0.7988165680473372, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 1.0, + "precision_case_support": 1, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "prediction_count": 1, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.20654931437723745, + "method": "Wilson score interval", + "status": "computed", + "support": 1, + "upper": 1.0 + }, + "reference_count": 1, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 1, + "references": 1 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 1.0 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "object_detection": { + "case_support": 1, + "macro": { + "f1": null, + "f1_case_support": 0, + "method": "unweighted mean across cases with defined values", + "precision": null, + "precision_case_support": 0, + "recall": null, + "recall_case_support": 0, + "status": "not_evaluable" + }, + "micro": { + "ap50": null, + "ap50_95": null, + "calibration": { + "bins": [], + "brier": null, + "ece": null, + "status": "undefined" + }, + "f1": null, + "false_discovery_rate": null, + "false_negative": 0, + "false_positive": 0, + "map50": null, + "map50_95": null, + "miss_rate": null, + "per_class_ap": { + "building": { + "ap50": null, + "ap50_95": null, + "prediction_count": 0, + "reference_count": 0 + } + }, + "precision": null, + "precision_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "prediction_count": 0, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": null, + "recall_ci95_wilson": { + "lower": null, + "status": "undefined", + "support": 0, + "upper": null + }, + "reference_count": 0, + "true_positive": 0 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 0, + "predictions": 0, + "ranking_predictions": 0, + "references": 0 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": null + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "vector_comparison": { + "case_support": 1, + "macro": { + "f1": 0.5, + "f1_case_support": 1, + "mean_iou": 1.0, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.5, + "precision_case_support": 1, + "recall": 0.5, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "f1": 0.5, + "false_discovery_rate": 0.5, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.5, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "prediction_count": 2, + "recall": 0.5, + "recall_ci95_wilson": { + "lower": 0.09453120573423074, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 0.9054687942657693 + }, + "reference_count": 2, + "true_positive": 1 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 2, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.5 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "change_detection", + "footprint_segmentation", + "object_detection", + "vector_comparison" + ] + }, + "mixed": { + "case_support": 1, + "failure_count": 2, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "geospatial_data_validation": { + "case_support": 1, + "macro": { + "blocker_or_critical_miss_count": 1.0, + "blocker_or_critical_miss_count_case_support": 1, + "f1": 0.6666666666666666, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.6666666666666666, + "precision_case_support": 1, + "recall": 0.6666666666666666, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "blocker_or_critical_miss_count": 1, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "predictions": 3, + "references": 3 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "geospatial_data_validation" + ] + }, + "moderate": { + "case_support": 4, + "failure_count": 7, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "task_metrics": { + "object_detection": { + "case_support": 2, + "macro": { + "f1": 0.8, + "f1_case_support": 1, + "method": "unweighted mean across cases with defined values", + "precision": 0.3333333333333333, + "precision_case_support": 2, + "recall": 1.0, + "recall_case_support": 1, + "status": "computed" + }, + "micro": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "calibration": { + "binning": "five fixed equal-width bins", + "bins": [ + { + "accuracy": null, + "count": 0, + "lower": 0.0, + "mean_confidence": null, + "upper": 0.2 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.2, + "mean_confidence": null, + "upper": 0.4 + }, + { + "accuracy": null, + "count": 0, + "lower": 0.4, + "mean_confidence": null, + "upper": 0.6 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.6, + "mean_confidence": 0.6499999999999999, + "upper": 0.8 + }, + { + "accuracy": 0.5, + "count": 2, + "lower": 0.8, + "mean_confidence": 0.91, + "upper": 1.0 + } + ], + "brier": 0.3266, + "ece": 0.27999999999999997, + "status": "computed" + }, + "f1": 0.6666666666666666, + "false_discovery_rate": 0.5, + "false_negative": 0, + "false_positive": 2, + "map50": 0.6666666666666666, + "map50_95": 0.5838283828382838, + "miss_rate": 0.0, + "per_class_ap": { + "building": { + "ap50": 0.6666666666666666, + "ap50_95": 0.5838283828382838, + "prediction_count": 5, + "reference_count": 2 + } + }, + "precision": 0.5, + "precision_ci95_wilson": { + "lower": 0.15003898915214947, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 0.8499610108478506 + }, + "prediction_count": 4, + "ranking_scope": "predictions pooled across cases; class- and sample-aware matching; no averaging of per-case AP", + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.34238022750665303, + "method": "Wilson score interval", + "status": "computed", + "support": 2, + "upper": 1.0 + }, + "reference_count": 2, + "true_positive": 2 + }, + "minimum_case_support": 5, + "observation_support": { + "calibration_predictions": 4, + "predictions": 4, + "ranking_predictions": 5, + "references": 2 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.f1", + "value": 0.6666666666666666 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "raster_classification": { + "case_support": 1, + "macro": { + "accuracy": 0.8333333333333334, + "accuracy_case_support": 1, + "macro_f1": 0.8222222222222223, + "macro_f1_case_support": 1, + "mean_iou": 0.7222222222222222, + "mean_iou_case_support": 1, + "method": "unweighted mean across cases with defined values", + "status": "computed" + }, + "micro": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "macro_f1_across_classes": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_present_pixel_count": 12 + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated_pixels": 12 + }, + "primary_metric": { + "direction": "higher_is_better", + "name": "micro.mean_iou", + "value": 0.7222222222222222 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + }, + "terrain_interpretation": { + "case_support": 1, + "macro": { + "bias": 0.02499999999999991, + "bias_case_support": 1, + "coverage": 0.8, + "coverage_case_support": 1, + "mae": 0.2749999999999999, + "mae_case_support": 1, + "method": "unweighted mean across cases with defined values", + "rmse": 0.2872281323269015, + "rmse_case_support": 1, + "status": "computed" + }, + "micro": { + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "reference_count": 5, + "rmse": 0.2872281323269015, + "status": "computed", + "units": "m_TAW" + }, + "minimum_case_support": 5, + "observation_support": { + "evaluated": 4, + "references": 5 + }, + "primary_metric": { + "direction": "lower_is_better", + "name": "micro.rmse", + "value": 0.2872281323269015 + }, + "release_gate_reason": "insufficient_support", + "release_gate_status": "not_evaluable", + "status": "insufficient_support" + } + }, + "tasks": [ + "object_detection", + "raster_classification", + "terrain_interpretation" + ] + } + }, + "worst_stratum_by_task": { + "change_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "footprint_segmentation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "geospatial_data_validation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "object_detection": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "raster_classification": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "terrain_interpretation": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + }, + "vector_comparison": { + "reason": "no stratum meets minimum support with a defined primary metric", + "status": "not_evaluable" + } + } + } + }, + "minimum_case_support": 5, + "overall_status": "not_evaluable", + "schema_version": 2, + "support_unit": "independent benchmark cases; pixel/object counts are reported separately and do not replace case support" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/tile-metrics.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/tile-metrics.json new file mode 100644 index 00000000..f51ec5ee --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/tile-metrics.json @@ -0,0 +1,207 @@ +{ + "results": [ + { + "metadata": { + "context": "dense_urban_raster", + "date": "2025-10-05", + "difficulty": "hard", + "label_review_state": "fixture_reviewed", + "municipality": "Brussel", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "brussels", + "resolution_m": 1, + "season": "autumn", + "sensor": "synthetic-multispectral", + "source": "fixture-thematic-raster", + "tile_edge": true, + "urbanity": "urban", + "vegetation": "moderate" + }, + "metrics": { + "accuracy": 0.8333333333333334, + "accuracy_ci95_wilson": { + "lower": 0.5519691377470266, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 0.9530348578161463 + }, + "alignment_validated": true, + "confusion_matrix": { + "0": { + "0": 4, + "1": 0, + "2": 0, + "__prediction_nodata__": 0 + }, + "1": { + "0": 0, + "1": 4, + "2": 1, + "__prediction_nodata__": 0 + }, + "2": { + "0": 0, + "1": 1, + "2": 2, + "__prediction_nodata__": 0 + } + }, + "correct_pixel_count": 10, + "evaluated_reference_pixel_count": 12, + "ignored_reference_pixel_count": 0, + "macro_f1": 0.8222222222222223, + "mean_iou": 0.7222222222222222, + "per_class": { + "0": { + "f1": 1.0, + "false_discovery_rate": 0.0, + "false_negative": 0, + "false_positive": 0, + "iou": 1.0, + "miss_rate": 0.0, + "precision": 1.0, + "precision_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "prediction_count": 4, + "recall": 1.0, + "recall_ci95_wilson": { + "lower": 0.5101091635454027, + "method": "Wilson score interval", + "status": "computed", + "support": 4, + "upper": 1.0 + }, + "reference_count": 4, + "true_positive": 4 + }, + "1": { + "f1": 0.8000000000000002, + "false_discovery_rate": 0.2, + "false_negative": 1, + "false_positive": 1, + "iou": 0.6666666666666666, + "miss_rate": 0.2, + "precision": 0.8, + "precision_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "prediction_count": 5, + "recall": 0.8, + "recall_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "reference_count": 5, + "true_positive": 4 + }, + "2": { + "f1": 0.6666666666666666, + "false_discovery_rate": 0.3333333333333333, + "false_negative": 1, + "false_positive": 1, + "iou": 0.5, + "miss_rate": 0.3333333333333333, + "precision": 0.6666666666666666, + "precision_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "prediction_count": 3, + "recall": 0.6666666666666666, + "recall_ci95_wilson": { + "lower": 0.2076596008020477, + "method": "Wilson score interval", + "status": "computed", + "support": 3, + "upper": 0.9385080552796037 + }, + "reference_count": 3, + "true_positive": 2 + } + }, + "pixel_count": 12, + "prediction_coverage": 1.0, + "prediction_coverage_ci95_wilson": { + "lower": 0.7575059933447592, + "method": "Wilson score interval", + "status": "computed", + "support": 12, + "upper": 1.0 + }, + "prediction_outside_reference_count": 0, + "prediction_present_pixel_count": 12 + }, + "sample_id": "raster-test-brussels-urban", + "task": "raster_classification" + }, + { + "metadata": { + "context": "terrain_profile", + "date": "2023-01-01", + "difficulty": "normal", + "label_review_state": "fixture_reviewed", + "municipality": "Hasselt", + "object_size": "not_applicable", + "occlusion": "not_applicable", + "ood": false, + "region": "flanders", + "resolution_m": 5, + "season": "not_applicable", + "sensor": "elevation-raster", + "source": "fixture-dhmv", + "tile_edge": false, + "urbanity": "rural", + "vegetation": "moderate" + }, + "metrics": { + "absolute_error_sum": 1.0999999999999996, + "bias": 0.02499999999999991, + "coverage": 0.8, + "coverage_ci95_wilson": { + "lower": 0.37553462976252533, + "method": "Wilson score interval", + "status": "computed", + "support": 5, + "upper": 0.9637758913675698 + }, + "error_distribution": { + "count": 4, + "max": 0.40000000000000036, + "mean": 0.02499999999999991, + "median": 0.0, + "min": -0.3000000000000007 + }, + "error_sum": 0.09999999999999964, + "evaluated_count": 4, + "mae": 0.2749999999999999, + "missing_count": 1, + "reference_count": 5, + "rmse": 0.2872281323269015, + "squared_error_sum": 0.3300000000000001, + "units": "m_TAW" + }, + "sample_id": "terrain-test-flanders-rural", + "task": "terrain_interpretation" + } + ], + "schema_version": 2, + "status": "fixture_contract_only" +} diff --git a/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/workflow-summary.json b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/workflow-summary.json new file mode 100644 index 00000000..5aaecc8f --- /dev/null +++ b/artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/workflow-summary.json @@ -0,0 +1,28 @@ +{ + "benchmark_manifest_sha256": "0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642", + "case_count": 9, + "evaluation_results_canonical_json_sha256": "55cdba9cce4dcdc88cee0829cd510d7e624a3472feb9d33f7e681621a792eba3", + "evidence_path": "artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b", + "evidence_run_id": "p4-2.0.1-9677d0ef37db82bcf39b", + "failure_example_count": 14, + "implemented_capability_count": 15, + "local_harness_status": "pass", + "phase4_done": false, + "phase5_ready": false, + "phase_decision": "blocked", + "product_benchmark_status": "fail", + "product_gate_evidence_sha256": "c9f5e3de3ab6d64911f8807b26caa661ccf911a5fd6b995107edda2004836e0c", + "promotion_allowed": false, + "reference_baseline_sha256": "aeb0f6510352f4e205e4fc4e8c423110ca72c2142016c7af846d758865f258c6", + "schema_version": 2, + "split_counts": { + "background-test": 2, + "calibration": 2, + "challenge": 4, + "test": 7, + "train": 3, + "val": 3 + }, + "status": "fail", + "task_family_count": 7 +} diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index cf745b76..826520e4 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -12391,3 +12391,56 @@ Open: **ready** for remediation/review. This does not unlock training, promotion, national validation or production release; those gates remain governed by the execution contract and the incomplete Phase 2 gates. + +## 2026-08-02 - Accuracy Improvement Program Phase 4 evaluation harness + +### Implemented and verified scope + +- Added a deterministic evaluator for object detection, building-footprint + segmentation, categorical raster interpretation, vector comparison, change + detection, terrain/height interpretation and geospatial data validation. It + retains exact references, pre-/post-filter predictions, config and lineage, + and reports task metrics, Wilson intervals, calibration, abstention, + subgroup support and a stable failure taxonomy. +- Added a six-role split generator for `train`, `val`, `calibration`, `test`, + `background-test` and sealed `challenge`, with exact/near-duplicate, + identity, acquisition, object, geometry and 2-km spatial leakage gates. + Product mode requires strict P3/provenance records and rehashes four physical + asset types; protected roles are rejected by the training firewall. +- Added a fail-closed governed product-baseline validator. It recomputes all + seven task families in-process and verifies current active-model bytes, + CUDA/GPU identity, authority, human review, leakage, vault and all thirteen + subgroup dimensions. Loose status fields cannot turn a productgate green. +- Fixed an evidence-ledger self-reference in workflow v2.0.1: only the + gate-relevant `/runtime/active_model` projection from `status.json` is + canonically hashed. Updating timestamps, Phase-4/5 bookkeeping, documents or + evidence IDs therefore leaves the run fingerprint stable; changing the + active model changes it. + +### Evidence and verification + +- Normative run `p4-2.0.1-9677d0ef37db82bcf39b` is bound to source commit + `70fb4b94e5cb7c248beec5a936ce186f38cc183c`. Its canonical benchmark hash is + `0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642` and + its evidence-manifest file hash is + `fdc15a95ee2a0754dfa169f4b41036e084b8d8909afc37fd8ea68ee6b9210f98`. +- The targeted Phase-4 suite passed 60 tests. The broader contracts, + provenance, Phase-3/4, golden-QA, runtime-model and migration suite passed + 103 tests with the installed Git Bash provider; the Windows Store WSL stub + was unavailable and is not a code failure. The focused migration/provenance + subset passed 29 tests and Alembic has one head, `202608010001`. +- Ruff, Ruff format check, compileall and diff check passed on all Phase-4 + paths. No Python mypy/pyright typecheck is configured in this repository. +- Repeated workflow execution produced the same run ID and byte-identical + immutable bundle. `--allow-product-blocked` returned 0; standard fail-closed + execution returned 2. + +### Decision + +- The local harness is `pass`, but the governed product benchmark is `fail` + because Phase-3 leakage remains `attention`. Active model execution, + authority coverage, representative review, independent product splits, + physical vault isolation and thirteen-dimension subgroup support remain + `not_evaluable` without governed evidence. Phase 4 remains **in progress**, + Phase 5 remains **not ready**, and promotion/training feedback from protected + data is not authorized. diff --git a/docs/TODO.md b/docs/TODO.md index 5c2d5c84..8b980b8b 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -16,6 +16,34 @@ Uitvoeringsbord: `docs/PYTORCH_TRAINING_ROADMAP_BELGIUM.md`. - [ ] Train en evalueer een Belgische gebouwchallenger onafhankelijk; promoveer alleen zonder achtergrondregressie. - [ ] Houd zonnepanelen en segmentatie `not_configured` tot gereviewde taaklabels en hold-outs bestaan. +## Accuracy-programma — fase 4 + +Normatieve protocollen: `docs/accuracy-program/10-evaluation-protocol.md`, +`11-baseline-benchmark.md` en `12-release-gates.md`. + +- [x] Inventariseer 15 geïmplementeerde capabilities in zeven afzonderlijke + evaluatorfamilies zonder deterministische GIS als geleerd model voor te stellen. +- [x] Implementeer taakgebonden metrics, raw prediction-retentie, + betrouwbaarheidsintervallen, subgroepsupport en een machineleesbare + failure-taxonomie. +- [x] Genereer zes code-owned splitrollen (`train`, `val`, `calibration`, + `test`, `background-test`, sealed `challenge`) met exacte, near-duplicate-, + identiteit-, tijd- en 2-km-leakagegates. +- [x] Bescherm test/background-test/challenge tegen training, thresholdselectie, + modelselectie, gewijzigde bytes en onbewezen provenance. +- [x] Voer de bestaande `QaService` als referentienulmeting uit en leg alle + Phase-4-evidence content-addressed en immutable vast. +- [x] Maak statusboekhouding zelfreferentievrij: alleen de actieve-modelprojectie + beïnvloedt de benchmarkfingerprint; productgates vereisen governed evidence. +- [ ] Los de bevestigde Phase-3-leakagebevinding op en freeze nieuwe onafhankelijke + product-splits zonder cross-splitparen onder 2 km. +- [ ] Lever één governed zeven-taken productbaseline met actuele modelbytes, + CUDA/GPU-receipt, authorityportfolio en in-process herberekende raw metrics. +- [ ] Lever checksumgebonden menselijke review, fysieke protected vault en + representatieve support voor alle 13 kritieke subgroepdimensies. +- [ ] Markeer fase 4 pas `done` en fase 5 pas `ready` wanneer alle productgates + in dezelfde workflow `pass` zijn; huidige status is local `pass`, product `fail`. + ## Actieve post-RC datadekkingsfase Professionaliseringspass (2026-07-27): diff --git a/docs/accuracy-program/08-data-contracts.md b/docs/accuracy-program/08-data-contracts.md index 8dbfeed6..a43228d5 100644 --- a/docs/accuracy-program/08-data-contracts.md +++ b/docs/accuracy-program/08-data-contracts.md @@ -555,3 +555,8 @@ The source/provenance part of Fase 2 is green only when: Until all gates are green, Fase 2 is in progress, Fase 3 is not ready and training/promotion remains blocked by the Phase-1 execution contract. + +> **Historische handoff:** deze zin legt de P2-gate vast zoals die bij oplevering +> van dit document gold. Fase 3 is nadien binnen de afgebakende lokale scope +> uitgevoerd. De actuele fasebeslissing staat uitsluitend in +> `docs/accuracy-program/status.json`. diff --git a/docs/accuracy-program/09-full-data-scan.md b/docs/accuracy-program/09-full-data-scan.md index 31c69c0d..b1685337 100644 --- a/docs/accuracy-program/09-full-data-scan.md +++ b/docs/accuracy-program/09-full-data-scan.md @@ -70,3 +70,8 @@ De fixturetest controleert geldige en foutieve GeoJSON, corrupte rasterinput, ex ## Fasebeslissing Fase 3 is `done` voor de afgebakende projectomgeving: alle 295 veilige lokale bestanden zijn verwerkt en de drie niet-bereikbare externe grenzen zijn expliciet geregistreerd. Fase 4 is `ready` voor verdere beoordeling van de gevonden quarantine- en leakage-signalen. Dit is geen modelpromotie- of nationale kwaliteitsclaim; de inhoudelijke anomalieën moeten eerst door de volgende fase worden hersteld of menselijk beoordeeld. + +> **Historische handoff:** `ready` beschrijft het overdrachtsmoment direct na +> de Phase-3-scan. Het Phase-4-harnas is inmiddels geïmplementeerd, maar de +> actuele productbenchmark is nog `not_evaluable`; de normatieve actuele +> status staat in `docs/accuracy-program/status.json`. diff --git a/docs/accuracy-program/10-evaluation-protocol.md b/docs/accuracy-program/10-evaluation-protocol.md new file mode 100644 index 00000000..04cb35e2 --- /dev/null +++ b/docs/accuracy-program/10-evaluation-protocol.md @@ -0,0 +1,175 @@ +# Fase 4 — Evaluatieprotocol + +## Doel en huidige claimgrens + +Het Phase-4-harnas maakt evaluatorgedrag, splits, ruwe voorspellingen en +releasebeslissingen reproduceerbaar. De lokale referentiecases zijn bewust +synthetisch en bewijzen uitsluitend dat het harnas correct en fail-closed +werkt. Zij zijn geen meting van productie-accuracy, België-brede +generaliseerbaarheid of menselijke aanvaardbaarheid. + +De volledige workflow is: + +```powershell +python scripts/run_accuracy_phase4_benchmark.py +``` + +De standaarduitvoering retourneert exitcode `2` zolang een productgate niet +groen is. Voor het uitsluitend regenereren en testen van lokale evidence mag: + +```powershell +python scripts/run_accuracy_phase4_benchmark.py --allow-product-blocked +``` + +worden gebruikt. Die vlag verandert geen gate, score of beslissing. Hij maakt +alleen een succesvolle lokale harnascontrole bruikbaar in CI terwijl externe +productinput aantoonbaar ontbreekt. + +## Geïmplementeerde taken + +De inventaris scheidt geleerde modellen van deterministische GIS-analyse. Een +officiële of deterministische GIS-functie wordt niet kunstmatig als +machine-learningmodel voorgesteld. + +De machineleesbare inventaris omvat 15 concrete platformcapabilities, gemapt +op zeven evaluatorfamilies. Een family-dekking geldt niet automatisch als een +zelfstandige benchmarkclaim voor iedere onderliggende microfunctionaliteit. + +| Taakfamilie | Werkelijke implementatie | Phase-4-metrics | Huidige productstatus | +| --- | --- | --- | --- | +| Objectdetectie | `backend/app/services/detection_service.py` | precision, recall, F1, AP50, AP50–95, matched IoU, ECE, Brier, coverage-risk | evaluatorcontract groen; actieve modelbenchmark lokaal niet uitvoerbaar | +| Gebouwfootprintsegmentatie | `backend/app/services/segmentation_service.py` | object-P/R/F1, IoU, Dice, boundary F1, centroidafstand, relatieve oppervlaktefout, topologie | evaluatorcontract groen; geen representatieve beschermde productset bereikbaar | +| Vectorvergelijking en detectie-QA | `backend/app/services/qa_service.py`, `backend/app/services/detection_qa_service.py` | object-P/R/F1, mean IoU, topologische geldigheid | referentie-implementatie via de golden QA-fixtures uitgevoerd | +| Veranderingsdetectie | `backend/app/services/change_detection_service.py` | event-level P/R/F1, apart voor toegevoegd en verwijderd | evaluatorcontract groen; productbaseline niet vastgesteld | +| Thematische rasterinterpretatie | `backend/app/services/thematic_raster_analysis_service.py` en rasterservices | confusion matrix, pixelaccuracy, class-P/R/F1, class-IoU, mean IoU | alleen metriekcontract voor categorische rasters; er is geen geleerd generiek rasterclassificatiemodel aangetroffen | +| Hoogte- en terreininterpretatie | terrein-, hoogte-, bathymetrie- en overstromingsservices | MAE, RMSE, bias, coverage en foutverdeling in gedeclareerde eenheid | deterministische bronanalyse; referentiedata en eenheden blijven taakgebonden | +| Geospatiale datavalidatie | `backend/app/services/data_contract_validation.py` en Phase-3-scanner | anomaly-P/R/F1, blocker/critical misses | contractfixture groen; echte bronscan blijft de Phase-3-evidence | + +Vector clip/buffer/intersect, ruimtelijke aggregatie, raster inspect/reproject/ +clip/tile, NDVI/NDWI/NDBI, flood hazard, bathymetrie en AOI-partitionering +vallen onder de overeenkomstige deterministische validatie-, vector-, raster- +of terreinfamilie. De geo-assistent is een orkestratie-interface en krijgt geen +misleidende zelfstandige accuracy-score; de onderliggende toolresultaten +blijven maatgevend. + +## Metriccontract + +Alle objectmatches zijn one-to-one en gebruiken de vooraf vastgelegde +taakconfiguratie. Er is geen data-afhankelijke threshold-, operating-point- of +modelselectie op test-, background-test- of challenge-input. Vaste diagnostische +AP- en coverage-riskcurves veranderen het vooraf geregistreerde operating point +niet. Ongedefinieerde delingen worden `null` met expliciete support, niet +kunstmatig `1.0`. + +| Uitvoertype | Verplichte kernmetingen | Aanvullende controle | +| --- | --- | --- | +| Objecten | TP, FP, FN, precision, recall, F1 | IoU, AP, calibration en abstention | +| Footprints | objectmetingen, mean IoU en Dice | boundary F1, centroid, area en topologie | +| Categorische rasters | confusion matrix, per-class F1/IoU, mean IoU | pixelaccuracy en class-support | +| Changes | event-level P/R/F1 per changeklasse | globale score mag een klasse niet maskeren | +| Continue hoogte | MAE, RMSE, bias | coverage, eenheid en foutverdeling | +| Validatie | anomaly-P/R/F1 | iedere gemiste blocker/critical anomaly blokkeert | + +Voor binomiale precision en recall rapporteert het harnas 95%-Wilsonintervallen. +Bij te weinig support blijft de subgroepgate `not_evaluable`; de supportreden +blijft afzonderlijk zichtbaar. Een breed interval of ontbrekende metric is geen +positief bewijs. Voor een toekomstige productbenchmark moet bij ruimtelijk +geclusterde observaties bovendien een vooraf vastgelegde AOI- of +clusterbootstrap worden gebruikt in plaats van pixels als onafhankelijke +steekproeven te behandelen. + +## Splitcontract en leakage-gates + +`scripts/generate_accuracy_phase4_splits.py` genereert twee afzonderlijke, +gehashte manifesten uit één versieerbare bron: + +- development: `train`, `val` en `calibration`; +- protected release only: immutable `test`, `background-test` en sealed + `challenge`. + +Iedere sample draagt minimaal taak, split, `sample_id`, ruimtelijke `group_id`, +`source_family`, `temporal_family`, acquisition- en parent-rasteridentiteit, +native feature- en object-ID's, bounding box, ruwe en verwerkte imagehash, +labelhash, perceptual imagehash, labelgeometriehash en een canonieke recordhash. +Bij automatische toewijzing worden gekoppelde records eerst als één component +gegroepeerd. De generator faalt wanneer: + +- een identiteit, object, bron-/tijdsfamilie, acquisition of parent-raster + meerdere splits raakt; +- exacte image-, label- of recordbytes meerdere splits raken; +- perceptuele of geometrische bijna-duplicaten meerdere splits raken; +- bounding boxes uit verschillende splits dichter liggen dan de vooraf + gedeclareerde onafhankelijkheidsbuffer; +- een verplichte split ontbreekt of een record niet valideerbaar is. + +Bronvolgorde heeft geen invloed op de bron- of splitmanifesthash. De huidige +fixture bevat 21 samples over alle zes normatieve rollen en gebruikt een buffer +van 2.000 meter. Zij kan alleen door een expliciete codeparameter als synthetische +fixture worden geopend en draagt altijd `production_accuracy_use_allowed=false`. +Productiemodus vereist een strikt P3-manifest, exact provenance-record, vier +toegankelijke assetpaden en herberekende bytes-hashes. Dit is dus geen bewijs dat +het historische Belgische corpus onafhankelijk is. De Phase-3-bevinding van 24 +cross-splitparen onder 2 km blijft blokkerend. + +## Bescherming van test en challenge + +De lokale workflow bindt evaluatie aan de exacte protected-manifesthash en +accepteert alleen de `test`- en `background-test`-ID's uit dat manifest. +Challengecases en -labels worden door de evaluator geweigerd. Configuraties zijn +vooraf vast. Per case bewaart `baseline-raw-predictions.json` de exacte +referenties, pre-filter- en post-filtervoorspellingen, configuratie en lineage. +`assert_training_inputs_safe()` en de twee echte trainingsdataset-builders +weigeren beschermde rollen, paden, inhoudshashes en alle relevante identiteit- +en lineagevelden. In productiemodus worden toegankelijke bronbytes en hun P3- +en provenancebinding opnieuw gecontroleerd; een padloos of hernoemd record faalt. +De regressietests bewijzen blokkering en een geldige development-run. + +Dit is logische bescherming in de repository. Fysieke isolatie met een +afzonderlijke vault, beperkte credentials en een immutable accesslog is in de +huidige projectomgeving niet bewezen. Daarom staat +`protected_storage_isolation` in de productgates op `not_evaluable`; de lokale +firewall mag niet als vervanging voor die productcontrole worden beschreven. + +## Stratificatie en failure-evidence + +Iedere beschermde case declareert waar relevant regio/gemeente, stedelijk of +landelijk karakter, objectgrootte, bron/sensor, resolutie, seizoen/datum, +vegetatie/occlusie, moeilijkheid en context. Het metricrapport groepeert die velden en +legt support expliciet vast. Kritieke subgroepen moeten vóór een echte baseline +worden vastgesteld; ontbrekende of te kleine groepen blokkeren een release. + +`baseline-raw-predictions.json` bewaart de ongesommeerde referenties, +pre-/postfiltervoorspellingen, matches, configuratie en lineage. +`failure-gallery.json` bewaart stabiele failure-ID's, taak, fouttype, strata en +concreet machineleesbaar bewijs. Een visuele gallery van echte productbeelden +kan pas worden gemaakt wanneer de beschermde imagery rechtmatig en gecontroleerd +bereikbaar is. + +## Reproduceerbaarheid + +Het benchmarkmanifest bindt repositorycommit, evaluator- en workflowversie, +alle drie Phase-4-scripts, de echte QA-service, bronfixtures, protected cases, +golden QA-fixture, Phase-3-manifesten, bronmatrix, metriekcontract, runtime en +beide splitmanifesten met SHA-256. Uit `status.json` wordt uitsluitend de +gate-relevante `/runtime/active_model`-projectie canoniek gehasht. Administratieve +velden zoals `generated_at`, Phase-4/5-status, documenten en evidence-run-ID's +zijn bewust geen benchmarkinput: zo kan het schrijven van de evidenceledger niet +zijn eigen runfingerprint veranderen. Review-, authority-, leakage-, vault- en +subgroepgates kunnen daardoor niet uit losse statusvelden slagen; zij vereisen +checksumgebonden product- en Phase-3-evidence. Een productbaseline wordt alleen +geaccepteerd wanneer dezelfde evaluator alle zeven taakfamilies in-process +herberekent en exact overeenkomt met checksumgebonden raw-, review-, authority-, +leakage-, vault- en CUDA-evidence. Dynamische UUID's uit de QA-service worden uit +de canonieke referentiescore verwijderd. Iedere run krijgt een content-addressed +ID; twee gelijke uitvoeringen leveren byte-identieke inhoud op en onverwachte +bestanden of submappen maken de immutable bundle ongeldig. + +```powershell +python scripts/run_accuracy_phase4_benchmark.py --allow-product-blocked +python -m pytest backend/tests/test_accuracy_phase4_evaluation.py backend/tests/test_accuracy_phase4_evaluator_hardening.py backend/tests/test_accuracy_phase4_split_hardening.py -q -p no:cacheprovider +python -m ruff check scripts/accuracy_phase4_evaluator.py scripts/generate_accuracy_phase4_splits.py scripts/run_accuracy_phase4_benchmark.py backend/tests/test_accuracy_phase4_evaluation.py backend/tests/test_accuracy_phase4_evaluator_hardening.py backend/tests/test_accuracy_phase4_split_hardening.py +``` + +De machineleesbare bewijsset staat onder +`artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/`. De +actuele fasebeslissing staat in `12-release-gates.md` en `status.json`. diff --git a/docs/accuracy-program/11-baseline-benchmark.md b/docs/accuracy-program/11-baseline-benchmark.md new file mode 100644 index 00000000..62ce1466 --- /dev/null +++ b/docs/accuracy-program/11-baseline-benchmark.md @@ -0,0 +1,124 @@ +# Fase 4 — Baselinebenchmark + +## Beslissing + +Het lokale evaluatieharnas is reproduceerbaar en groen. Een betrouwbare +baseline van het actieve GeoIntel-productiemodel is in deze omgeving niet +uitvoerbaar. De productbenchmark is `fail` door de bevestigde Phase-3-leakage; +andere productgates blijven `not_evaluable` zonder governed bewijs. Er wordt +daarom geen productie-accuracy-, België-brede of promotieclaim afgeleid. + +| Statusveld | Waarde | +| --- | --- | +| Benchmark-ID | `geointel-p4-reference-harness-v2` | +| Lokale harnasstatus | `pass` | +| Productbenchmark | `fail` | +| Promotie toegestaan | `false` | +| Phase 4 done | `false` | +| Phase 5 ready | `false` | + +## Referentie-implementatie als nulmeting + +De bestaande productiecode in `backend/app/services/qa_service.py` is via +`scripts/run_golden_qa_benchmark.py` uitgevoerd op vier bevroren golden +QA-scenario's. De dynamische service-UUID wordt niet meegenomen in de +canonieke inhoudshash. + +| Golden scenario | Precision | Recall | F1 | Mean matched IoU | +| --- | ---: | ---: | ---: | ---: | +| Partial match | 0,50 | 0,50 | 0,50 | 0,8339768339652592 | +| Perfect match | 1,00 | 1,00 | 1,00 | 1,00 | +| No overlap | 0,00 | 0,00 | `null` | `null` | +| Multipolygon match | 1,00 | 1,00 | 1,00 | 1,00 | + +De canonieke referentiehash is +`aeb0f6510352f4e205e4fc4e8c423110ca72c2142016c7af846d758865f258c6`. +Dit bewijst de rekenkundige regressiestabiliteit van de QA-service, niet de +kwaliteit van een detector of segmentatiemodel. + +## Multi-task evaluatorbaseline + +Negen synthetische, deterministische `test`/`background-test`-cases oefenen +zeven evaluatorfamilies en 15 concrete capabilityrecords uit. De gecombineerde +resultaathash is +`55cdba9cce4dcdc88cee0829cd510d7e624a3472feb9d33f7e681621a792eba3`. +De cases zijn bewust klein; hun scores testen formules, null-semantiek, +raw-retentie, stratificatie en foutclassificatie, niet productkwaliteit. + +| Familie | Voornaamste fixture-uitkomst | Claimgrens | +| --- | --- | --- | +| Objectdetectie | P 0,667; R 1,000; F1 0,800; AP50 1,000; AP50–95 0,901 | formule-/harnastest | +| Footprintsegmentatie | mean IoU 0,799; Dice 0,888; boundary F1 1,000 | formule-/harnastest | +| Categorisch raster | accuracy 0,833; macro F1 0,822; mean IoU 0,722 | metriekcontract, geen geleerd rastermodel | +| Vectorvergelijking | P/R/F1 0,500; mean matched IoU 1,000 | formule-/harnastest | +| Change detection | globale P/R/F1 0,500; afzonderlijke added/removed metrics | eventclassificatie zichtbaar | +| Terrein | MAE 0,275 m TAW; RMSE 0,287 m TAW; coverage 0,800 | eenheids-/coveragecontract | +| Datavalidatie | P/R/F1 0,667; één critical miss | fail-closed anomaliecontract | + +Alle subgroepen hebben in deze kleine fixture onvoldoende statistische support. +Hun `release_gate_status` is daarom `not_evaluable`, met de supportreden apart +geregistreerd; geen gemiddelde kan dit omzetten in releasebewijs. De +95%-Wilsonintervallen zijn overeenkomstig breed. + +## Ruwe voorspellingen en fouten + +De workflow bewaart exacte referenties, pre-/postfiltervoorspellingen, +configuratie en lineage vóór iedere samenvatting. De failure gallery bevat 14 +deterministische voorbeelden, waaronder FP/FN, boundary- en area-bias, +rastermisclassificatie, calibratie en een gemiste critical-validatieafwijking. +De failure-ID's zijn inhoudsgebonden en kunnen in regressietests worden +hergebruikt. + +Relevante evidence: + +- `baseline-raw-predictions.json` — exacte raw referenties, pre-/postfilteroutputs, config en lineage; +- `metric-report.json` — taak- en subgroepmetingen met support en intervallen; +- `failure-gallery.json` — machineleesbare fouttaxonomie en voorbeelden; +- `reference-implementation-baseline.json` — werkelijke QA-service-uitvoering; +- `benchmark-manifest.json` — alle input-, split-, code- en evaluatoridentiteiten. + +De canonieke benchmarkmanifesthash is +`0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642`; +de exacte bestandshash is +`868dd7eb2dfa8344e417879ed3f5dd7c7d9e5673add7e3e2294c5cbde4b75d57`. +Evidence-run `p4-2.0.1-9677d0ef37db82bcf39b` bindt uitvoerbare codecommit +`70fb4b94e5cb7c248beec5a936ce186f38cc183c`. De evidence-manifestbestandshash +is `fdc15a95ee2a0754dfa169f4b41036e084b8d8909afc37fd8ea68ee6b9210f98`; +de release-gatereportbestandshash is +`e5f1c9c43c8da22acfa5486e641c010a278431f90634f91335b039ab7bdb50a4`. +Deze hashes bewijzen reproduceerbaarheid en integriteit, niet productaccuracy. + +## Waarom de productbaseline niet uitvoerbaar is + +De productgates zijn bewijsgebonden en kunnen niet vanuit losse statusvelden +groen worden gemaakt: + +1. de actieve modelconfiguratie verwijst naar `/app/models/...pt`, die lokaal + niet als bestand beschikbaar is; +2. er is geen governed productmanifest met in-process herberekende raw evidence + voor alle zeven taakfamilies en een actueel CUDA/GPU-receipt; +3. een volledige GRB/PICC/UrbIS/DHMV/SPW/MDK-taak-/zoneportfolio ontbreekt; +4. een checksumgebonden reviewledger ontbreekt; historisch staat V56 op 0/180; +5. een productgebonden geometrische leakage-audit ontbreekt; historisch zijn er + 24 cross-splitparen onder 2 km; +6. de Phase-3-leakagestatus is bevestigd `attention` en dus `fail`; +7. vaultisolatie met hash-chained accesslog is niet bewezen; +8. representatieve support over alle 13 vereiste subgroupdimensies ontbreekt. + +Door de fail-first aggregatie maakt de bevestigde Phase-3-leakage de huidige +productbenchmark `fail`; ontbrekende productartefacten blijven afzonderlijk +`not_evaluable`. Geen van beide toestanden laat promotie toe. + +Daarnaast waren Tower-corpora, gemounte modelvolumes en de productie-PostGIS/API +in Phase 3 expliciet onbereikbaar. Het harnas verzint daarvoor geen vervangende +ground truth en gebruikt OSM niet automatisch als gezagsbron voor gebouwen. + +## Betekenis voor volgende training + +Test- en challenge-data mogen niet worden geopend voor training, +thresholdselectie, modelselectie of iteratieve foutcorrectie. Eerst moeten de +representatieve review, nieuwe onafhankelijke splits, lokale gezagsdata, +protected vault en een checksumbare incumbent-run beschikbaar zijn. Daarna +worden modelkeuzes op `val` en operating points op `calibration` vooraf +vastgelegd; pas vervolgens mag één immutable kandidaat de protected test en +background-test doorlopen. Challenge-labels blijven sealed. diff --git a/docs/accuracy-program/12-release-gates.md b/docs/accuracy-program/12-release-gates.md new file mode 100644 index 00000000..316b265b --- /dev/null +++ b/docs/accuracy-program/12-release-gates.md @@ -0,0 +1,109 @@ +# Fase 4 — Releasegates + +## Gateprincipe + +GeoIntel gebruikt een conjunctieve, fail-closed beslissing: iedere verplichte +gate moet groen zijn. Een hogere gemiddelde score compenseert nooit een +ernstige regressie, ontbrekende support of een critical miss in een vereiste +regio, bron, context of taak. + +Er wordt in Phase 4 geen arbitrair hoog accuracydoel verzonnen. Numerieke +releasecriteria blijven `not_frozen_without_reviewed_representative_incumbent_baseline`. +Ze worden pas vastgesteld uit taakrisico, operationele productvereisten, +datakwaliteit en een representatieve, gereviewde incumbentbaseline, en altijd +vóór protected-testtoegang. + +## Lokale harnasgates + +De volgende controles zijn in `release-gate-report.json` groen: + +| Gate | Status | Bewijs | +| --- | --- | --- | +| Alle gedeclareerde taakfamilies | pass | zeven families en 15 capabilities gedeclareerd en geobserveerd | +| Split leakage | pass | zes code-owned rollen; exacte/near-duplicate/identity/2-km-gates groen voor de fixture | +| Training firewall | pass | protected path/role/lineage en padloze of gewijzigde bytes worden geweigerd | +| Geen protected modelselectie | pass | vooraf geregistreerde config; diagnostische curves selecteren niets | +| Ruwe voorspellingen | pass | exacte referenties, pre-/postfilteroutputs, config en lineage behouden | +| Referentie-implementatie | pass | vier golden cases via de echte `QaService` | +| Subgroepcontract | pass | alle 13 dimensies, support, CI en worst-stratum zichtbaar | +| Null-semantiek | pass | ongedefinieerd is `null`; promotion blijft uit | + +Deze gates bewijzen dat de evaluator zich lokaal correct gedraagt. Ze bewijzen +niet dat het productiemodel voldoet. +Een test-only governed-contractfixture met 70 cases bewijst dat de groene route +structureel bereikbaar is en dat tampering wordt geweigerd. De gemockte runtime +en handgemaakte cases zijn uitdrukkelijk geen productbewijs. + + +## Productgates + +De volgende actuele gates blokkeren Phase 4: + +| Gate | Status | Objectief bewijs | +| --- | --- | --- | +| Actief model bereikbaar en hashbaar | not_evaluable | geconfigureerd `/app/models`-asset is lokaal afwezig | +| Volledige gezagsportfolio | not_evaluable | geen governed GRB/PICC/UrbIS/DHMV/SPW/MDK-taak-/zonebewijs | +| Representatieve menselijke review | not_evaluable | geen checksumgebonden reviewledger; historisch 0/180 | +| Ruimtelijke split-onafhankelijkheid | not_evaluable | geen productgebonden geometrische audit; historisch 24 paren <2 km | +| Phase-3 leakage opgelost | fail | actuele status `attention` | +| Fysieke protected storage isolation | not_evaluable | geen vaultbewijs met hash-chained accesslog | +| Uitgevoerde product-incumbent | not_evaluable | geen model/raw/config/CUDA-gebonden productmanifest | +| Representatieve subgroup-support | not_evaluable | geen productmetrics over alle 13 dimensies | + +Door `fail > not_evaluable > pass` is de productbenchmarkstatus `fail`. Dat is +geen gemeten accuracy failure: de bevestigde leakagegate faalt en de overige +productmetingen zijn nog niet evalueerbaar. `promotion_allowed` blijft `false`. + +## Toekomstige numerieke gatefreeze + +Een latere gatefreeze moet per taak minimaal vastleggen: + +- exacte model-, evaluator-, preprocessing-, bron- en splitmanifesthashes; +- klasse- en matchbeleid, operating threshold en NMS-configuratie; +- globale primaire metric met betrouwbaarheidsinterval; +- vereiste regio-, gemeente-, stedelijkheids-, objectgrootte-, bron-, sensor-, + resolutie-, seizoen-, datum-, vegetatie-, occlusie-, moeilijkheids- en + contextsubgroepen; +- minimale onafhankelijke AOI-/objectsupport per groep; +- non-inferioritymarge tegenover de incumbent; +- harde maxima voor kritieke false negatives, false positives, + topologiefouten en calibration error; +- abstention/coverage-regel voor onzekere gevallen; +- runtime-, GPU-, latency- en resourcecontract waar operationeel relevant. + +Een releasebeslissing is alleen positief wanneer zowel de globale gate als +iedere kritieke subgroepgate groen is. `insufficient_support`, `null`, een +ontbrekende groep, ongeldige lineage, een checksumverschil of een gemiste +blocker/critical anomaly blokkeert. + +## Beschermde evaluatievolgorde + +1. Herstel of excludeer Phase-3-quarantine-items traceerbaar. +2. Laat representatieve labels menselijk beoordelen en freeze een nieuwe + corpusversie; wijzig V56 niet in place. +3. Genereer onafhankelijke `train`, `val`, `calibration`, `test`, + `background-test` en sealed `challenge`-manifesten; laat alle leakage-gates + slagen. +4. Plaats test/background-test en externe challenge-labels in een afzonderlijke + vault met beperkte credentials en hash-chained immutable accesslog. +5. Voer de incumbent uit en freeze targets op `val`, operating points op + `calibration` en operationele vereisten vóór protected toegang. +6. Train uitsluitend op `train`; selecteer checkpoints alleen op `val`. +7. Pre-registreer één immutable kandidaat en open `test` en `background-test` + één keer; challenge-labels blijven sealed. +8. Bewaar raw predictions, herbereken paired/subgroepmetrics in-process en neem + één fail-closed releasebeslissing. +9. Gebruik protected resultaten nooit als feedback voor dezelfde kandidaat- of + trainingsfamilie. + +## Actuele fasebeslissing + +Phase 4 blijft `in_progress`: het reproduceerbare lokale harnas is klaar, maar +de echte protected productbaseline kan nog niet geldig worden uitgevoerd. +Phase 5 blijft `not_ready`. Deze status verandert pas wanneer alle productgates +in dezelfde checksumbare workflow groen zijn; documentatie of fixture-scores +kunnen die voorwaarde niet omzeilen. + +Normatieve evidence-run: `p4-2.0.1-9677d0ef37db82bcf39b`, gebonden aan +codecommit `70fb4b94e5cb7c248beec5a936ce186f38cc183c` en canonieke +benchmarkmanifesthash `0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642`. diff --git a/docs/accuracy-program/status.json b/docs/accuracy-program/status.json index d1287b77..248f1033 100644 --- a/docs/accuracy-program/status.json +++ b/docs/accuracy-program/status.json @@ -2,7 +2,7 @@ "schema_version": 1, "program": "GeoIntel Accuracy Improvement Program", "phase": "P4", - "generated_at": "2026-08-02T01:04:48+02:00", + "generated_at": "2026-08-02T04:40:51+02:00", "scope": { "product": "Belgium and the Belgian North Sea", "active_building_model_claim": "Mol/Kempen only, operator review required", @@ -104,9 +104,56 @@ ] }, "phase4": { - "status": "ready", - "meaning": "Ready to remediate and review the P3 anomaly and quarantine manifests; release gates remain governed by P2 and the execution contract.", - "evidence_root": "artifacts/evidence/accuracy/P3" + "status": "in_progress", + "meaning": "The content-addressed local evaluation harness passes, but the governed product benchmark fails on confirmed Phase-3 leakage and remains not evaluable for missing product evidence.", + "workflow": "scripts/run_accuracy_phase4_benchmark.py", + "workflow_version": "2.0.1", + "evaluator_version": "2.1.0", + "split_generator_version": "1.3.0", + "local_harness_status": "pass", + "product_benchmark_status": "fail", + "phase4_done": false, + "evidence_run_id": "p4-2.0.1-9677d0ef37db82bcf39b", + "evidence_root": "artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b", + "repository_commit": "70fb4b94e5cb7c248beec5a936ce186f38cc183c", + "benchmark_manifest_sha256": "0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642", + "benchmark_file_sha256": "868dd7eb2dfa8344e417879ed3f5dd7c7d9e5673add7e3e2294c5cbde4b75d57", + "evidence_manifest_file_sha256": "fdc15a95ee2a0754dfa169f4b41036e084b8d8909afc37fd8ea68ee6b9210f98", + "release_gate_report_file_sha256": "e5f1c9c43c8da22acfa5486e641c010a278431f90634f91335b039ab7bdb50a4", + "product_gate_evidence_sha256": "c9f5e3de3ab6d64911f8807b26caa661ccf911a5fd6b995107edda2004836e0c", + "case_count": 9, + "task_family_count": 7, + "implemented_capability_count": 15, + "failure_example_count": 14, + "split_counts": { + "background-test": 2, + "calibration": 2, + "challenge": 4, + "test": 7, + "train": 3, + "val": 3 + }, + "blockers": [ + "configured active model bytes are not locally accessible", + "no governed seven-task product baseline manifest with current CUDA receipt exists", + "Phase-3 leakage status is attention and therefore a failing product gate", + "no checksum-bound representative human review ledger exists", + "no governed zero-under-2-km product split audit exists", + "no protected vault evidence with hash-chained access log exists", + "no complete task-zone authority portfolio exists", + "no representative support over all thirteen subgroup dimensions exists" + ], + "does_not_mean": [ + "Phase 4 is complete", + "production accuracy is measured", + "Phase 5 is ready", + "training or promotion is allowed" + ] + }, + "phase5": { + "status": "not_ready", + "meaning": "Phase 5 remains blocked until every governed Phase-4 product gate passes in the same checksum-bound workflow.", + "blocked_by": "phase4" }, "runtime": { "cuda_available": true, @@ -276,6 +323,35 @@ "static_inventory": "artifacts/evidence/accuracy/P2/source-contract-inventory.json", "claim_boundary": "This is not a production migration deployment, corpus-release, accuracy or promotion result." }, + "phase4_evaluation": { + "status": "local_pass_product_fail", + "targeted_tests": { + "passed": 60, + "duration_seconds": 22.28 + }, + "relevant_regression_tests": { + "passed": 103, + "duration_seconds": 27.61, + "shell_provider": "C:/Program Files/Git/bin/bash.exe", + "note": "The Windows Store WSL bash stub was unavailable; the same shell syntax test passed with the installed Git Bash provider." + }, + "ruff": "pass", + "format_check": "pass", + "compileall": "pass", + "python_typecheck": "not_configured", + "alembic_head": "202608010001", + "migration_and_provenance_tests": { + "passed": 29, + "duration_seconds": 5.73 + }, + "workflow_reproduction": { + "allow_product_blocked_exit": 0, + "default_exit": 2, + "byte_identical": true, + "status_bookkeeping_stable": true + }, + "claim_boundary": "Local harness and contract gates pass. Product accuracy is not measured; Phase-3 leakage fails and missing governed product evidence remains not evaluable." + }, "golden_qa": { "semantic_results_stable": true, "byte_identical": false, @@ -311,12 +387,21 @@ "docs/accuracy-program/06-implementation-roadmap.md", "docs/accuracy-program/07-source-authority-matrix.md", "docs/accuracy-program/08-data-contracts.md", - "docs/accuracy-program/09-full-data-scan.md" + "docs/accuracy-program/09-full-data-scan.md", + "docs/accuracy-program/10-evaluation-protocol.md", + "docs/accuracy-program/11-baseline-benchmark.md", + "docs/accuracy-program/12-release-gates.md" ], "evidence_root": "artifacts/evidence/accuracy/P1", "evidence_manifest": "artifacts/evidence/accuracy/P1/evidence-manifest.json", "phase2_evidence_root": "artifacts/evidence/accuracy/P2", "phase2_evidence_manifest": "artifacts/evidence/accuracy/P2/evidence-manifest.json", "phase3_evidence_root": "artifacts/evidence/accuracy/P3", - "phase3_manifest": "artifacts/evidence/accuracy/P3/full-scan-manifest.json" + "phase3_manifest": "artifacts/evidence/accuracy/P3/full-scan-manifest.json", + "phase4_evidence_root": "artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b", + "phase4_evidence_manifest": "artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/evidence-manifest.json", + "phase4_benchmark_manifest": "artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/benchmark-manifest.json", + "phase4_release_gate_report": "artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/release-gate-report.json", + "phase4_metric_report": "artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/metric-report.json", + "phase4_failure_gallery": "artifacts/evidence/accuracy/P4/runs/p4-2.0.1-9677d0ef37db82bcf39b/failure-gallery.json" }