diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration-summary.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration-summary.json new file mode 100644 index 00000000..46d247e8 --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration-summary.json @@ -0,0 +1,61 @@ +{ + "schema_version": 1, + "run_id": "building-be-v72-fresh-calibration-r1", + "status": "completed_diagnostic_no_production_change", + "claim_boundary": "Fresh non-protected calibration diagnostics only. Wallonia and Brussels remain evaluation-only while regional authority harmonisation contracts are pending; no protected-test, release or national accuracy claim.", + "portfolio": { + "region_count": 3, + "aoi_count": 6, + "tile_count": 24, + "label_count": 348, + "tile_size": 512, + "stride": 512, + "overlapping_edge_cover_tiles_excluded": 30, + "training_allowed": false, + "release_claim_allowed": false, + "minimum_model_lineage_distance_m": 4869.625822725313, + "model_lineage_sample_count": 31, + "model_lineage_spatial_status": "independent" + }, + "visual_review": { + "reviewer_type": "ai_assisted", + "human": false, + "reviewed_tile_count": 24, + "finding": "All retained tiles contain real imagery. Havelange is a positive village context despite its pre-registered background-candidate name; Assenede and Laeken are sparse/hard-negative contexts, not pure-empty backgrounds. No pure-background claim was made." + }, + "gpu": "NVIDIA GeForce RTX 4080 SUPER", + "models": { + "active": { + "sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1", + "precision": 0.1576550586330707, + "recall": 0.11781609195402298, + "map50": 0.05677865740374553, + "map50_95": 0.02175035531261916, + "f1_at_0_25_iou_0_25": 0.2315270935960591 + }, + "challenger": { + "sha256": "038f1f97a6afd534f29e1f392a730a58207b928ca01e31ab8d8fed6106705820", + "precision": 0.13343411909876413, + "recall": 0.1235632183908046, + "map50": 0.04470477656341424, + "map50_95": 0.01600413321513577, + "f1_at_0_25_iou_0_25": 0.23588456712672523 + } + }, + "threshold_decision": { + "best_aggregate_active_f1_threshold": 0.3, + "best_aggregate_active_f1": 0.25035561877667145, + "production_threshold_changed": false, + "reason": "The aggregate optimum reduces the already weak Flemish subgroup F1, and regional authority/human-review gates remain closed." + }, + "evidence_sha256": { + "operator_samples_manifest": "66bfaacaee11806ed035d3ac5725f7b3967dfc82e7a12ecf603dc3836bc26ad7", + "yolo_tile_dataset_summary": "78e16f46fa2a313a147404a19f2ce99417d2cb384b5db5f6ba33e934ff99c631", + "spatial_independence": "4971887d0f2356bfba674b0f0a4606530920b30d458765a1fec5b3086ac47e72", + "contact_sheet": "f7734c5a9db8837adeda7f8e0f1a2942148330abfcb588d41dc245017ec2435a", + "checkpoint_matrix": "ad178fac5968c7e1d6b1c79b192624e4cd2956f054d5cf70270275a0be0b80c5", + "active_stratified": "eefa9bab20674c30427f02ca19571f869f728eccf2dca376acf02ae4ef0e7678", + "challenger_stratified": "9c22f6cdc30555616f2d1b595fdfde544474be396f2bd86ff22f684d3ddd6e58", + "active_threshold_extension": "7752f2e2bd84306f3863fb32eb141b30d473581c9ef0dbb5a6970a1d3c73a0ec" + } +} diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/active-stratified.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/active-stratified.json new file mode 100644 index 00000000..dad8799a --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/active-stratified.json @@ -0,0 +1,468 @@ +{ + "schema_version": 1, + "model": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt", + "summary": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/yolo_tile_dataset_summary.json", + "split": "val", + "match_iou": 0.25, + "test_time_augmentation": false, + "inference_imgsz": 640, + "inference_batch": 8, + "max_detections_per_tile": 1000, + "nms_iou": 0.7, + "containment_nms": 1.0, + "box_scale": 1.0, + "box_offset_x": 0.0, + "box_offset_y": 0.0, + "additional_model": null, + "regional_models": {}, + "regional_model_tile_counts": {}, + "ensemble_mode": null, + "ensemble_match_iou": null, + "proposal_classifier": null, + "proposal_classifier_threshold": null, + "proposal_crop_scale": null, + "proposal_classifier_batch": null, + "tile_count": 24, + "sweeps": [ + { + "threshold": 0.05, + "aggregate": { + "true_positive": 157, + "false_positive": 1466, + "false_negative": 191, + "precision": 0.09673444239063463, + "recall": 0.4511494252873563, + "f1": 0.15930999492643327 + }, + "regions": { + "brussels": { + "true_positive": 41, + "false_positive": 181, + "false_negative": 24, + "precision": 0.18468468468468469, + "recall": 0.6307692307692307, + "f1": 0.2857142857142857 + }, + "flanders": { + "true_positive": 25, + "false_positive": 365, + "false_negative": 98, + "precision": 0.0641025641025641, + "recall": 0.2032520325203252, + "f1": 0.09746588693957114 + }, + "wallonia": { + "true_positive": 91, + "false_positive": 920, + "false_negative": 69, + "precision": 0.09000989119683482, + "recall": 0.56875, + "f1": 0.15542271562766868 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 23, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 39, + "false_positive": 162, + "false_negative": 23, + "precision": 0.19402985074626866, + "recall": 0.6290322580645161, + "f1": 0.2965779467680608 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 75, + "false_positive": 366, + "false_negative": 59, + "precision": 0.17006802721088435, + "recall": 0.5597014925373134, + "f1": 0.2608695652173913 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 2, + "false_positive": 19, + "false_negative": 1, + "precision": 0.09523809523809523, + "recall": 0.6666666666666666, + "f1": 0.16666666666666666 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 16, + "false_positive": 554, + "false_negative": 10, + "precision": 0.028070175438596492, + "recall": 0.6153846153846154, + "f1": 0.05369127516778524 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 25, + "false_positive": 342, + "false_negative": 94, + "precision": 0.0681198910081744, + "recall": 0.21008403361344538, + "f1": 0.10288065843621398 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.1, + "aggregate": { + "true_positive": 139, + "false_positive": 930, + "false_negative": 209, + "precision": 0.13002806361085126, + "recall": 0.3994252873563218, + "f1": 0.19618913196894847 + }, + "regions": { + "brussels": { + "true_positive": 36, + "false_positive": 117, + "false_negative": 29, + "precision": 0.23529411764705882, + "recall": 0.5538461538461539, + "f1": 0.33027522935779813 + }, + "flanders": { + "true_positive": 20, + "false_positive": 215, + "false_negative": 103, + "precision": 0.0851063829787234, + "recall": 0.16260162601626016, + "f1": 0.11173184357541897 + }, + "wallonia": { + "true_positive": 83, + "false_positive": 598, + "false_negative": 77, + "precision": 0.12187958883994127, + "recall": 0.51875, + "f1": 0.19738406658739593 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 13, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 34, + "false_positive": 105, + "false_negative": 28, + "precision": 0.2446043165467626, + "recall": 0.5483870967741935, + "f1": 0.3383084577114428 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 68, + "false_positive": 235, + "false_negative": 66, + "precision": 0.22442244224422442, + "recall": 0.5074626865671642, + "f1": 0.311212814645309 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 2, + "false_positive": 12, + "false_negative": 1, + "precision": 0.14285714285714285, + "recall": 0.6666666666666666, + "f1": 0.23529411764705882 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 15, + "false_positive": 363, + "false_negative": 11, + "precision": 0.03968253968253968, + "recall": 0.5769230769230769, + "f1": 0.07425742574257425 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 20, + "false_positive": 202, + "false_negative": 99, + "precision": 0.09009009009009009, + "recall": 0.16806722689075632, + "f1": 0.11730205278592373 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.15, + "aggregate": { + "true_positive": 121, + "false_positive": 675, + "false_negative": 227, + "precision": 0.15201005025125627, + "recall": 0.34770114942528735, + "f1": 0.2115384615384615 + }, + "regions": { + "brussels": { + "true_positive": 29, + "false_positive": 82, + "false_negative": 36, + "precision": 0.26126126126126126, + "recall": 0.4461538461538462, + "f1": 0.3295454545454546 + }, + "flanders": { + "true_positive": 17, + "false_positive": 165, + "false_negative": 106, + "precision": 0.09340659340659341, + "recall": 0.13821138211382114, + "f1": 0.11147540983606558 + }, + "wallonia": { + "true_positive": 75, + "false_positive": 428, + "false_negative": 85, + "precision": 0.14910536779324055, + "recall": 0.46875, + "f1": 0.22624434389140272 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 8, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 28, + "false_positive": 73, + "false_negative": 34, + "precision": 0.27722772277227725, + "recall": 0.45161290322580644, + "f1": 0.3435582822085889 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 62, + "false_positive": 175, + "false_negative": 72, + "precision": 0.2616033755274262, + "recall": 0.4626865671641791, + "f1": 0.33423180592991913 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 9, + "false_negative": 2, + "precision": 0.1, + "recall": 0.3333333333333333, + "f1": 0.15384615384615383 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 13, + "false_positive": 253, + "false_negative": 13, + "precision": 0.04887218045112782, + "recall": 0.5, + "f1": 0.08904109589041095 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 17, + "false_positive": 157, + "false_negative": 102, + "precision": 0.09770114942528736, + "recall": 0.14285714285714285, + "f1": 0.11604095563139932 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.2, + "aggregate": { + "true_positive": 104, + "false_positive": 515, + "false_negative": 244, + "precision": 0.1680129240710824, + "recall": 0.2988505747126437, + "f1": 0.21509824198552224 + }, + "regions": { + "brussels": { + "true_positive": 26, + "false_positive": 65, + "false_negative": 39, + "precision": 0.2857142857142857, + "recall": 0.4, + "f1": 0.3333333333333333 + }, + "flanders": { + "true_positive": 13, + "false_positive": 128, + "false_negative": 110, + "precision": 0.09219858156028368, + "recall": 0.10569105691056911, + "f1": 0.09848484848484848 + }, + "wallonia": { + "true_positive": 65, + "false_positive": 322, + "false_negative": 95, + "precision": 0.16795865633074936, + "recall": 0.40625, + "f1": 0.23765996343692872 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 7, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 25, + "false_positive": 59, + "false_negative": 37, + "precision": 0.2976190476190476, + "recall": 0.4032258064516129, + "f1": 0.3424657534246575 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 55, + "false_positive": 129, + "false_negative": 79, + "precision": 0.29891304347826086, + "recall": 0.41044776119402987, + "f1": 0.34591194968553457 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 6, + "false_negative": 2, + "precision": 0.14285714285714285, + "recall": 0.3333333333333333, + "f1": 0.2 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 10, + "false_positive": 193, + "false_negative": 16, + "precision": 0.04926108374384237, + "recall": 0.38461538461538464, + "f1": 0.08733624454148471 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 13, + "false_positive": 121, + "false_negative": 106, + "precision": 0.09701492537313433, + "recall": 0.1092436974789916, + "f1": 0.10276679841897234 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.25, + "aggregate": { + "true_positive": 94, + "false_positive": 370, + "false_negative": 254, + "precision": 0.2025862068965517, + "recall": 0.27011494252873564, + "f1": 0.2315270935960591 + }, + "regions": { + "brussels": { + "true_positive": 25, + "false_positive": 45, + "false_negative": 40, + "precision": 0.35714285714285715, + "recall": 0.38461538461538464, + "f1": 0.3703703703703704 + }, + "flanders": { + "true_positive": 12, + "false_positive": 103, + "false_negative": 111, + "precision": 0.10434782608695652, + "recall": 0.0975609756097561, + "f1": 0.1008403361344538 + }, + "wallonia": { + "true_positive": 57, + "false_positive": 222, + "false_negative": 103, + "precision": 0.20430107526881722, + "recall": 0.35625, + "f1": 0.25968109339407747 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 6, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 24, + "false_positive": 42, + "false_negative": 38, + "precision": 0.36363636363636365, + "recall": 0.3870967741935484, + "f1": 0.37500000000000006 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 47, + "false_positive": 91, + "false_negative": 87, + "precision": 0.34057971014492755, + "recall": 0.35074626865671643, + "f1": 0.3455882352941177 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 3, + "false_negative": 2, + "precision": 0.25, + "recall": 0.3333333333333333, + "f1": 0.28571428571428575 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 10, + "false_positive": 131, + "false_negative": 16, + "precision": 0.07092198581560284, + "recall": 0.38461538461538464, + "f1": 0.11976047904191617 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 12, + "false_positive": 97, + "false_negative": 107, + "precision": 0.11009174311926606, + "recall": 0.10084033613445378, + "f1": 0.10526315789473684 + } + }, + "pure_empty_false_positives": 0 + } + ] +} \ No newline at end of file diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/active-threshold-extension.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/active-threshold-extension.json new file mode 100644 index 00000000..e40e3b16 --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/active-threshold-extension.json @@ -0,0 +1,732 @@ +{ + "schema_version": 1, + "model": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt", + "summary": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/yolo_tile_dataset_summary.json", + "split": "val", + "match_iou": 0.25, + "test_time_augmentation": false, + "inference_imgsz": 640, + "inference_batch": 8, + "max_detections_per_tile": 1000, + "nms_iou": 0.7, + "containment_nms": 1.0, + "box_scale": 1.0, + "box_offset_x": 0.0, + "box_offset_y": 0.0, + "additional_model": null, + "regional_models": {}, + "regional_model_tile_counts": {}, + "ensemble_mode": null, + "ensemble_match_iou": null, + "proposal_classifier": null, + "proposal_classifier_threshold": null, + "proposal_crop_scale": null, + "proposal_classifier_batch": null, + "tile_count": 24, + "sweeps": [ + { + "threshold": 0.25, + "aggregate": { + "true_positive": 94, + "false_positive": 370, + "false_negative": 254, + "precision": 0.2025862068965517, + "recall": 0.27011494252873564, + "f1": 0.2315270935960591 + }, + "regions": { + "brussels": { + "true_positive": 25, + "false_positive": 45, + "false_negative": 40, + "precision": 0.35714285714285715, + "recall": 0.38461538461538464, + "f1": 0.3703703703703704 + }, + "flanders": { + "true_positive": 12, + "false_positive": 103, + "false_negative": 111, + "precision": 0.10434782608695652, + "recall": 0.0975609756097561, + "f1": 0.1008403361344538 + }, + "wallonia": { + "true_positive": 57, + "false_positive": 222, + "false_negative": 103, + "precision": 0.20430107526881722, + "recall": 0.35625, + "f1": 0.25968109339407747 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 6, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 24, + "false_positive": 42, + "false_negative": 38, + "precision": 0.36363636363636365, + "recall": 0.3870967741935484, + "f1": 0.37500000000000006 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 47, + "false_positive": 91, + "false_negative": 87, + "precision": 0.34057971014492755, + "recall": 0.35074626865671643, + "f1": 0.3455882352941177 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 3, + "false_negative": 2, + "precision": 0.25, + "recall": 0.3333333333333333, + "f1": 0.28571428571428575 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 10, + "false_positive": 131, + "false_negative": 16, + "precision": 0.07092198581560284, + "recall": 0.38461538461538464, + "f1": 0.11976047904191617 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 12, + "false_positive": 97, + "false_negative": 107, + "precision": 0.11009174311926606, + "recall": 0.10084033613445378, + "f1": 0.10526315789473684 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.3, + "aggregate": { + "true_positive": 88, + "false_positive": 267, + "false_negative": 260, + "precision": 0.24788732394366197, + "recall": 0.25287356321839083, + "f1": 0.25035561877667145 + }, + "regions": { + "brussels": { + "true_positive": 24, + "false_positive": 34, + "false_negative": 41, + "precision": 0.41379310344827586, + "recall": 0.36923076923076925, + "f1": 0.39024390243902446 + }, + "flanders": { + "true_positive": 10, + "false_positive": 84, + "false_negative": 113, + "precision": 0.10638297872340426, + "recall": 0.08130081300813008, + "f1": 0.09216589861751154 + }, + "wallonia": { + "true_positive": 54, + "false_positive": 149, + "false_negative": 106, + "precision": 0.2660098522167488, + "recall": 0.3375, + "f1": 0.2975206611570248 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 6, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 23, + "false_positive": 31, + "false_negative": 39, + "precision": 0.42592592592592593, + "recall": 0.3709677419354839, + "f1": 0.39655172413793105 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 44, + "false_positive": 66, + "false_negative": 90, + "precision": 0.4, + "recall": 0.3283582089552239, + "f1": 0.36065573770491804 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 3, + "false_negative": 2, + "precision": 0.25, + "recall": 0.3333333333333333, + "f1": 0.28571428571428575 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 10, + "false_positive": 83, + "false_negative": 16, + "precision": 0.10752688172043011, + "recall": 0.38461538461538464, + "f1": 0.1680672268907563 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 10, + "false_positive": 78, + "false_negative": 109, + "precision": 0.11363636363636363, + "recall": 0.08403361344537816, + "f1": 0.09661835748792272 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.35, + "aggregate": { + "true_positive": 72, + "false_positive": 197, + "false_negative": 276, + "precision": 0.26765799256505574, + "recall": 0.20689655172413793, + "f1": 0.233387358184765 + }, + "regions": { + "brussels": { + "true_positive": 20, + "false_positive": 24, + "false_negative": 45, + "precision": 0.45454545454545453, + "recall": 0.3076923076923077, + "f1": 0.3669724770642202 + }, + "flanders": { + "true_positive": 4, + "false_positive": 66, + "false_negative": 119, + "precision": 0.05714285714285714, + "recall": 0.032520325203252036, + "f1": 0.04145077720207254 + }, + "wallonia": { + "true_positive": 48, + "false_positive": 107, + "false_negative": 112, + "precision": 0.3096774193548387, + "recall": 0.3, + "f1": 0.3047619047619048 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 6, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 19, + "false_positive": 21, + "false_negative": 43, + "precision": 0.475, + "recall": 0.3064516129032258, + "f1": 0.37254901960784315 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 38, + "false_positive": 49, + "false_negative": 96, + "precision": 0.4367816091954023, + "recall": 0.2835820895522388, + "f1": 0.34389140271493207 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 3, + "false_negative": 2, + "precision": 0.25, + "recall": 0.3333333333333333, + "f1": 0.28571428571428575 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 10, + "false_positive": 58, + "false_negative": 16, + "precision": 0.14705882352941177, + "recall": 0.38461538461538464, + "f1": 0.21276595744680854 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 4, + "false_positive": 60, + "false_negative": 115, + "precision": 0.0625, + "recall": 0.03361344537815126, + "f1": 0.04371584699453552 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.4, + "aggregate": { + "true_positive": 62, + "false_positive": 133, + "false_negative": 286, + "precision": 0.31794871794871793, + "recall": 0.1781609195402299, + "f1": 0.2283609576427256 + }, + "regions": { + "brussels": { + "true_positive": 19, + "false_positive": 14, + "false_negative": 46, + "precision": 0.5757575757575758, + "recall": 0.2923076923076923, + "f1": 0.3877551020408164 + }, + "flanders": { + "true_positive": 4, + "false_positive": 49, + "false_negative": 119, + "precision": 0.07547169811320754, + "recall": 0.032520325203252036, + "f1": 0.045454545454545456 + }, + "wallonia": { + "true_positive": 39, + "false_positive": 70, + "false_negative": 121, + "precision": 0.3577981651376147, + "recall": 0.24375, + "f1": 0.2899628252788104 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 5, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 18, + "false_positive": 13, + "false_negative": 44, + "precision": 0.5806451612903226, + "recall": 0.2903225806451613, + "f1": 0.3870967741935484 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 31, + "false_positive": 34, + "false_negative": 103, + "precision": 0.47692307692307695, + "recall": 0.23134328358208955, + "f1": 0.3115577889447237 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 1, + "false_negative": 2, + "precision": 0.5, + "recall": 0.3333333333333333, + "f1": 0.4 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 8, + "false_positive": 36, + "false_negative": 18, + "precision": 0.18181818181818182, + "recall": 0.3076923076923077, + "f1": 0.2285714285714286 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 4, + "false_positive": 44, + "false_negative": 115, + "precision": 0.08333333333333333, + "recall": 0.03361344537815126, + "f1": 0.04790419161676646 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.45, + "aggregate": { + "true_positive": 42, + "false_positive": 87, + "false_negative": 306, + "precision": 0.32558139534883723, + "recall": 0.1206896551724138, + "f1": 0.17610062893081763 + }, + "regions": { + "brussels": { + "true_positive": 14, + "false_positive": 10, + "false_negative": 51, + "precision": 0.5833333333333334, + "recall": 0.2153846153846154, + "f1": 0.31460674157303375 + }, + "flanders": { + "true_positive": 3, + "false_positive": 32, + "false_negative": 120, + "precision": 0.08571428571428572, + "recall": 0.024390243902439025, + "f1": 0.0379746835443038 + }, + "wallonia": { + "true_positive": 25, + "false_positive": 45, + "false_negative": 135, + "precision": 0.35714285714285715, + "recall": 0.15625, + "f1": 0.21739130434782608 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 3, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 13, + "false_positive": 10, + "false_negative": 49, + "precision": 0.5652173913043478, + "recall": 0.20967741935483872, + "f1": 0.30588235294117644 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 21, + "false_positive": 23, + "false_negative": 113, + "precision": 0.4772727272727273, + "recall": 0.15671641791044777, + "f1": 0.2359550561797753 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 0, + "false_negative": 2, + "precision": 1.0, + "recall": 0.3333333333333333, + "f1": 0.5 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 4, + "false_positive": 22, + "false_negative": 22, + "precision": 0.15384615384615385, + "recall": 0.15384615384615385, + "f1": 0.15384615384615385 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 3, + "false_positive": 29, + "false_negative": 116, + "precision": 0.09375, + "recall": 0.025210084033613446, + "f1": 0.03973509933774835 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.5, + "aggregate": { + "true_positive": 34, + "false_positive": 61, + "false_negative": 314, + "precision": 0.35789473684210527, + "recall": 0.09770114942528736, + "f1": 0.15349887133182846 + }, + "regions": { + "brussels": { + "true_positive": 10, + "false_positive": 7, + "false_negative": 55, + "precision": 0.5882352941176471, + "recall": 0.15384615384615385, + "f1": 0.24390243902439027 + }, + "flanders": { + "true_positive": 3, + "false_positive": 22, + "false_negative": 120, + "precision": 0.12, + "recall": 0.024390243902439025, + "f1": 0.04054054054054054 + }, + "wallonia": { + "true_positive": 21, + "false_positive": 32, + "false_negative": 139, + "precision": 0.39622641509433965, + "recall": 0.13125, + "f1": 0.1971830985915493 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 2, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 9, + "false_positive": 7, + "false_negative": 53, + "precision": 0.5625, + "recall": 0.14516129032258066, + "f1": 0.2307692307692308 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 18, + "false_positive": 18, + "false_negative": 116, + "precision": 0.5, + "recall": 0.13432835820895522, + "f1": 0.21176470588235297 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 0, + "false_negative": 2, + "precision": 1.0, + "recall": 0.3333333333333333, + "f1": 0.5 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 3, + "false_positive": 14, + "false_negative": 23, + "precision": 0.17647058823529413, + "recall": 0.11538461538461539, + "f1": 0.13953488372093026 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 3, + "false_positive": 20, + "false_negative": 116, + "precision": 0.13043478260869565, + "recall": 0.025210084033613446, + "f1": 0.04225352112676057 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.6, + "aggregate": { + "true_positive": 24, + "false_positive": 21, + "false_negative": 324, + "precision": 0.5333333333333333, + "recall": 0.06896551724137931, + "f1": 0.12213740458015267 + }, + "regions": { + "brussels": { + "true_positive": 6, + "false_positive": 1, + "false_negative": 59, + "precision": 0.8571428571428571, + "recall": 0.09230769230769231, + "f1": 0.16666666666666666 + }, + "flanders": { + "true_positive": 2, + "false_positive": 8, + "false_negative": 121, + "precision": 0.2, + "recall": 0.016260162601626018, + "f1": 0.030075187969924814 + }, + "wallonia": { + "true_positive": 16, + "false_positive": 12, + "false_negative": 144, + "precision": 0.5714285714285714, + "recall": 0.1, + "f1": 0.1702127659574468 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 1, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 6, + "false_positive": 1, + "false_negative": 56, + "precision": 0.8571428571428571, + "recall": 0.0967741935483871, + "f1": 0.17391304347826084 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 13, + "false_positive": 8, + "false_negative": 121, + "precision": 0.6190476190476191, + "recall": 0.09701492537313433, + "f1": 0.16774193548387098 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 0, + "false_positive": 0, + "false_negative": 3, + "precision": 1.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 3, + "false_positive": 4, + "false_negative": 23, + "precision": 0.42857142857142855, + "recall": 0.11538461538461539, + "f1": 0.18181818181818182 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 2, + "false_positive": 7, + "false_negative": 117, + "precision": 0.2222222222222222, + "recall": 0.01680672268907563, + "f1": 0.03125 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.7, + "aggregate": { + "true_positive": 7, + "false_positive": 8, + "false_negative": 341, + "precision": 0.4666666666666667, + "recall": 0.020114942528735632, + "f1": 0.03856749311294766 + }, + "regions": { + "brussels": { + "true_positive": 3, + "false_positive": 0, + "false_negative": 62, + "precision": 1.0, + "recall": 0.046153846153846156, + "f1": 0.08823529411764706 + }, + "flanders": { + "true_positive": 1, + "false_positive": 3, + "false_negative": 122, + "precision": 0.25, + "recall": 0.008130081300813009, + "f1": 0.015748031496062995 + }, + "wallonia": { + "true_positive": 3, + "false_positive": 5, + "false_negative": 157, + "precision": 0.375, + "recall": 0.01875, + "f1": 0.03571428571428571 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 1, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 3, + "false_positive": 0, + "false_negative": 59, + "precision": 1.0, + "recall": 0.04838709677419355, + "f1": 0.09230769230769231 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 3, + "false_positive": 4, + "false_negative": 131, + "precision": 0.42857142857142855, + "recall": 0.022388059701492536, + "f1": 0.0425531914893617 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 0, + "false_positive": 0, + "false_negative": 3, + "precision": 1.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 0, + "false_positive": 1, + "false_negative": 26, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 1, + "false_positive": 2, + "false_negative": 118, + "precision": 0.3333333333333333, + "recall": 0.008403361344537815, + "f1": 0.016393442622950817 + } + }, + "pure_empty_false_positives": 0 + } + ] +} \ No newline at end of file diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/challenger-stratified.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/challenger-stratified.json new file mode 100644 index 00000000..8a902696 --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/challenger-stratified.json @@ -0,0 +1,468 @@ +{ + "schema_version": 1, + "model": "/app/models/geointel-building-yolov8s-reviewedexp6-minpx3-img640-ft20.pt", + "summary": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/yolo_tile_dataset_summary.json", + "split": "val", + "match_iou": 0.25, + "test_time_augmentation": false, + "inference_imgsz": 640, + "inference_batch": 8, + "max_detections_per_tile": 1000, + "nms_iou": 0.7, + "containment_nms": 1.0, + "box_scale": 1.0, + "box_offset_x": 0.0, + "box_offset_y": 0.0, + "additional_model": null, + "regional_models": {}, + "regional_model_tile_counts": {}, + "ensemble_mode": null, + "ensemble_match_iou": null, + "proposal_classifier": null, + "proposal_classifier_threshold": null, + "proposal_crop_scale": null, + "proposal_classifier_batch": null, + "tile_count": 24, + "sweeps": [ + { + "threshold": 0.05, + "aggregate": { + "true_positive": 138, + "false_positive": 1385, + "false_negative": 210, + "precision": 0.09061063690085358, + "recall": 0.39655172413793105, + "f1": 0.14751469802244788 + }, + "regions": { + "brussels": { + "true_positive": 40, + "false_positive": 151, + "false_negative": 25, + "precision": 0.2094240837696335, + "recall": 0.6153846153846154, + "f1": 0.3125 + }, + "flanders": { + "true_positive": 26, + "false_positive": 342, + "false_negative": 97, + "precision": 0.07065217391304347, + "recall": 0.21138211382113822, + "f1": 0.10590631364562117 + }, + "wallonia": { + "true_positive": 72, + "false_positive": 892, + "false_negative": 88, + "precision": 0.07468879668049792, + "recall": 0.45, + "f1": 0.1281138790035587 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 41, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 39, + "false_positive": 139, + "false_negative": 23, + "precision": 0.21910112359550563, + "recall": 0.6290322580645161, + "f1": 0.32500000000000007 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 60, + "false_positive": 351, + "false_negative": 74, + "precision": 0.145985401459854, + "recall": 0.44776119402985076, + "f1": 0.2201834862385321 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 12, + "false_negative": 2, + "precision": 0.07692307692307693, + "recall": 0.3333333333333333, + "f1": 0.125 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 12, + "false_positive": 541, + "false_negative": 14, + "precision": 0.0216998191681736, + "recall": 0.46153846153846156, + "f1": 0.04145077720207254 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 26, + "false_positive": 301, + "false_negative": 93, + "precision": 0.07951070336391437, + "recall": 0.2184873949579832, + "f1": 0.11659192825112107 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.1, + "aggregate": { + "true_positive": 118, + "false_positive": 906, + "false_negative": 230, + "precision": 0.115234375, + "recall": 0.3390804597701149, + "f1": 0.17201166180758018 + }, + "regions": { + "brussels": { + "true_positive": 34, + "false_positive": 103, + "false_negative": 31, + "precision": 0.24817518248175183, + "recall": 0.5230769230769231, + "f1": 0.3366336633663366 + }, + "flanders": { + "true_positive": 20, + "false_positive": 221, + "false_negative": 103, + "precision": 0.08298755186721991, + "recall": 0.16260162601626016, + "f1": 0.10989010989010989 + }, + "wallonia": { + "true_positive": 64, + "false_positive": 582, + "false_negative": 96, + "precision": 0.09907120743034056, + "recall": 0.4, + "f1": 0.1588089330024814 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 25, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 33, + "false_positive": 98, + "false_negative": 29, + "precision": 0.25190839694656486, + "recall": 0.532258064516129, + "f1": 0.34196891191709844 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 53, + "false_positive": 232, + "false_negative": 81, + "precision": 0.18596491228070175, + "recall": 0.39552238805970147, + "f1": 0.25298329355608595 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 5, + "false_negative": 2, + "precision": 0.16666666666666666, + "recall": 0.3333333333333333, + "f1": 0.2222222222222222 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 11, + "false_positive": 350, + "false_negative": 15, + "precision": 0.030470914127423823, + "recall": 0.4230769230769231, + "f1": 0.056847545219638244 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 20, + "false_positive": 196, + "false_negative": 99, + "precision": 0.09259259259259259, + "recall": 0.16806722689075632, + "f1": 0.11940298507462685 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.15, + "aggregate": { + "true_positive": 107, + "false_positive": 654, + "false_negative": 241, + "precision": 0.14060446780551905, + "recall": 0.3074712643678161, + "f1": 0.19296663660955815 + }, + "regions": { + "brussels": { + "true_positive": 30, + "false_positive": 83, + "false_negative": 35, + "precision": 0.26548672566371684, + "recall": 0.46153846153846156, + "f1": 0.33707865168539325 + }, + "flanders": { + "true_positive": 17, + "false_positive": 166, + "false_negative": 106, + "precision": 0.09289617486338798, + "recall": 0.13821138211382114, + "f1": 0.1111111111111111 + }, + "wallonia": { + "true_positive": 60, + "false_positive": 405, + "false_negative": 100, + "precision": 0.12903225806451613, + "recall": 0.375, + "f1": 0.192 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 17, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 29, + "false_positive": 79, + "false_negative": 33, + "precision": 0.26851851851851855, + "recall": 0.46774193548387094, + "f1": 0.3411764705882353 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 50, + "false_positive": 168, + "false_negative": 84, + "precision": 0.22935779816513763, + "recall": 0.373134328358209, + "f1": 0.28409090909090906 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 4, + "false_negative": 2, + "precision": 0.2, + "recall": 0.3333333333333333, + "f1": 0.25 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 10, + "false_positive": 237, + "false_negative": 16, + "precision": 0.04048582995951417, + "recall": 0.38461538461538464, + "f1": 0.07326007326007326 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 17, + "false_positive": 149, + "false_negative": 102, + "precision": 0.10240963855421686, + "recall": 0.14285714285714285, + "f1": 0.11929824561403508 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.2, + "aggregate": { + "true_positive": 100, + "false_positive": 487, + "false_negative": 248, + "precision": 0.17035775127768313, + "recall": 0.28735632183908044, + "f1": 0.213903743315508 + }, + "regions": { + "brussels": { + "true_positive": 29, + "false_positive": 63, + "false_negative": 36, + "precision": 0.31521739130434784, + "recall": 0.4461538461538462, + "f1": 0.3694267515923567 + }, + "flanders": { + "true_positive": 15, + "false_positive": 128, + "false_negative": 108, + "precision": 0.1048951048951049, + "recall": 0.12195121951219512, + "f1": 0.11278195488721804 + }, + "wallonia": { + "true_positive": 56, + "false_positive": 296, + "false_negative": 104, + "precision": 0.1590909090909091, + "recall": 0.35, + "f1": 0.21875 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 9, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 28, + "false_positive": 60, + "false_negative": 34, + "precision": 0.3181818181818182, + "recall": 0.45161290322580644, + "f1": 0.3733333333333333 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 47, + "false_positive": 125, + "false_negative": 87, + "precision": 0.27325581395348836, + "recall": 0.35074626865671643, + "f1": 0.3071895424836601 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 3, + "false_negative": 2, + "precision": 0.25, + "recall": 0.3333333333333333, + "f1": 0.28571428571428575 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 9, + "false_positive": 171, + "false_negative": 17, + "precision": 0.05, + "recall": 0.34615384615384615, + "f1": 0.08737864077669903 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 15, + "false_positive": 119, + "false_negative": 104, + "precision": 0.11194029850746269, + "recall": 0.12605042016806722, + "f1": 0.11857707509881422 + } + }, + "pure_empty_false_positives": 0 + }, + { + "threshold": 0.25, + "aggregate": { + "true_positive": 94, + "false_positive": 355, + "false_negative": 254, + "precision": 0.20935412026726058, + "recall": 0.27011494252873564, + "f1": 0.23588456712672523 + }, + "regions": { + "brussels": { + "true_positive": 29, + "false_positive": 45, + "false_negative": 36, + "precision": 0.3918918918918919, + "recall": 0.4461538461538462, + "f1": 0.4172661870503597 + }, + "flanders": { + "true_positive": 13, + "false_positive": 102, + "false_negative": 110, + "precision": 0.11304347826086956, + "recall": 0.10569105691056911, + "f1": 0.1092436974789916 + }, + "wallonia": { + "true_positive": 52, + "false_positive": 208, + "false_negative": 108, + "precision": 0.2, + "recall": 0.325, + "f1": 0.24761904761904763 + } + }, + "samples": { + "v72-assenede-field-cal-bg": { + "true_positive": 0, + "false_positive": 5, + "false_negative": 4, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0 + }, + "v72-ganshoren-lowrise-cal": { + "true_positive": 28, + "false_positive": 42, + "false_negative": 34, + "precision": 0.4, + "recall": 0.45161290322580644, + "f1": 0.4242424242424243 + }, + "v72-havelange-field-cal-bg": { + "true_positive": 43, + "false_positive": 87, + "false_negative": 91, + "precision": 0.33076923076923076, + "recall": 0.3208955223880597, + "f1": 0.32575757575757575 + }, + "v72-laeken-park-cal-bg": { + "true_positive": 1, + "false_positive": 3, + "false_negative": 2, + "precision": 0.25, + "recall": 0.3333333333333333, + "f1": 0.28571428571428575 + }, + "v72-libramont-lowrise-cal": { + "true_positive": 9, + "false_positive": 121, + "false_negative": 17, + "precision": 0.06923076923076923, + "recall": 0.34615384615384615, + "f1": 0.11538461538461539 + }, + "v72-zandhoven-lowrise-cal": { + "true_positive": 13, + "false_positive": 97, + "false_negative": 106, + "precision": 0.11818181818181818, + "recall": 0.1092436974789916, + "f1": 0.11353711790393013 + } + }, + "pure_empty_false_positives": 0 + } + ] +} \ No newline at end of file diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/checkpoint-matrix.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/checkpoint-matrix.json new file mode 100644 index 00000000..d4f81d89 --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/checkpoint-matrix.json @@ -0,0 +1,109 @@ +{ + "attempts": [ + { + "map50": 0.05677865740374553, + "map50_95": 0.02175035531261916, + "model_path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt", + "model_sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1", + "precision": 0.1576550586330707, + "recall": 0.11781609195402298, + "size_bytes": 22516074, + "status": "ok" + }, + { + "map50": 0.04470477656341424, + "map50_95": 0.01600413321513577, + "model_path": "/app/models/geointel-building-yolov8s-reviewedexp6-minpx3-img640-ft20.pt", + "model_sha256": "038f1f97a6afd534f29e1f392a730a58207b928ca01e31ab8d8fed6106705820", + "precision": 0.13343411909876413, + "recall": 0.1235632183908046, + "size_bytes": 22514794, + "status": "ok" + } + ], + "candidate_count": 2, + "claim_boundary": "Non-protected validation ranking only; no test, challenge or promotion claim.", + "dataset_overlap_evidence": { + "interpretation": "Tiles do not overlap, but spatial/statistical independence is not established because adjacent tiles share AOI context and edge objects can remain split.", + "overlap_pixels": 0, + "statistical_independence_established": false, + "status": "non_overlapping", + "stride": 512, + "summary_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/yolo_tile_dataset_summary.json", + "summary_sha256": "78e16f46fa2a313a147404a19f2ce99417d2cb384b5db5f6ba33e934ff99c631", + "tile_size": 512, + "validation_tiles_non_overlapping": true + }, + "dataset_yaml": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/dataset.yaml", + "dataset_yaml_sha256": "d1e85ffb6776d92a4534830f2214fa6444f5a9160329aca0a68d0ea0a1a40144", + "device": "cuda:0", + "generated_at": "2026-08-09T20:31:15.072077+00:00", + "model_lineage_independence_evidence": { + "evaluation_samples": [ + "v72-assenede-field-cal-bg", + "v72-ganshoren-lowrise-cal", + "v72-havelange-field-cal-bg", + "v72-laeken-park-cal-bg", + "v72-libramont-lowrise-cal", + "v72-zandhoven-lowrise-cal" + ], + "evaluation_summary_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/yolo_tile_dataset_summary.json", + "evaluation_summary_sha256": "78e16f46fa2a313a147404a19f2ce99417d2cb384b5db5f6ba33e934ff99c631", + "independent_for_all_supplied_lineage_corpora": true, + "interpretation": "No evaluation AOI occurs in any split of any supplied ancestral corpus.", + "lineage_corpora": [ + { + "exposure_roles": {}, + "lineage_sample_count": 19, + "lineage_summary_path": "/app/storage/operator-data/yolo-building-aoi1024-expanded-minpx4vis035/yolo_tile_dataset_summary.json", + "lineage_summary_sha256": "1887e2ba5c719d33675a9ed85db99a274fd66d98d5f6ad44558d673620917201", + "overlapping_evaluation_samples": [] + }, + { + "exposure_roles": {}, + "lineage_sample_count": 22, + "lineage_summary_path": "/app/storage/operator-data/yolo-building-aoi1024-smallbld-minpx3vis035/yolo_tile_dataset_summary.json", + "lineage_summary_sha256": "49b2a07d2105d08356431757b83eafc1498eaf1fb76965b1efe05b776824942a", + "overlapping_evaluation_samples": [] + }, + { + "exposure_roles": {}, + "lineage_sample_count": 28, + "lineage_summary_path": "/app/storage/operator-data/yolo-building-aoi1024-reviewedexp6-minpx3vis035/yolo_tile_dataset_summary.json", + "lineage_summary_sha256": "7c917e31216d1df2174c0f9c736f88a81f3835aa991971f8fb8665e17ddf5c9c", + "overlapping_evaluation_samples": [] + } + ], + "overlapping_evaluation_samples": [], + "status": "independent" + }, + "pure_background_confidence": 0.15, + "pure_background_prefixes": [], + "ranking": [ + { + "map50": 0.05677865740374553, + "map50_95": 0.02175035531261916, + "model_path": "/app/models/geointel-building-yolov8s-smallbld-minpx3-img640-ft30.pt", + "model_sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1", + "precision": 0.1576550586330707, + "recall": 0.11781609195402298, + "size_bytes": 22516074, + "status": "ok" + }, + { + "map50": 0.04470477656341424, + "map50_95": 0.01600413321513577, + "model_path": "/app/models/geointel-building-yolov8s-reviewedexp6-minpx3-img640-ft20.pt", + "model_sha256": "038f1f97a6afd534f29e1f392a730a58207b928ca01e31ab8d8fed6106705820", + "precision": 0.13343411909876413, + "recall": 0.1235632183908046, + "size_bytes": 22514794, + "status": "ok" + } + ], + "schema_version": 3, + "status": "ok", + "successful_candidate_count": 2, + "torch_version": "2.11.0+cu128", + "validation_image_count": 24 +} diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/contact_sheet_001.png b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/contact_sheet_001.png new file mode 100644 index 00000000..0bb03a7a Binary files /dev/null and b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/contact_sheet_001.png differ diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/corpus-NO_TRAINING.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/corpus-NO_TRAINING.json new file mode 100644 index 00000000..91edc4ca --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/corpus-NO_TRAINING.json @@ -0,0 +1,7 @@ +{ + "manifest_sha256": "66bfaacaee11806ed035d3ac5725f7b3967dfc82e7a12ecf603dc3836bc26ad7", + "reason": "evaluation_only_pending_regional_authority_contracts", + "release_claim_allowed": false, + "schema_version": 1, + "training_allowed": false +} diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/corpus-freeze.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/corpus-freeze.json new file mode 100644 index 00000000..c97032a5 --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/corpus-freeze.json @@ -0,0 +1,11 @@ +{ + "schema_version": 2, + "dataset_version": "building-be-v72-fresh-calibration-r1", + "manifest_sha256": "66bfaacaee11806ed035d3ac5725f7b3967dfc82e7a12ecf603dc3836bc26ad7", + "sample_count": 6, + "immutable": true, + "training_eligibility_policy": "geointel-training-source-eligibility/v1", + "fixture_mode": false, + "training_allowed": false, + "release_claim_allowed": false +} \ No newline at end of file diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/internal-spatial-leakage-audit.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/internal-spatial-leakage-audit.json new file mode 100644 index 00000000..8a43890f --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/internal-spatial-leakage-audit.json @@ -0,0 +1,5 @@ +{ + "status": "ok", + "buffer_m": 64.0, + "findings": [] +} \ No newline at end of file diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/operator_samples_manifest.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/operator_samples_manifest.json new file mode 100644 index 00000000..e80df6f1 --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/operator_samples_manifest.json @@ -0,0 +1,755 @@ +{ + "schema_version": 2, + "dataset_version": "building-be-v72-fresh-calibration-r1", + "immutable": true, + "training_eligibility": { + "policy_version": "geointel-training-source-eligibility/v1", + "status": "not_eligible_evaluation_only", + "fixture_mode": false + }, + "purpose": "non_protected_diagnostic_evaluation", + "samples": [ + { + "sample_slug": "v72-zandhoven-lowrise-cal", + "sample_role": "positive", + "require_empty": false, + "region": "flanders", + "context": "detached-lowrise", + "split": "calibration", + "raster_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-zandhoven-lowrise-cal/image.tif", + "reference_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-zandhoven-lowrise-cal/buildings.normalized.geojson", + "reference_source": "grb", + "reference_layer": "buildings", + "reference_feature_count": 113, + "raster_dataset_id": "0ab0ab1b-bf30-4974-a620-87da027f42a6", + "reference_dataset_id": "3bc34f62-8531-4efd-840d-2d16dcb5529a", + "raster_sha256": "d9cd1cc1ad7b00ef849c3d06874efdcdb85388c5cd1b51eb31c83ae56de8ff92", + "reference_sha256": "a86f2ac64c0687ae9d4535e0d5462104bf00aace142d9095b75bbbf31215f06e", + "label_audit_sha256": "208bb734d47ec3bff579f1089057339e52b058a85a2cdeaaed824eb9015c1448", + "training_eligibility": { + "policy_version": "geointel-training-source-eligibility/v1", + "eligible": true, + "fixture_mode": false, + "raster": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "raster", + "eligible": true, + "fixture_mode": false, + "reasons": [], + "evidence": { + "dataset_id": "0ab0ab1b-bf30-4974-a620-87da027f42a6", + "dataset_type": "raster", + "dataset_role": "source", + "source": "digitaal vlaanderen wms omwrgb25vl", + "source_name": "digitaal_vlaanderen_orthophoto", + "checksum_sha256": "d9cd1cc1ad7b00ef849c3d06874efdcdb85388c5cd1b51eb31c83ae56de8ff92", + "data_contract_key": "geointel.raster.geotiff", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "not_applicable", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "dataset_source_snapshot_id": "36a99c85-4b9f-4c03-be46-19ec8d01ca54", + "source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "source_key": "digitaal_vlaanderen_orthophoto", + "source_classification": "contextual", + "source_freshness_status": "unknown", + "source_ingest_status": "configured", + "source_training_allowed": true, + "source_ground_truth_allowed": false, + "source_allowed_tasks": [ + "imagery", + "training_input", + "visual_context" + ], + "source_validation_authority": {}, + "source_building_validation_authority": null, + "source_snapshot_id": "36a99c85-4b9f-4c03-be46-19ec8d01ca54", + "snapshot_source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "snapshot_key": "digitaal_vlaanderen_orthophoto:2025:d9cd1cc1ad7b00ef849c3d06874efdcdb85388c5cd1b51eb31c83ae56de8ff92", + "snapshot_checksum_sha256": "d9cd1cc1ad7b00ef849c3d06874efdcdb85388c5cd1b51eb31c83ae56de8ff92", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + }, + "reference": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "reference", + "eligible": true, + "fixture_mode": false, + "reasons": [], + "evidence": { + "dataset_id": "3bc34f62-8531-4efd-840d-2d16dcb5529a", + "dataset_type": "vector", + "dataset_role": "reference", + "source": "digitaal vlaanderen grb ogc api features", + "source_name": "grb", + "checksum_sha256": "f5ea5b17b7fac5c731fbb044befb63b82ce7faf54de9cccb2cea0c063c66fbbb", + "data_contract_key": "geointel.vector.geojson", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "complete", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "3cc9ab48-ac58-5271-8ea7-55d250925ec6", + "dataset_source_snapshot_id": "f88d125e-b901-4e22-b20c-913a7431df4d", + "source_registry_id": "3cc9ab48-ac58-5271-8ea7-55d250925ec6", + "source_key": "grb", + "source_classification": "authoritative", + "source_freshness_status": "unknown", + "source_ingest_status": "configured", + "source_training_allowed": true, + "source_ground_truth_allowed": true, + "source_allowed_tasks": [ + "building_validation", + "building_labels", + "reference_context" + ], + "source_validation_authority": { + "building_validation": "primary" + }, + "source_building_validation_authority": "primary", + "source_snapshot_id": "f88d125e-b901-4e22-b20c-913a7431df4d", + "snapshot_source_registry_id": "3cc9ab48-ac58-5271-8ea7-55d250925ec6", + "snapshot_key": "grb:2026-08-09:f5ea5b17b7fac5c731fbb044befb63b82ce7faf54de9cccb2cea0c063c66fbbb", + "snapshot_checksum_sha256": "f5ea5b17b7fac5c731fbb044befb63b82ce7faf54de9cccb2cea0c063c66fbbb", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + } + }, + "bbox_epsg4326": [ + 4.66116095047693, + 51.21284491062596, + 4.664839131648428, + 51.21515506560048 + ] + }, + { + "sample_slug": "v72-assenede-field-cal-bg", + "sample_role": "background_candidate", + "require_empty": false, + "region": "flanders", + "context": "farmland-hard-negative", + "split": "calibration", + "raster_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-assenede-field-cal-bg/image.tif", + "reference_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-assenede-field-cal-bg/buildings.normalized.geojson", + "reference_source": "grb", + "reference_layer": "buildings", + "reference_feature_count": 4, + "raster_dataset_id": "eeeb112f-93cf-47e7-8f2a-c3766cfb21b4", + "reference_dataset_id": "e48999d9-9dc7-413a-ab31-2cb6c28b2bee", + "raster_sha256": "1f4667fb9500ad1659b2d6cf9f82683d6ae2b17fcb4bfb156aada2b66f7cf3f9", + "reference_sha256": "e64fae95b0be38a5dee1424f931a97ffe7e8017a6fa8bdadfe70cc2b7c4b6a8e", + "label_audit_sha256": "ab8aa15039bf14144285c23273386bc18fa5c78746a7e4b5faf97292f8a4d2fe", + "training_eligibility": { + "policy_version": "geointel-training-source-eligibility/v1", + "eligible": true, + "fixture_mode": false, + "raster": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "raster", + "eligible": true, + "fixture_mode": false, + "reasons": [], + "evidence": { + "dataset_id": "eeeb112f-93cf-47e7-8f2a-c3766cfb21b4", + "dataset_type": "raster", + "dataset_role": "source", + "source": "digitaal vlaanderen wms omwrgb25vl", + "source_name": "digitaal_vlaanderen_orthophoto", + "checksum_sha256": "1f4667fb9500ad1659b2d6cf9f82683d6ae2b17fcb4bfb156aada2b66f7cf3f9", + "data_contract_key": "geointel.raster.geotiff", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "not_applicable", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "dataset_source_snapshot_id": "2d14b495-ddeb-4c44-bf3f-aed759787f30", + "source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "source_key": "digitaal_vlaanderen_orthophoto", + "source_classification": "contextual", + "source_freshness_status": "unknown", + "source_ingest_status": "configured", + "source_training_allowed": true, + "source_ground_truth_allowed": false, + "source_allowed_tasks": [ + "imagery", + "training_input", + "visual_context" + ], + "source_validation_authority": {}, + "source_building_validation_authority": null, + "source_snapshot_id": "2d14b495-ddeb-4c44-bf3f-aed759787f30", + "snapshot_source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "snapshot_key": "digitaal_vlaanderen_orthophoto:2025:1f4667fb9500ad1659b2d6cf9f82683d6ae2b17fcb4bfb156aada2b66f7cf3f9", + "snapshot_checksum_sha256": "1f4667fb9500ad1659b2d6cf9f82683d6ae2b17fcb4bfb156aada2b66f7cf3f9", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + }, + "reference": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "reference", + "eligible": true, + "fixture_mode": false, + "reasons": [], + "evidence": { + "dataset_id": "e48999d9-9dc7-413a-ab31-2cb6c28b2bee", + "dataset_type": "vector", + "dataset_role": "reference", + "source": "digitaal vlaanderen grb ogc api features", + "source_name": "grb", + "checksum_sha256": "e1f883a0b41b24c4dd07e8b4c939479b19ab14bff6f28645a842cc8ba5335e72", + "data_contract_key": "geointel.vector.geojson", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "complete", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "3cc9ab48-ac58-5271-8ea7-55d250925ec6", + "dataset_source_snapshot_id": "5e0f1e4e-46c1-4f7f-87f9-4aa76474b328", + "source_registry_id": "3cc9ab48-ac58-5271-8ea7-55d250925ec6", + "source_key": "grb", + "source_classification": "authoritative", + "source_freshness_status": "unknown", + "source_ingest_status": "configured", + "source_training_allowed": true, + "source_ground_truth_allowed": true, + "source_allowed_tasks": [ + "building_validation", + "building_labels", + "reference_context" + ], + "source_validation_authority": { + "building_validation": "primary" + }, + "source_building_validation_authority": "primary", + "source_snapshot_id": "5e0f1e4e-46c1-4f7f-87f9-4aa76474b328", + "snapshot_source_registry_id": "3cc9ab48-ac58-5271-8ea7-55d250925ec6", + "snapshot_key": "grb:2026-08-09:e1f883a0b41b24c4dd07e8b4c939479b19ab14bff6f28645a842cc8ba5335e72", + "snapshot_checksum_sha256": "e1f883a0b41b24c4dd07e8b4c939479b19ab14bff6f28645a842cc8ba5335e72", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + } + }, + "bbox_epsg4326": [ + 3.7481516695287884, + 51.24483994940647, + 3.7518482321727147, + 51.24716002681137 + ] + }, + { + "sample_slug": "v72-libramont-lowrise-cal", + "sample_role": "positive", + "require_empty": false, + "region": "wallonia", + "context": "rural-lowrise", + "split": "calibration", + "raster_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-libramont-lowrise-cal/image.tif", + "reference_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-libramont-lowrise-cal/buildings.normalized.geojson", + "reference_source": "spw_picc", + "reference_layer": "buildings", + "reference_feature_count": 26, + "raster_dataset_id": "2764ddaa-5fad-4327-bd74-c402871c8c76", + "reference_dataset_id": "78944e97-0663-4c0a-9b8f-e8c1db5edb99", + "raster_sha256": "d79151e1e5ae83fd143cd49e096af8fe8fa93c8eb0fef9733ba8ab3d41b4d243", + "reference_sha256": "8e5a0fb37bbaca3fc1697a155e7259acff1622e759080a8e36aed69cb1422d67", + "label_audit_sha256": "dea480fbbac32547dfad4e1f7ca549f9206c9fcbb83b83497c6a2a59f8e58503", + "training_eligibility": { + "policy_version": "geointel-training-source-eligibility/v1", + "eligible": false, + "fixture_mode": false, + "raster": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "raster", + "eligible": true, + "fixture_mode": false, + "reasons": [], + "evidence": { + "dataset_id": "2764ddaa-5fad-4327-bd74-c402871c8c76", + "dataset_type": "raster", + "dataset_role": "source", + "source": "spw ortho_2023_ete wms 0", + "source_name": "spw_orthophoto", + "checksum_sha256": "d79151e1e5ae83fd143cd49e096af8fe8fa93c8eb0fef9733ba8ab3d41b4d243", + "data_contract_key": "geointel.raster.geotiff", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "not_applicable", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "1304d42d-d006-5e49-8660-f0fa33e0fc31", + "dataset_source_snapshot_id": "579334bb-ec27-4d0c-ba3a-2382440e7829", + "source_registry_id": "1304d42d-d006-5e49-8660-f0fa33e0fc31", + "source_key": "spw_orthophoto", + "source_classification": "contextual", + "source_freshness_status": "unknown", + "source_ingest_status": "configured", + "source_training_allowed": true, + "source_ground_truth_allowed": false, + "source_allowed_tasks": [ + "imagery", + "training_input", + "visual_context" + ], + "source_validation_authority": {}, + "source_building_validation_authority": null, + "source_snapshot_id": "579334bb-ec27-4d0c-ba3a-2382440e7829", + "snapshot_source_registry_id": "1304d42d-d006-5e49-8660-f0fa33e0fc31", + "snapshot_key": "spw_orthophoto:wallonia_2023:d79151e1e5ae83fd143cd49e096af8fe8fa93c8eb0fef9733ba8ab3d41b4d243", + "snapshot_checksum_sha256": "d79151e1e5ae83fd143cd49e096af8fe8fa93c8eb0fef9733ba8ab3d41b4d243", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + }, + "reference": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "reference", + "eligible": false, + "fixture_mode": false, + "reasons": [ + "reference_building_validation_not_primary" + ], + "evidence": { + "dataset_id": "78944e97-0663-4c0a-9b8f-e8c1db5edb99", + "dataset_type": "vector", + "dataset_role": "reference", + "source": "spw picc arcgis rest", + "source_name": "spw_picc", + "checksum_sha256": "65264d0e9bac2ebf2330fb7f47379529d271cb6ad0a5c6e23504efd60e893b64", + "data_contract_key": "geointel.vector.geojson", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "complete", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "584cd1cb-7a95-5404-b9fc-e9d252e62b1b", + "dataset_source_snapshot_id": "0191d275-4c19-4e27-af1e-abf7e065022e", + "source_registry_id": "584cd1cb-7a95-5404-b9fc-e9d252e62b1b", + "source_key": "spw_picc", + "source_classification": "authoritative", + "source_freshness_status": "unknown", + "source_ingest_status": "registered", + "source_training_allowed": true, + "source_ground_truth_allowed": true, + "source_allowed_tasks": [ + "building_validation", + "building_labels" + ], + "source_validation_authority": { + "building_validation": "regional_primary_pending_contract" + }, + "source_building_validation_authority": "regional_primary_pending_contract", + "source_snapshot_id": "0191d275-4c19-4e27-af1e-abf7e065022e", + "snapshot_source_registry_id": "584cd1cb-7a95-5404-b9fc-e9d252e62b1b", + "snapshot_key": "spw_picc:2026-07-11:65264d0e9bac2ebf2330fb7f47379529d271cb6ad0a5c6e23504efd60e893b64", + "snapshot_checksum_sha256": "65264d0e9bac2ebf2330fb7f47379529d271cb6ad0a5c6e23504efd60e893b64", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + }, + "evaluation_only_exception": { + "allowed": true, + "reason": "regional_building_authority_contract_pending", + "training_allowed": false, + "release_claim_allowed": false + } + }, + "bbox_epsg4326": [ + 5.378193608487878, + 49.91683360963306, + 5.381806468144563, + 49.91916636935071 + ] + }, + { + "sample_slug": "v72-havelange-field-cal-bg", + "sample_role": "background_candidate", + "require_empty": false, + "region": "wallonia", + "context": "farmland-hard-negative", + "split": "calibration", + "raster_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-havelange-field-cal-bg/image.tif", + "reference_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-havelange-field-cal-bg/buildings.normalized.geojson", + "reference_source": "spw_picc", + "reference_layer": "buildings", + "reference_feature_count": 136, + "raster_dataset_id": "d76519e0-f952-4446-b640-93eb758bac49", + "reference_dataset_id": "817e9265-b38f-41e4-8ca8-a2e85d536eda", + "raster_sha256": "5f9a5aa51b6eb4c00da638a11f225f2b1f786e8a9d34a3c66f97313c1fe02c1c", + "reference_sha256": "9e79333a8f09b3edc0b02bb5a56d908f8f46c05315ef44669715b1803de5b13d", + "label_audit_sha256": "6a43366fc4906b32e2a4976a1aeafb37bb68357955f55073c7e05993cb3206b4", + "training_eligibility": { + "policy_version": "geointel-training-source-eligibility/v1", + "eligible": false, + "fixture_mode": false, + "raster": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "raster", + "eligible": true, + "fixture_mode": false, + "reasons": [], + "evidence": { + "dataset_id": "d76519e0-f952-4446-b640-93eb758bac49", + "dataset_type": "raster", + "dataset_role": "source", + "source": "spw ortho_2023_ete wms 0", + "source_name": "spw_orthophoto", + "checksum_sha256": "5f9a5aa51b6eb4c00da638a11f225f2b1f786e8a9d34a3c66f97313c1fe02c1c", + "data_contract_key": "geointel.raster.geotiff", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "not_applicable", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "1304d42d-d006-5e49-8660-f0fa33e0fc31", + "dataset_source_snapshot_id": "d3d34fa1-4e40-4fe9-974b-63b303c552f8", + "source_registry_id": "1304d42d-d006-5e49-8660-f0fa33e0fc31", + "source_key": "spw_orthophoto", + "source_classification": "contextual", + "source_freshness_status": "unknown", + "source_ingest_status": "configured", + "source_training_allowed": true, + "source_ground_truth_allowed": false, + "source_allowed_tasks": [ + "imagery", + "training_input", + "visual_context" + ], + "source_validation_authority": {}, + "source_building_validation_authority": null, + "source_snapshot_id": "d3d34fa1-4e40-4fe9-974b-63b303c552f8", + "snapshot_source_registry_id": "1304d42d-d006-5e49-8660-f0fa33e0fc31", + "snapshot_key": "spw_orthophoto:wallonia_2023:5f9a5aa51b6eb4c00da638a11f225f2b1f786e8a9d34a3c66f97313c1fe02c1c", + "snapshot_checksum_sha256": "5f9a5aa51b6eb4c00da638a11f225f2b1f786e8a9d34a3c66f97313c1fe02c1c", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + }, + "reference": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "reference", + "eligible": false, + "fixture_mode": false, + "reasons": [ + "reference_building_validation_not_primary" + ], + "evidence": { + "dataset_id": "817e9265-b38f-41e4-8ca8-a2e85d536eda", + "dataset_type": "vector", + "dataset_role": "reference", + "source": "spw picc arcgis rest", + "source_name": "spw_picc", + "checksum_sha256": "c93ff374beb3acdd8501878de99767de57475fb5441af437f36bed23e8212d25", + "data_contract_key": "geointel.vector.geojson", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "complete", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "584cd1cb-7a95-5404-b9fc-e9d252e62b1b", + "dataset_source_snapshot_id": "bb1da21a-2599-4749-a55f-1606d0f4bc9d", + "source_registry_id": "584cd1cb-7a95-5404-b9fc-e9d252e62b1b", + "source_key": "spw_picc", + "source_classification": "authoritative", + "source_freshness_status": "unknown", + "source_ingest_status": "registered", + "source_training_allowed": true, + "source_ground_truth_allowed": true, + "source_allowed_tasks": [ + "building_validation", + "building_labels" + ], + "source_validation_authority": { + "building_validation": "regional_primary_pending_contract" + }, + "source_building_validation_authority": "regional_primary_pending_contract", + "source_snapshot_id": "bb1da21a-2599-4749-a55f-1606d0f4bc9d", + "snapshot_source_registry_id": "584cd1cb-7a95-5404-b9fc-e9d252e62b1b", + "snapshot_key": "spw_picc:2026-07-11:c93ff374beb3acdd8501878de99767de57475fb5441af437f36bed23e8212d25", + "snapshot_checksum_sha256": "c93ff374beb3acdd8501878de99767de57475fb5441af437f36bed23e8212d25", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + }, + "evaluation_only_exception": { + "allowed": true, + "reason": "regional_building_authority_contract_pending", + "training_allowed": false, + "release_claim_allowed": false + } + }, + "bbox_epsg4326": [ + 5.237179060792866, + 50.38783580367982, + 5.2408210177303065, + 50.39016417441682 + ] + }, + { + "sample_slug": "v72-ganshoren-lowrise-cal", + "sample_role": "positive", + "require_empty": false, + "region": "brussels", + "context": "urban-lowrise", + "split": "calibration", + "raster_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-ganshoren-lowrise-cal/image.tif", + "reference_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-ganshoren-lowrise-cal/buildings.normalized.geojson", + "reference_source": "urbis", + "reference_layer": "buildings", + "reference_feature_count": 60, + "raster_dataset_id": "463d89c5-b9fa-4a23-96fc-91f374717c31", + "reference_dataset_id": "71678e7f-3592-41b6-ae07-5205fa3903f9", + "raster_sha256": "e02fe4d11300395c421791d0d48118dc05ea7d9f017cd7cd1251f8fa75378a89", + "reference_sha256": "41e3005909984a2a76811387789c744ec5720ab7cc8df7f9af78608ab2887372", + "label_audit_sha256": "e9da810985fa8f139e675a8e859682a64c65b69a66b63474021363af66def7c7", + "training_eligibility": { + "policy_version": "geointel-training-source-eligibility/v1", + "eligible": false, + "fixture_mode": false, + "raster": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "raster", + "eligible": true, + "fixture_mode": false, + "reasons": [], + "evidence": { + "dataset_id": "463d89c5-b9fa-4a23-96fc-91f374717c31", + "dataset_type": "raster", + "dataset_role": "source", + "source": "digitaal vlaanderen omwrgb25vl wms omwrgb25vl", + "source_name": "digitaal_vlaanderen_orthophoto", + "checksum_sha256": "e02fe4d11300395c421791d0d48118dc05ea7d9f017cd7cd1251f8fa75378a89", + "data_contract_key": "geointel.raster.geotiff", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "not_applicable", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "dataset_source_snapshot_id": "6395096a-9c46-4ed6-ae10-edc8d9e76354", + "source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "source_key": "digitaal_vlaanderen_orthophoto", + "source_classification": "contextual", + "source_freshness_status": "unknown", + "source_ingest_status": "configured", + "source_training_allowed": true, + "source_ground_truth_allowed": false, + "source_allowed_tasks": [ + "imagery", + "training_input", + "visual_context" + ], + "source_validation_authority": {}, + "source_building_validation_authority": null, + "source_snapshot_id": "6395096a-9c46-4ed6-ae10-edc8d9e76354", + "snapshot_source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "snapshot_key": "digitaal_vlaanderen_orthophoto:brussels_2025:e02fe4d11300395c421791d0d48118dc05ea7d9f017cd7cd1251f8fa75378a89", + "snapshot_checksum_sha256": "e02fe4d11300395c421791d0d48118dc05ea7d9f017cd7cd1251f8fa75378a89", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + }, + "reference": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "reference", + "eligible": false, + "fixture_mode": false, + "reasons": [ + "reference_building_validation_not_primary" + ], + "evidence": { + "dataset_id": "71678e7f-3592-41b6-ae07-5205fa3903f9", + "dataset_type": "vector", + "dataset_role": "reference", + "source": "urbis wfs", + "source_name": "urbis", + "checksum_sha256": "652020de27e18eec4f844959d707fadcb47801a4c9339307b0efd87259442c58", + "data_contract_key": "geointel.vector.geojson", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "complete", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "8fe7116e-65e5-582e-a316-4ba7de5d132b", + "dataset_source_snapshot_id": "ca69e4c6-05f4-4f17-8db1-1b2b4705a735", + "source_registry_id": "8fe7116e-65e5-582e-a316-4ba7de5d132b", + "source_key": "urbis", + "source_classification": "authoritative", + "source_freshness_status": "unknown", + "source_ingest_status": "registered", + "source_training_allowed": true, + "source_ground_truth_allowed": true, + "source_allowed_tasks": [ + "building_validation", + "building_labels" + ], + "source_validation_authority": { + "building_validation": "regional_primary_pending_contract" + }, + "source_building_validation_authority": "regional_primary_pending_contract", + "source_snapshot_id": "ca69e4c6-05f4-4f17-8db1-1b2b4705a735", + "snapshot_source_registry_id": "8fe7116e-65e5-582e-a316-4ba7de5d132b", + "snapshot_key": "urbis:2026-06-06:652020de27e18eec4f844959d707fadcb47801a4c9339307b0efd87259442c58", + "snapshot_checksum_sha256": "652020de27e18eec4f844959d707fadcb47801a4c9339307b0efd87259442c58", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + }, + "evaluation_only_exception": { + "allowed": true, + "reason": "regional_building_authority_contract_pending", + "training_allowed": false, + "release_claim_allowed": false + } + }, + "bbox_epsg4326": [ + 4.305179924172925, + 50.871848393352586, + 4.308819978598048, + 50.87415158315595 + ] + }, + { + "sample_slug": "v72-laeken-park-cal-bg", + "sample_role": "background_candidate", + "require_empty": false, + "region": "brussels", + "context": "park-hard-negative", + "split": "calibration", + "raster_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-laeken-park-cal-bg/image.tif", + "reference_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/pairs/v72-laeken-park-cal-bg/buildings.normalized.geojson", + "reference_source": "urbis", + "reference_layer": "buildings", + "reference_feature_count": 3, + "raster_dataset_id": "76e24a3a-bf72-42cf-aec0-eb8c155f4f90", + "reference_dataset_id": "14f34ae2-da4a-4483-bb2b-74df40aa47ad", + "raster_sha256": "dab741904787f194c94562ca6e289c9016b4efd0fcca790d1c6bd3fca4196d0c", + "reference_sha256": "f3e29fc9d528c5086e69be9d574dbed007f7cb9132cdc69a1f5ee4fdb238dbbe", + "label_audit_sha256": "4320d7bd6b25029a819acd8d520bee106e7c4f783a813ec8e211f4fc5848014e", + "training_eligibility": { + "policy_version": "geointel-training-source-eligibility/v1", + "eligible": false, + "fixture_mode": false, + "raster": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "raster", + "eligible": true, + "fixture_mode": false, + "reasons": [], + "evidence": { + "dataset_id": "76e24a3a-bf72-42cf-aec0-eb8c155f4f90", + "dataset_type": "raster", + "dataset_role": "source", + "source": "digitaal vlaanderen omwrgb25vl wms omwrgb25vl", + "source_name": "digitaal_vlaanderen_orthophoto", + "checksum_sha256": "dab741904787f194c94562ca6e289c9016b4efd0fcca790d1c6bd3fca4196d0c", + "data_contract_key": "geointel.raster.geotiff", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "not_applicable", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "dataset_source_snapshot_id": "1f8b68c8-b66d-4b3a-bc47-40708441194f", + "source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "source_key": "digitaal_vlaanderen_orthophoto", + "source_classification": "contextual", + "source_freshness_status": "unknown", + "source_ingest_status": "configured", + "source_training_allowed": true, + "source_ground_truth_allowed": false, + "source_allowed_tasks": [ + "imagery", + "training_input", + "visual_context" + ], + "source_validation_authority": {}, + "source_building_validation_authority": null, + "source_snapshot_id": "1f8b68c8-b66d-4b3a-bc47-40708441194f", + "snapshot_source_registry_id": "766e0d47-96d9-5e78-a584-47e924a4d6b7", + "snapshot_key": "digitaal_vlaanderen_orthophoto:brussels_2025:dab741904787f194c94562ca6e289c9016b4efd0fcca790d1c6bd3fca4196d0c", + "snapshot_checksum_sha256": "dab741904787f194c94562ca6e289c9016b4efd0fcca790d1c6bd3fca4196d0c", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + }, + "reference": { + "policy_version": "geointel-training-source-eligibility/v1", + "role": "reference", + "eligible": false, + "fixture_mode": false, + "reasons": [ + "reference_building_validation_not_primary" + ], + "evidence": { + "dataset_id": "14f34ae2-da4a-4483-bb2b-74df40aa47ad", + "dataset_type": "vector", + "dataset_role": "reference", + "source": "urbis wfs", + "source_name": "urbis", + "checksum_sha256": "37f2f5d52c7785e8e692128f480d5b580ea88ebd8ac1e1502ecb02badc93bfde", + "data_contract_key": "geointel.vector.geojson", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "complete", + "quarantine_status": "not_quarantined", + "dataset_status": "ready", + "dataset_source_registry_id": "8fe7116e-65e5-582e-a316-4ba7de5d132b", + "dataset_source_snapshot_id": "4823232c-4e56-4b8c-8b2d-59e6c59f12fd", + "source_registry_id": "8fe7116e-65e5-582e-a316-4ba7de5d132b", + "source_key": "urbis", + "source_classification": "authoritative", + "source_freshness_status": "unknown", + "source_ingest_status": "registered", + "source_training_allowed": true, + "source_ground_truth_allowed": true, + "source_allowed_tasks": [ + "building_validation", + "building_labels" + ], + "source_validation_authority": { + "building_validation": "regional_primary_pending_contract" + }, + "source_building_validation_authority": "regional_primary_pending_contract", + "source_snapshot_id": "4823232c-4e56-4b8c-8b2d-59e6c59f12fd", + "snapshot_source_registry_id": "8fe7116e-65e5-582e-a316-4ba7de5d132b", + "snapshot_key": "urbis:2026-06-06:37f2f5d52c7785e8e692128f480d5b580ea88ebd8ac1e1502ecb02badc93bfde", + "snapshot_checksum_sha256": "37f2f5d52c7785e8e692128f480d5b580ea88ebd8ac1e1502ecb02badc93bfde", + "snapshot_freshness_status": "current", + "snapshot_ingest_status": "ingested" + } + }, + "evaluation_only_exception": { + "allowed": true, + "reason": "regional_building_authority_contract_pending", + "training_allowed": false, + "release_claim_allowed": false + } + }, + "bbox_epsg4326": [ + 4.349180265292469, + 50.89084908113465, + 4.3528196373886185, + 50.893150895320936 + ] + } + ] +} \ No newline at end of file diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/provisioning-spec.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/provisioning-spec.json new file mode 100644 index 00000000..15f0c842 --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/provisioning-spec.json @@ -0,0 +1,110 @@ +{ + "schema_version": 1, + "side_m": 256.0, + "resolution_m": 0.25, + "status": "complete", + "samples": [ + { + "sample_slug": "v72-zandhoven-lowrise-cal", + "region": "flanders", + "context": "detached-lowrise", + "split": "calibration", + "sample_role": "positive", + "require_empty": false, + "bbox_epsg4326": [ + 4.66116095047693, + 51.21284491062596, + 4.664839131648428, + 51.21515506560048 + ], + "raster_dataset_id": "0ab0ab1b-bf30-4974-a620-87da027f42a6", + "reference_dataset_id": "3bc34f62-8531-4efd-840d-2d16dcb5529a", + "provider_reference_feature_count": 118 + }, + { + "sample_slug": "v72-assenede-field-cal-bg", + "region": "flanders", + "context": "farmland-hard-negative", + "split": "calibration", + "sample_role": "background_candidate", + "require_empty": false, + "bbox_epsg4326": [ + 3.7481516695287884, + 51.24483994940647, + 3.7518482321727147, + 51.24716002681137 + ], + "raster_dataset_id": "eeeb112f-93cf-47e7-8f2a-c3766cfb21b4", + "reference_dataset_id": "e48999d9-9dc7-413a-ab31-2cb6c28b2bee", + "provider_reference_feature_count": 4 + }, + { + "sample_slug": "v72-libramont-lowrise-cal", + "region": "wallonia", + "context": "rural-lowrise", + "split": "calibration", + "sample_role": "positive", + "require_empty": false, + "bbox_epsg4326": [ + 5.378193608487878, + 49.91683360963306, + 5.381806468144563, + 49.91916636935071 + ], + "raster_dataset_id": "2764ddaa-5fad-4327-bd74-c402871c8c76", + "reference_dataset_id": "78944e97-0663-4c0a-9b8f-e8c1db5edb99", + "provider_reference_feature_count": 27 + }, + { + "sample_slug": "v72-havelange-field-cal-bg", + "region": "wallonia", + "context": "farmland-hard-negative", + "split": "calibration", + "sample_role": "background_candidate", + "require_empty": false, + "bbox_epsg4326": [ + 5.237179060792866, + 50.38783580367982, + 5.2408210177303065, + 50.39016417441682 + ], + "raster_dataset_id": "d76519e0-f952-4446-b640-93eb758bac49", + "reference_dataset_id": "817e9265-b38f-41e4-8ca8-a2e85d536eda", + "provider_reference_feature_count": 137 + }, + { + "sample_slug": "v72-ganshoren-lowrise-cal", + "region": "brussels", + "context": "urban-lowrise", + "split": "calibration", + "sample_role": "positive", + "require_empty": false, + "bbox_epsg4326": [ + 4.305179924172925, + 50.871848393352586, + 4.308819978598048, + 50.87415158315595 + ], + "raster_dataset_id": "463d89c5-b9fa-4a23-96fc-91f374717c31", + "reference_dataset_id": "71678e7f-3592-41b6-ae07-5205fa3903f9", + "provider_reference_feature_count": 60 + }, + { + "sample_slug": "v72-laeken-park-cal-bg", + "region": "brussels", + "context": "park-hard-negative", + "split": "calibration", + "sample_role": "background_candidate", + "require_empty": false, + "bbox_epsg4326": [ + 4.349180265292469, + 50.89084908113465, + 4.3528196373886185, + 50.893150895320936 + ], + "raster_dataset_id": "76e24a3a-bf72-42cf-aec0-eb8c155f4f90", + "reference_dataset_id": "14f34ae2-da4a-4483-bb2b-74df40aa47ad", + "provider_reference_feature_count": 3 + } + ] +} \ No newline at end of file diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/spatial-independence.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/spatial-independence.json new file mode 100644 index 00000000..3b76aa59 --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/spatial-independence.json @@ -0,0 +1,179 @@ +{ + "evaluation_manifest_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/operator_samples_manifest.json", + "evaluation_manifest_sha256": "66bfaacaee11806ed035d3ac5725f7b3967dfc82e7a12ecf603dc3836bc26ad7", + "evaluation_samples": [ + { + "bbox_epsg4326": [ + 4.66116095047693, + 51.21284491062596, + 4.664839131648428, + 51.21515506560048 + ], + "minimum_lineage_distance_m": 4869.625822725313, + "nearest_lineage_raster_path": "/app/storage/operator-data/operator-samples-1024/grobbendonk_center_orthophoto_wms_1024.tif", + "nearest_lineage_sample_slug": "grobbendonk_center", + "passes_minimum_distance": true, + "sample_slug": "v72-zandhoven-lowrise-cal" + }, + { + "bbox_epsg4326": [ + 3.7481516695287884, + 51.24483994940647, + 3.7518482321727147, + 51.24716002681137 + ], + "minimum_lineage_distance_m": 64295.09109543035, + "nearest_lineage_raster_path": "/app/storage/operator-data/operator-samples-1024/nijlen_center_orthophoto_wms_1024.tif", + "nearest_lineage_sample_slug": "nijlen_center", + "passes_minimum_distance": true, + "sample_slug": "v72-assenede-field-cal-bg" + }, + { + "bbox_epsg4326": [ + 5.378193608487878, + 49.91683360963306, + 5.381806468144563, + 49.91916636935071 + ], + "minimum_lineage_distance_m": 131973.3804837556, + "nearest_lineage_raster_path": "/app/storage/operator-data/operator-samples-1024/laakdal_center_orthophoto_wms_1024.tif", + "nearest_lineage_sample_slug": "laakdal_center", + "passes_minimum_distance": true, + "sample_slug": "v72-libramont-lowrise-cal" + }, + { + "bbox_epsg4326": [ + 5.237179060792866, + 50.38783580367982, + 5.2408210177303065, + 50.39016417441682 + ], + "minimum_lineage_distance_m": 78675.28903338463, + "nearest_lineage_raster_path": "/app/storage/operator-data/operator-samples-1024/laakdal_center_orthophoto_wms_1024.tif", + "nearest_lineage_sample_slug": "laakdal_center", + "passes_minimum_distance": true, + "sample_slug": "v72-havelange-field-cal-bg" + }, + { + "bbox_epsg4326": [ + 4.305179924172925, + 50.871848393352586, + 4.308819978598048, + 50.87415158315595 + ], + "minimum_lineage_distance_m": 39755.9878726915, + "nearest_lineage_raster_path": "/app/storage/operator-data/operator-samples-1024/hulshout_center_orthophoto_wms_1024.tif", + "nearest_lineage_sample_slug": "hulshout_center", + "passes_minimum_distance": true, + "sample_slug": "v72-ganshoren-lowrise-cal" + }, + { + "bbox_epsg4326": [ + 4.349180265292469, + 50.89084908113465, + 4.3528196373886185, + 50.893150895320936 + ], + "minimum_lineage_distance_m": 36007.49331185159, + "nearest_lineage_raster_path": "/app/storage/operator-data/operator-samples-1024/hulshout_center_orthophoto_wms_1024.tif", + "nearest_lineage_sample_slug": "hulshout_center", + "passes_minimum_distance": true, + "sample_slug": "v72-laeken-park-cal-bg" + } + ], + "lineage_sample_count": 31, + "lineage_summaries": [ + { + "path": "/app/storage/operator-data/yolo-building-aoi1024-expanded-minpx4vis035/yolo_tile_dataset_summary.json", + "sample_slugs": [ + "balen", + "dessel_heide", + "geel", + "geel_bel", + "herentals", + "herenthout_bos", + "kasterlee_bos", + "kasterlee_center", + "lille_center", + "lommel_heide", + "meerhout_bos", + "mol", + "olen_center", + "oud_turnhout_center", + "postel_bos", + "ravels_bos", + "retie", + "turnhout", + "westerlo" + ], + "sha256": "1887e2ba5c719d33675a9ed85db99a274fd66d98d5f6ad44558d673620917201" + }, + { + "path": "/app/storage/operator-data/yolo-building-aoi1024-smallbld-minpx3vis035/yolo_tile_dataset_summary.json", + "sample_slugs": [ + "balen", + "beerse_center", + "dessel_heide", + "geel", + "geel_bel", + "grobbendonk_center", + "herentals", + "herenthout_bos", + "hoogstraten_center", + "kasterlee_bos", + "kasterlee_center", + "lille_center", + "lommel_heide", + "meerhout_bos", + "mol", + "olen_center", + "oud_turnhout_center", + "postel_bos", + "ravels_bos", + "rijkevorsel_center", + "vorselaar_center", + "vosselaar_center" + ], + "sha256": "49b2a07d2105d08356431757b83eafc1498eaf1fb76965b1efe05b776824942a" + }, + { + "path": "/app/storage/operator-data/yolo-building-aoi1024-reviewedexp6-minpx3vis035/yolo_tile_dataset_summary.json", + "sample_slugs": [ + "arendonk_center", + "balen", + "beerse_center", + "dessel_center", + "dessel_heide", + "geel", + "geel_bel", + "grobbendonk_center", + "herentals", + "herenthout_bos", + "hoogstraten_center", + "hulshout_center", + "kasterlee_bos", + "kasterlee_center", + "laakdal_center", + "lille_center", + "lommel_heide", + "meerhout_bos", + "meerhout_center", + "mol", + "nijlen_center", + "olen_center", + "oud_turnhout_center", + "postel_bos", + "ravels_bos", + "rijkevorsel_center", + "vorselaar_center", + "vosselaar_center" + ], + "sha256": "7c917e31216d1df2174c0f9c736f88a81f3835aa991971f8fb8665e17ddf5c9c" + } + ], + "minimum_distance_m": 2000.0, + "missing_lineage_sample_slugs": [], + "resolved_lineage_sample_count": 31, + "schema_version": 1, + "status": "independent" +} diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/visual-review-summary.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/visual-review-summary.json new file mode 100644 index 00000000..a5a510af --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/visual-review-summary.json @@ -0,0 +1,426 @@ +{ + "blank_range_threshold": 3, + "class_names": [], + "columns": 4, + "contact_sheets": [ + { + "columns": 4, + "path": "contact_sheet_001.png", + "thumb_size": 512, + "tile_count": 24 + } + ], + "dataset_output_dir": null, + "highlight_category_counts": {}, + "highlighted_label_count": 0, + "invalid_label_count": 0, + "low_visual_variance_tile_count": 0, + "max_tiles": 24, + "missing_image_count": 0, + "missing_label_file_count": 0, + "outlier_highlight_count": 0, + "relationship_highlight_count": 0, + "rendered_tile_count": 24, + "requested_sample_slugs": [], + "selected_sample_count": 6, + "selected_sample_slugs": [ + "v72-assenede-field-cal-bg", + "v72-ganshoren-lowrise-cal", + "v72-havelange-field-cal-bg", + "v72-laeken-park-cal-bg", + "v72-libramont-lowrise-cal", + "v72-zandhoven-lowrise-cal" + ], + "selected_tile_count": 24, + "selected_tiles": [ + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-zandhoven-lowrise-cal_0000_r0_c0.png", + "invalid_label_count": 0, + "label_count": 60, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-zandhoven-lowrise-cal_0000_r0_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-zandhoven-lowrise-cal", + "split": "val", + "tile_index": 0, + "valid_label_count": 60 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-havelange-field-cal-bg_0000_r0_c0.png", + "invalid_label_count": 0, + "label_count": 48, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-havelange-field-cal-bg_0000_r0_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-havelange-field-cal-bg", + "split": "val", + "tile_index": 0, + "valid_label_count": 48 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-havelange-field-cal-bg_0004_r512_c512.png", + "invalid_label_count": 0, + "label_count": 48, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-havelange-field-cal-bg_0004_r512_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-havelange-field-cal-bg", + "split": "val", + "tile_index": 4, + "valid_label_count": 48 + }, + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-ganshoren-lowrise-cal_0001_r0_c512.png", + "invalid_label_count": 0, + "label_count": 28, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-ganshoren-lowrise-cal_0001_r0_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-ganshoren-lowrise-cal", + "split": "val", + "tile_index": 1, + "valid_label_count": 28 + }, + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-zandhoven-lowrise-cal_0001_r0_c512.png", + "invalid_label_count": 0, + "label_count": 28, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-zandhoven-lowrise-cal_0001_r0_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-zandhoven-lowrise-cal", + "split": "val", + "tile_index": 1, + "valid_label_count": 28 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-havelange-field-cal-bg_0003_r512_c0.png", + "invalid_label_count": 0, + "label_count": 24, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-havelange-field-cal-bg_0003_r512_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-havelange-field-cal-bg", + "split": "val", + "tile_index": 3, + "valid_label_count": 24 + }, + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-zandhoven-lowrise-cal_0003_r512_c0.png", + "invalid_label_count": 0, + "label_count": 22, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-zandhoven-lowrise-cal_0003_r512_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-zandhoven-lowrise-cal", + "split": "val", + "tile_index": 3, + "valid_label_count": 22 + }, + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-ganshoren-lowrise-cal_0003_r512_c0.png", + "invalid_label_count": 0, + "label_count": 17, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-ganshoren-lowrise-cal_0003_r512_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-ganshoren-lowrise-cal", + "split": "val", + "tile_index": 3, + "valid_label_count": 17 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-havelange-field-cal-bg_0001_r0_c512.png", + "invalid_label_count": 0, + "label_count": 14, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-havelange-field-cal-bg_0001_r0_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-havelange-field-cal-bg", + "split": "val", + "tile_index": 1, + "valid_label_count": 14 + }, + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-libramont-lowrise-cal_0000_r0_c0.png", + "invalid_label_count": 0, + "label_count": 13, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-libramont-lowrise-cal_0000_r0_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-libramont-lowrise-cal", + "split": "val", + "tile_index": 0, + "valid_label_count": 13 + }, + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-libramont-lowrise-cal_0003_r512_c0.png", + "invalid_label_count": 0, + "label_count": 11, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-libramont-lowrise-cal_0003_r512_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-libramont-lowrise-cal", + "split": "val", + "tile_index": 3, + "valid_label_count": 11 + }, + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-ganshoren-lowrise-cal_0004_r512_c512.png", + "invalid_label_count": 0, + "label_count": 10, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-ganshoren-lowrise-cal_0004_r512_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-ganshoren-lowrise-cal", + "split": "val", + "tile_index": 4, + "valid_label_count": 10 + }, + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-zandhoven-lowrise-cal_0004_r512_c512.png", + "invalid_label_count": 0, + "label_count": 9, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-zandhoven-lowrise-cal_0004_r512_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-zandhoven-lowrise-cal", + "split": "val", + "tile_index": 4, + "valid_label_count": 9 + }, + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-ganshoren-lowrise-cal_0000_r0_c0.png", + "invalid_label_count": 0, + "label_count": 7, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-ganshoren-lowrise-cal_0000_r0_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-ganshoren-lowrise-cal", + "split": "val", + "tile_index": 0, + "valid_label_count": 7 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-assenede-field-cal-bg_0004_r512_c512.png", + "invalid_label_count": 0, + "label_count": 4, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-assenede-field-cal-bg_0004_r512_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-assenede-field-cal-bg", + "split": "val", + "tile_index": 4, + "valid_label_count": 4 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-laeken-park-cal-bg_0000_r0_c0.png", + "invalid_label_count": 0, + "label_count": 3, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-laeken-park-cal-bg_0000_r0_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-laeken-park-cal-bg", + "split": "val", + "tile_index": 0, + "valid_label_count": 3 + }, + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-libramont-lowrise-cal_0001_r0_c512.png", + "invalid_label_count": 0, + "label_count": 2, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-libramont-lowrise-cal_0001_r0_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-libramont-lowrise-cal", + "split": "val", + "tile_index": 1, + "valid_label_count": 2 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-assenede-field-cal-bg_0000_r0_c0.png", + "invalid_label_count": 0, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-assenede-field-cal-bg_0000_r0_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-assenede-field-cal-bg", + "split": "val", + "tile_index": 0, + "valid_label_count": 0 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-assenede-field-cal-bg_0001_r0_c512.png", + "invalid_label_count": 0, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-assenede-field-cal-bg_0001_r0_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-assenede-field-cal-bg", + "split": "val", + "tile_index": 1, + "valid_label_count": 0 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-assenede-field-cal-bg_0003_r512_c0.png", + "invalid_label_count": 0, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-assenede-field-cal-bg_0003_r512_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-assenede-field-cal-bg", + "split": "val", + "tile_index": 3, + "valid_label_count": 0 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-laeken-park-cal-bg_0001_r0_c512.png", + "invalid_label_count": 0, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-laeken-park-cal-bg_0001_r0_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-laeken-park-cal-bg", + "split": "val", + "tile_index": 1, + "valid_label_count": 0 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-laeken-park-cal-bg_0003_r512_c0.png", + "invalid_label_count": 0, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-laeken-park-cal-bg_0003_r512_c0.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-laeken-park-cal-bg", + "split": "val", + "tile_index": 3, + "valid_label_count": 0 + }, + { + "background_category": "sparse_building_context", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-laeken-park-cal-bg_0004_r512_c512.png", + "invalid_label_count": 0, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-laeken-park-cal-bg_0004_r512_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "background_candidate", + "sample_slug": "v72-laeken-park-cal-bg", + "split": "val", + "tile_index": 4, + "valid_label_count": 0 + }, + { + "background_category": "reference_aoi", + "highlighted_label_count": 0, + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-libramont-lowrise-cal_0004_r512_c512.png", + "invalid_label_count": 0, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-libramont-lowrise-cal_0004_r512_c512.txt", + "low_visual_variance": false, + "missing_label_file": false, + "rendered": true, + "sample_role": "positive", + "sample_slug": "v72-libramont-lowrise-cal", + "split": "val", + "tile_index": 4, + "valid_label_count": 0 + } + ], + "status": "ok", + "summary_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/yolo_tile_dataset_summary.json", + "thumb_size": 512, + "tiles_per_sheet": 24, + "valid_label_count": 348 +} \ No newline at end of file diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/yolo-NO_TRAINING.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/yolo-NO_TRAINING.json new file mode 100644 index 00000000..5e9368c2 --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/yolo-NO_TRAINING.json @@ -0,0 +1,8 @@ +{ + "reason": "non_protected_diagnostic_evaluation_only", + "release_claim_allowed": false, + "schema_version": 1, + "source_manifest_sha256": "66bfaacaee11806ed035d3ac5725f7b3967dfc82e7a12ecf603dc3836bc26ad7", + "summary_sha256": "78e16f46fa2a313a147404a19f2ce99417d2cb384b5db5f6ba33e934ff99c631", + "training_allowed": false +} diff --git a/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/yolo_tile_dataset_summary.json b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/yolo_tile_dataset_summary.json new file mode 100644 index 00000000..72060abb --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/yolo_tile_dataset_summary.json @@ -0,0 +1,886 @@ +{ + "dataset_yaml": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/dataset.yaml", + "dataset_yaml_sha256": "d1e85ffb6776d92a4534830f2214fa6444f5a9160329aca0a68d0ea0a1a40144", + "excluded_overlapping_edge_cover_tile_count": 30, + "excluded_overlapping_edge_cover_tiles": [ + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-zandhoven-lowrise-cal", + "tile_index": 2, + "window": { + "col_off": 521, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-zandhoven-lowrise-cal", + "tile_index": 5, + "window": { + "col_off": 521, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-zandhoven-lowrise-cal", + "tile_index": 6, + "window": { + "col_off": 0, + "height": 512, + "row_off": 521, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-zandhoven-lowrise-cal", + "tile_index": 7, + "window": { + "col_off": 512, + "height": 512, + "row_off": 521, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-zandhoven-lowrise-cal", + "tile_index": 8, + "window": { + "col_off": 521, + "height": 512, + "row_off": 521, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-assenede-field-cal-bg", + "tile_index": 2, + "window": { + "col_off": 530, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-assenede-field-cal-bg", + "tile_index": 5, + "window": { + "col_off": 530, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-assenede-field-cal-bg", + "tile_index": 6, + "window": { + "col_off": 0, + "height": 512, + "row_off": 530, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-assenede-field-cal-bg", + "tile_index": 7, + "window": { + "col_off": 512, + "height": 512, + "row_off": 530, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-assenede-field-cal-bg", + "tile_index": 8, + "window": { + "col_off": 530, + "height": 512, + "row_off": 530, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-libramont-lowrise-cal", + "tile_index": 2, + "window": { + "col_off": 540, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-libramont-lowrise-cal", + "tile_index": 5, + "window": { + "col_off": 540, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-libramont-lowrise-cal", + "tile_index": 6, + "window": { + "col_off": 0, + "height": 512, + "row_off": 540, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-libramont-lowrise-cal", + "tile_index": 7, + "window": { + "col_off": 512, + "height": 512, + "row_off": 540, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-libramont-lowrise-cal", + "tile_index": 8, + "window": { + "col_off": 540, + "height": 512, + "row_off": 540, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-havelange-field-cal-bg", + "tile_index": 2, + "window": { + "col_off": 537, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-havelange-field-cal-bg", + "tile_index": 5, + "window": { + "col_off": 537, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-havelange-field-cal-bg", + "tile_index": 6, + "window": { + "col_off": 0, + "height": 512, + "row_off": 537, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-havelange-field-cal-bg", + "tile_index": 7, + "window": { + "col_off": 512, + "height": 512, + "row_off": 537, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-havelange-field-cal-bg", + "tile_index": 8, + "window": { + "col_off": 537, + "height": 512, + "row_off": 537, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-ganshoren-lowrise-cal", + "tile_index": 2, + "window": { + "col_off": 514, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-ganshoren-lowrise-cal", + "tile_index": 5, + "window": { + "col_off": 514, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-ganshoren-lowrise-cal", + "tile_index": 6, + "window": { + "col_off": 0, + "height": 512, + "row_off": 514, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-ganshoren-lowrise-cal", + "tile_index": 7, + "window": { + "col_off": 512, + "height": 512, + "row_off": 514, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-ganshoren-lowrise-cal", + "tile_index": 8, + "window": { + "col_off": 514, + "height": 512, + "row_off": 514, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-laeken-park-cal-bg", + "tile_index": 2, + "window": { + "col_off": 513, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-laeken-park-cal-bg", + "tile_index": 5, + "window": { + "col_off": 513, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-laeken-park-cal-bg", + "tile_index": 6, + "window": { + "col_off": 0, + "height": 512, + "row_off": 513, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-laeken-park-cal-bg", + "tile_index": 7, + "window": { + "col_off": 512, + "height": 512, + "row_off": 513, + "width": 512 + } + }, + { + "reason": "overlapping_edge_cover_tile", + "sample_slug": "v72-laeken-park-cal-bg", + "tile_index": 8, + "window": { + "col_off": 513, + "height": 512, + "row_off": 513, + "width": 512 + } + } + ], + "label_count": 348, + "purpose": "non_protected_diagnostic_evaluation", + "release_claim_allowed": false, + "schema_version": 1, + "selected_sample_slugs": [ + "v72-assenede-field-cal-bg", + "v72-ganshoren-lowrise-cal", + "v72-havelange-field-cal-bg", + "v72-laeken-park-cal-bg", + "v72-libramont-lowrise-cal", + "v72-zandhoven-lowrise-cal" + ], + "source_manifest": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-corpus/operator_samples_manifest.json", + "source_manifest_sha256": "66bfaacaee11806ed035d3ac5725f7b3967dfc82e7a12ecf603dc3836bc26ad7", + "status": "ok_evaluation_only", + "stride": 512, + "tile_count": 24, + "tile_size": 512, + "tiles": [ + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-zandhoven-lowrise-cal_0000_r0_c0.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 60, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-zandhoven-lowrise-cal_0000_r0_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-zandhoven-lowrise-cal", + "split": "val", + "tile_index": 0, + "window": { + "col_off": 0, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-zandhoven-lowrise-cal_0001_r0_c512.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 28, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-zandhoven-lowrise-cal_0001_r0_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-zandhoven-lowrise-cal", + "split": "val", + "tile_index": 1, + "window": { + "col_off": 512, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-zandhoven-lowrise-cal_0003_r512_c0.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 22, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-zandhoven-lowrise-cal_0003_r512_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-zandhoven-lowrise-cal", + "split": "val", + "tile_index": 3, + "window": { + "col_off": 0, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-zandhoven-lowrise-cal_0004_r512_c512.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 9, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-zandhoven-lowrise-cal_0004_r512_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-zandhoven-lowrise-cal", + "split": "val", + "tile_index": 4, + "window": { + "col_off": 512, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-assenede-field-cal-bg_0000_r0_c0.png", + "is_negative": true, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-assenede-field-cal-bg_0000_r0_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-assenede-field-cal-bg", + "split": "val", + "tile_index": 0, + "window": { + "col_off": 0, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-assenede-field-cal-bg_0001_r0_c512.png", + "is_negative": true, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-assenede-field-cal-bg_0001_r0_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-assenede-field-cal-bg", + "split": "val", + "tile_index": 1, + "window": { + "col_off": 512, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-assenede-field-cal-bg_0003_r512_c0.png", + "is_negative": true, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-assenede-field-cal-bg_0003_r512_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-assenede-field-cal-bg", + "split": "val", + "tile_index": 3, + "window": { + "col_off": 0, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-assenede-field-cal-bg_0004_r512_c512.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 4, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-assenede-field-cal-bg_0004_r512_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-assenede-field-cal-bg", + "split": "val", + "tile_index": 4, + "window": { + "col_off": 512, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-libramont-lowrise-cal_0000_r0_c0.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 13, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-libramont-lowrise-cal_0000_r0_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-libramont-lowrise-cal", + "split": "val", + "tile_index": 0, + "window": { + "col_off": 0, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-libramont-lowrise-cal_0001_r0_c512.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 2, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-libramont-lowrise-cal_0001_r0_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-libramont-lowrise-cal", + "split": "val", + "tile_index": 1, + "window": { + "col_off": 512, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-libramont-lowrise-cal_0003_r512_c0.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 11, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-libramont-lowrise-cal_0003_r512_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-libramont-lowrise-cal", + "split": "val", + "tile_index": 3, + "window": { + "col_off": 0, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-libramont-lowrise-cal_0004_r512_c512.png", + "is_negative": true, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-libramont-lowrise-cal_0004_r512_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-libramont-lowrise-cal", + "split": "val", + "tile_index": 4, + "window": { + "col_off": 512, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-havelange-field-cal-bg_0000_r0_c0.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 48, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-havelange-field-cal-bg_0000_r0_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-havelange-field-cal-bg", + "split": "val", + "tile_index": 0, + "window": { + "col_off": 0, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-havelange-field-cal-bg_0001_r0_c512.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 14, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-havelange-field-cal-bg_0001_r0_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-havelange-field-cal-bg", + "split": "val", + "tile_index": 1, + "window": { + "col_off": 512, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-havelange-field-cal-bg_0003_r512_c0.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 24, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-havelange-field-cal-bg_0003_r512_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-havelange-field-cal-bg", + "split": "val", + "tile_index": 3, + "window": { + "col_off": 0, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-havelange-field-cal-bg_0004_r512_c512.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 48, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-havelange-field-cal-bg_0004_r512_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-havelange-field-cal-bg", + "split": "val", + "tile_index": 4, + "window": { + "col_off": 512, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-ganshoren-lowrise-cal_0000_r0_c0.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 7, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-ganshoren-lowrise-cal_0000_r0_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-ganshoren-lowrise-cal", + "split": "val", + "tile_index": 0, + "window": { + "col_off": 0, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-ganshoren-lowrise-cal_0001_r0_c512.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 28, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-ganshoren-lowrise-cal_0001_r0_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-ganshoren-lowrise-cal", + "split": "val", + "tile_index": 1, + "window": { + "col_off": 512, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-ganshoren-lowrise-cal_0003_r512_c0.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 17, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-ganshoren-lowrise-cal_0003_r512_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-ganshoren-lowrise-cal", + "split": "val", + "tile_index": 3, + "window": { + "col_off": 0, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "background_category": "reference_aoi", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-ganshoren-lowrise-cal_0004_r512_c512.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 10, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-ganshoren-lowrise-cal_0004_r512_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "positive", + "sample_slug": "v72-ganshoren-lowrise-cal", + "split": "val", + "tile_index": 4, + "window": { + "col_off": 512, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-laeken-park-cal-bg_0000_r0_c0.png", + "is_negative": false, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 3, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-laeken-park-cal-bg_0000_r0_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-laeken-park-cal-bg", + "split": "val", + "tile_index": 0, + "window": { + "col_off": 0, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-laeken-park-cal-bg_0001_r0_c512.png", + "is_negative": true, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-laeken-park-cal-bg_0001_r0_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-laeken-park-cal-bg", + "split": "val", + "tile_index": 1, + "window": { + "col_off": 512, + "height": 512, + "row_off": 0, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-laeken-park-cal-bg_0003_r512_c0.png", + "is_negative": true, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-laeken-park-cal-bg_0003_r512_c0.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-laeken-park-cal-bg", + "split": "val", + "tile_index": 3, + "window": { + "col_off": 0, + "height": 512, + "row_off": 512, + "width": 512 + } + }, + { + "background_category": "sparse_building_context", + "image_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/images/val/v72-laeken-park-cal-bg_0004_r512_c512.png", + "is_negative": true, + "is_repeated_background_negative": false, + "kept": true, + "label_count": 0, + "label_path": "/app/storage/operator-data/building-be-v72-fresh-calibration-r1/diagnostic-yolo512-r2/labels/val/v72-laeken-park-cal-bg_0004_r512_c512.txt", + "low_visual_variance": false, + "recommended_split": "", + "repeat_index": 0, + "sample_role": "background_candidate", + "sample_slug": "v72-laeken-park-cal-bg", + "split": "val", + "tile_index": 4, + "window": { + "col_off": 512, + "height": 512, + "row_off": 512, + "width": 512 + } + } + ], + "training_allowed": false +} diff --git a/configs/accuracy/v72-fresh-calibration-aois.json b/configs/accuracy/v72-fresh-calibration-aois.json new file mode 100644 index 00000000..50cfcd58 --- /dev/null +++ b/configs/accuracy/v72-fresh-calibration-aois.json @@ -0,0 +1,61 @@ +{ + "schema_version": 1, + "purpose": "Fresh non-protected calibration portfolio for the retained Kempen model family", + "frozen_before_inference": true, + "aois": [ + { + "slug": "v72-zandhoven-lowrise-cal", + "region": "flanders", + "context": "detached-lowrise", + "split": "calibration", + "lon": 4.663, + "lat": 51.214, + "sample_role": "positive" + }, + { + "slug": "v72-assenede-field-cal-bg", + "region": "flanders", + "context": "farmland-hard-negative", + "split": "calibration", + "lon": 3.750, + "lat": 51.246, + "sample_role": "background_candidate" + }, + { + "slug": "v72-libramont-lowrise-cal", + "region": "wallonia", + "context": "rural-lowrise", + "split": "calibration", + "lon": 5.380, + "lat": 49.918, + "sample_role": "positive" + }, + { + "slug": "v72-havelange-field-cal-bg", + "region": "wallonia", + "context": "farmland-hard-negative", + "split": "calibration", + "lon": 5.239, + "lat": 50.389, + "sample_role": "background_candidate" + }, + { + "slug": "v72-ganshoren-lowrise-cal", + "region": "brussels", + "context": "urban-lowrise", + "split": "calibration", + "lon": 4.307, + "lat": 50.873, + "sample_role": "positive" + }, + { + "slug": "v72-laeken-park-cal-bg", + "region": "brussels", + "context": "park-hard-negative", + "split": "calibration", + "lon": 4.351, + "lat": 50.892, + "sample_role": "background_candidate" + } + ] +} diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index e16b8599..cd643ca7 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -12771,3 +12771,29 @@ Open: No independent positive or background accuracy evidence currently exists for this model family; new AOIs must be provisioned without ancestry or spatial exposure before another ranking is defensible. + +## 2026-08-09 - Fresh V72 regional calibration + +### Delivered + +- Added custom non-protected AOI provisioning with explicit rejection of test + and background-test roles. +- Acquired six new governed orthophoto/reference pairs across all three Belgian + regions and froze an evaluation-only, training-prohibited corpus. +- Added a diagnostic tiler that refuses training-eligible manifests, emits no + training data and excludes generic overlapping edge-cover windows. +- Added a raster-bound spatial lineage audit. All 31 ancestral AOIs resolved; + every V72 AOI passes the 2 km separation floor. +- Added strict pure-background validation: non-empty labels can no longer be + silently counted as pure background. + +### GPU result and decision + +- Corrected view: 24 non-overlapping tiles, 348 labels, six AOIs. +- Active mAP50/mAP50-95: `0.056779`/`0.021750`. +- Challenger mAP50/mAP50-95: `0.044705`/`0.016004`. +- Confidence 0.25 F1: active `0.231527`, challenger `0.235885`. +- The active aggregate F1 peaks at 0.30, but its Flemish subgroup regresses; + threshold, active weights and production image therefore remain unchanged. +- The weak fresh-domain scores are retained as a real blocker, not hidden by + the older exposed Turnhout/Westerlo/Postel diagnostics. diff --git a/docs/TODO.md b/docs/TODO.md index 179d0fd4..b6860198 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1185,3 +1185,14 @@ This file now starts with the current implementation status. Older preparation/b - [ ] Convert the AI-assisted ledger into no stronger claim than experimental triage; a real human must independently review and sign the frozen artifacts before the governed training wrapper may unlock. +- [x] Provision and visually inspect six fresh non-protected calibration AOIs + across Vlaanderen, Wallonië and Brussel; prove all are at least 2 km from all + 31 resolved ancestral AOIs and absent from every ancestral split. +- [x] Reject overlapping edge-cover tiles in diagnostic evaluation; retain 24 + canonical 512 px cells and record all 30 excluded overlapping views. +- [x] Run active/challenger GPU calibration on the fresh portfolio and retain + aggregate, regional and per-AOI metrics without promoting either model. +- [ ] Provision a separately frozen train-only failure corpus after human + review. Do not copy V72 calibration tiles or use their labels for fitting. +- [ ] Finalise version-specific PICC/UrbIS semantic harmonisation contracts + before treating Walloon/Brussels metrics as release-grade ground truth. diff --git a/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md b/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md index 3d290e85..f3c23212 100644 --- a/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md +++ b/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md @@ -263,3 +263,41 @@ The byte-matching parent/output chain is retained separately in `artifacts/evidence/accuracy/model-training/20260809-v71-model-lineage-receipt.json`. Fresh geographically separated AOIs with complete spatial and lineage checks are required for the next meaningful evaluation. + +## Fresh V72 calibration portfolio + +Six AOIs were pre-registered before inference and acquired again from the +governed regional orthophoto and building adapters: two each in Vlaanderen, +Wallonië and Brussel. All six are absent from every split of the retained model +lineage. A raster-bound spatial audit resolved all 31 ancestral AOIs and found +the nearest new AOI at 4,869.63 metres, above the frozen 2 km floor. + +The initial generic edge-cover tiling would have counted 54 tiles because its +final-row/final-column coverage nearly duplicated the preceding 512 px cells. +That view was rejected. The corrected evaluation-only exporter retains exactly +24 full grid cells, excludes 30 overlapping edge-cover views with reason codes, +and writes no training tiles. Both the corpus and tile view contain bound +`NO_TRAINING.json` markers. + +All 24 tiles and 348 retained labels were visually inspected. Havelange is a +positive village context despite its pre-registered `field-cal-bg` name. +Assenede and Laeken contain sparse official building labels and are difficult +contexts, not pure-empty backgrounds. The evaluator now refuses to accept a +pure-background prefix when any corresponding label file is non-empty. + +On the RTX 4080 SUPER the active model scores mAP50 `0.056779` and mAP50-95 +`0.021750`; the challenger scores `0.044705` and `0.016004`. At confidence 0.25 +and match IoU 0.25 their F1 scores are `0.231527` and `0.235885`. These low +values expose substantial domain-generalisation failure. The active model stays +in production because the challenger has weaker mAP, only a marginal F1 gain, +missing historical release provenance and no valid promotion bundle. + +The active model's aggregate calibration F1 peaks at threshold 0.30 +(`0.250356`), but its Flemish subgroup becomes worse than at lower thresholds. +No threshold was changed because a global average may not mask that subgroup +regression. PICC and UrbIS remain diagnostic-only while their regional semantic +harmonisation contracts are pending; no national release claim is made. + +Complete evidence is retained under +`artifacts/evidence/accuracy/model-training/20260809-v72-fresh-calibration/` +and summarized by `20260809-v72-fresh-calibration-summary.json`. diff --git a/scripts/README.md b/scripts/README.md index ee8e90e3..9fc33953 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -53,6 +53,23 @@ marker before release verification or CUDA allocation. Non-overlap proves only that tiles do not share pixels; checkpoint reports explicitly avoid claiming statistical independence for adjacent tiles from the same AOI. +`provision_belgium_building_training_portfolio.py --aoi-spec` provisions a +pre-registered custom portfolio but deliberately accepts only `train`, `val` +and `calibration`; protected test roles are rejected. For regional diagnostic +work where the PICC/UrbIS harmonisation contract is still pending, +`assemble_belgium_building_corpus.py +--evaluation-only-pending-regional-contracts` preserves the exact authority +failure, writes `NO_TRAINING.json` and makes no release claim. + +`export_yolo_diagnostic_evaluation_tiles.py` accepts only such a +training-prohibited calibration manifest. It emits empty train directories and +retains only full canonical non-overlapping grid cells; generic final-edge +coverage windows are removed from the generated view and listed explicitly. +`audit_evaluation_lineage_spatial_independence.py` then resolves every ancestral +sample back to its georeferenced source raster and checks the frozen metric +distance floor. These controls supplement, rather than replace, exact sample +lineage checking in the checkpoint evaluator. + Setup-, import-, demo- en maintenance-scripts voor GeoIntel. ## WALOUS source provisioning diff --git a/scripts/assemble_belgium_building_corpus.py b/scripts/assemble_belgium_building_corpus.py index a1fcc3b7..4844aab4 100644 --- a/scripts/assemble_belgium_building_corpus.py +++ b/scripts/assemble_belgium_building_corpus.py @@ -62,6 +62,7 @@ def _validate_pair( reference: Dataset, *, fixture_mode: bool, + evaluation_only_pending_regional_contracts: bool, ) -> tuple[str, str, dict[str, Any]]: region = str(sample.get("region") or "").lower() if region not in REGION_SOURCES: @@ -89,9 +90,22 @@ def _validate_pair( for reason in eligibility[role]["reasons"] } ) - raise SystemExit( - f"Dataset pair is not eligible for training for {sample['sample_slug']}: {', '.join(reasons)}" + allowed_pending = bool( + evaluation_only_pending_regional_contracts + and split == "calibration" + and region in {"wallonia", "brussels"} + and set(reasons) == {"reference_building_validation_not_primary"} ) + if not allowed_pending: + raise SystemExit( + f"Dataset pair is not eligible for training for {sample['sample_slug']}: {', '.join(reasons)}" + ) + eligibility["evaluation_only_exception"] = { + "allowed": True, + "reason": "regional_building_authority_contract_pending", + "training_allowed": False, + "release_claim_allowed": False, + } return region, expected_reference, eligibility @@ -130,6 +144,15 @@ def main() -> int: parser.add_argument("--min-label-px", type=float, default=3.0) parser.add_argument("--merge-touching-roofs", action="store_true") parser.add_argument("--freeze", action="store_true") + parser.add_argument( + "--evaluation-only-pending-regional-contracts", + action="store_true", + help=( + "Allow calibration-only PICC/UrbIS pairs whose sole training gate " + "failure is a pending regional authority contract. Writes an " + "explicit NO_TRAINING marker and never makes a release claim." + ), + ) parser.add_argument( "--fixture-mode", action="store_true", @@ -168,6 +191,9 @@ def main() -> int: raster, reference, fixture_mode=args.fixture_mode, + evaluation_only_pending_regional_contracts=( + args.evaluation_only_pending_regional_contracts + ), ) raster_source = _dataset_path(raster) reference_source_path = _dataset_path(reference) @@ -223,9 +249,18 @@ def main() -> int: "immutable": bool(args.freeze), "training_eligibility": { "policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION, - "status": "eligible", + "status": ( + "not_eligible_evaluation_only" + if args.evaluation_only_pending_regional_contracts + else "eligible" + ), "fixture_mode": bool(args.fixture_mode), }, + "purpose": ( + "non_protected_diagnostic_evaluation" + if args.evaluation_only_pending_regional_contracts + else "training_corpus" + ), "samples": manifest_samples, } manifest_path = output_dir / "operator_samples_manifest.json" @@ -244,8 +279,21 @@ def main() -> int: "immutable": bool(args.freeze), "training_eligibility_policy": TRAINING_ELIGIBILITY_POLICY_VERSION, "fixture_mode": bool(args.fixture_mode), + "training_allowed": not args.evaluation_only_pending_regional_contracts, + "release_claim_allowed": False, } (output_dir / "corpus-freeze.json").write_text(json.dumps(freeze, indent=2), encoding="utf-8") + if args.evaluation_only_pending_regional_contracts: + marker = { + "schema_version": 1, + "reason": "evaluation_only_pending_regional_authority_contracts", + "training_allowed": False, + "release_claim_allowed": False, + "manifest_sha256": freeze["manifest_sha256"], + } + (output_dir / "NO_TRAINING.json").write_text( + json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) print(json.dumps(freeze)) return 0 diff --git a/scripts/audit_evaluation_lineage_spatial_independence.py b/scripts/audit_evaluation_lineage_spatial_independence.py new file mode 100644 index 00000000..bbe79274 --- /dev/null +++ b/scripts/audit_evaluation_lineage_spatial_independence.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Audit fresh evaluation AOIs against every raster in a model's corpus lineage.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +from pyproj import Transformer +from shapely.geometry import box +from shapely.ops import transform as shapely_transform + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def metric_box(bounds: list[float] | tuple[float, ...]) -> Any: + transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) + return shapely_transform(transformer.transform, box(*map(float, bounds))) + + +def raster_bounds_epsg4326(path: Path) -> list[float]: + import rasterio + from rasterio.warp import transform_bounds + + with rasterio.open(path) as dataset: + if dataset.crs is None: + raise ValueError(f"lineage raster has no CRS: {path}") + bounds = transform_bounds(dataset.crs, "EPSG:4326", *dataset.bounds) + return list(map(float, bounds)) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--evaluation-manifest", type=Path, required=True) + parser.add_argument("--lineage-summary", type=Path, action="append", required=True) + parser.add_argument("--manifest-search-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--minimum-distance-m", type=float, default=2000.0) + args = parser.parse_args() + if args.output.exists(): + parser.error(f"output already exists: {args.output}") + evaluation_manifest = args.evaluation_manifest.resolve(strict=True) + evaluation = json.loads(evaluation_manifest.read_text(encoding="utf-8-sig")) + evaluation_rows = evaluation.get("samples") + if not isinstance(evaluation_rows, list) or not evaluation_rows: + raise SystemExit("evaluation manifest contains no samples") + + lineage_slugs: set[str] = set() + summary_evidence: list[dict[str, Any]] = [] + for raw in args.lineage_summary: + summary = raw.resolve(strict=True) + payload = json.loads(summary.read_text(encoding="utf-8")) + tiles = payload.get("tiles") + if not isinstance(tiles, list): + raise SystemExit(f"lineage summary has no tiles: {summary}") + slugs = {str(row.get("sample_slug") or "").strip() for row in tiles} + slugs.discard("") + lineage_slugs.update(slugs) + summary_evidence.append( + {"path": str(summary), "sha256": sha256_file(summary), "sample_slugs": sorted(slugs)} + ) + + candidates: dict[str, dict[str, dict[str, Any]]] = { + slug: {} for slug in lineage_slugs + } + for manifest_path in sorted( + args.manifest_search_root.rglob("operator_samples_manifest.json") + ): + try: + payload = json.loads(manifest_path.read_text(encoding="utf-8-sig")) + except (OSError, json.JSONDecodeError): + continue + for sample in payload.get("samples") or []: + if not isinstance(sample, dict): + continue + slug = str(sample.get("sample_slug") or "").strip() + raster_raw = sample.get("raster_path") + if slug not in candidates or not isinstance(raster_raw, str): + continue + raster = Path(raster_raw) + if not raster.is_file(): + continue + key = str(raster.resolve()) + candidates[slug].setdefault( + key, + { + "sample_slug": slug, + "raster_path": key, + "declared_raster_sha256": sample.get("raster_sha256"), + "source_manifest_path": str(manifest_path), + "source_manifest_sha256": sha256_file(manifest_path), + }, + ) + + missing = sorted(slug for slug, rows in candidates.items() if not rows) + lineage_geometries: list[tuple[dict[str, Any], Any]] = [] + for rows in candidates.values(): + for record in rows.values(): + bounds = raster_bounds_epsg4326(Path(record["raster_path"])) + record["bbox_epsg4326"] = bounds + lineage_geometries.append((record, metric_box(bounds))) + + results: list[dict[str, Any]] = [] + for sample in evaluation_rows: + slug = str(sample.get("sample_slug") or "").strip() + bounds = sample.get("bbox_epsg4326") + if not isinstance(bounds, list) or len(bounds) != 4: + raise SystemExit(f"evaluation sample has no governed bbox: {slug}") + geometry = metric_box(bounds) + distances = [ + (float(geometry.distance(other)), record) + for record, other in lineage_geometries + ] + distances.sort(key=lambda row: row[0]) + minimum, nearest = distances[0] + results.append( + { + "sample_slug": slug, + "bbox_epsg4326": bounds, + "minimum_lineage_distance_m": minimum, + "nearest_lineage_sample_slug": nearest["sample_slug"], + "nearest_lineage_raster_path": nearest["raster_path"], + "passes_minimum_distance": minimum >= args.minimum_distance_m, + } + ) + + status = "independent" if not missing and all( + row["passes_minimum_distance"] for row in results + ) else "failed" + output = { + "schema_version": 1, + "status": status, + "minimum_distance_m": args.minimum_distance_m, + "evaluation_manifest_path": str(evaluation_manifest), + "evaluation_manifest_sha256": sha256_file(evaluation_manifest), + "lineage_summaries": summary_evidence, + "lineage_sample_count": len(lineage_slugs), + "resolved_lineage_sample_count": len(lineage_slugs) - len(missing), + "missing_lineage_sample_slugs": missing, + "evaluation_samples": results, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(output, indent=2, sort_keys=True)) + return 0 if status == "independent" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluate_yolo_checkpoint_matrix.py b/scripts/evaluate_yolo_checkpoint_matrix.py index 6afa1d7c..9cb50f9d 100644 --- a/scripts/evaluate_yolo_checkpoint_matrix.py +++ b/scripts/evaluate_yolo_checkpoint_matrix.py @@ -270,12 +270,39 @@ def background_detection_count( return len(selected), count +def validate_pure_background_prefixes( + images: list[Path], prefixes: tuple[str, ...] +) -> list[Path]: + selected = [path for path in images if path.stem.casefold().startswith(prefixes)] + if not selected: + raise ValueError( + "no validation images match the declared pure-background prefixes" + ) + nonempty: list[str] = [] + for image in selected: + try: + relative = image.relative_to(image.parents[1]) + except ValueError as exc: + raise ValueError(f"cannot resolve label path for {image}") from exc + label = image.parents[1].parent / "labels" / relative.with_suffix(".txt") + if not label.is_file(): + raise ValueError(f"pure-background image has no label file: {image}") + if label.read_text(encoding="utf-8").strip(): + nonempty.append(str(label)) + if nonempty: + raise ValueError( + "pure-background prefixes include non-empty labels: " + + ", ".join(nonempty[:10]) + ) + return selected + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--dataset-yaml", type=Path, required=True) parser.add_argument("--model", type=Path, action="append", required=True) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--background-prefix", action="append", required=True) + parser.add_argument("--background-prefix", action="append", default=[]) parser.add_argument("--background-confidence", type=float, default=0.15) parser.add_argument( "--lineage-summary", @@ -321,6 +348,8 @@ def main() -> int: write_blocked_manifest(args.output, dataset_yaml, independence_evidence) return 3 prefixes = tuple(value.casefold() for value in args.background_prefix) + if prefixes: + validate_pure_background_prefixes(images, prefixes) import torch from ultralytics import YOLO @@ -359,14 +388,6 @@ def main() -> int: save_json=False, verbose=False, ) - background_images, background_detections = background_detection_count( - model, - images, - prefixes=prefixes, - confidence=args.background_confidence, - image_size=args.imgsz, - device=args.device, - ) row.update( { "status": "ok", @@ -374,10 +395,23 @@ def main() -> int: "recall": metric_value(metrics, "mr"), "map50": metric_value(metrics, "map50"), "map50_95": metric_value(metrics, "map"), - "pure_background_image_count": background_images, - "pure_background_detection_count": background_detections, } ) + if prefixes: + background_images, background_detections = background_detection_count( + model, + images, + prefixes=prefixes, + confidence=args.background_confidence, + image_size=args.imgsz, + device=args.device, + ) + row.update( + { + "pure_background_image_count": background_images, + "pure_background_detection_count": background_detections, + } + ) except Exception as exc: # preserve the complete attempted matrix row.update( { @@ -396,7 +430,7 @@ def main() -> int: successful = [row for row in rows if row["status"] == "ok"] successful.sort( key=lambda row: ( - int(row["pure_background_detection_count"] == 0), + int(row.get("pure_background_detection_count") == 0 and bool(prefixes)), row["map50_95"], row["map50"], row["precision"], diff --git a/scripts/export_yolo_diagnostic_evaluation_tiles.py b/scripts/export_yolo_diagnostic_evaluation_tiles.py new file mode 100644 index 00000000..9eea88a3 --- /dev/null +++ b/scripts/export_yolo_diagnostic_evaluation_tiles.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Tile an explicitly training-prohibited calibration corpus for diagnostics.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +try: + from scripts.export_operator_yolo_tile_dataset import ( + ensure_dependencies, + ensure_yolo_directories, + export_sample_tiles, + file_sha256, + write_dataset_yaml, + ) +except ModuleNotFoundError: # direct execution from /app/scripts + from export_operator_yolo_tile_dataset import ( + ensure_dependencies, + ensure_yolo_directories, + export_sample_tiles, + file_sha256, + write_dataset_yaml, + ) + + +def validate_diagnostic_manifest(manifest_path: Path) -> dict[str, Any]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig")) + if manifest.get("purpose") != "non_protected_diagnostic_evaluation": + raise ValueError("manifest is not a diagnostic-evaluation corpus") + eligibility = manifest.get("training_eligibility") + if not isinstance(eligibility, dict) or eligibility.get("status") != ( + "not_eligible_evaluation_only" + ): + raise ValueError("diagnostic manifest must be explicitly training-ineligible") + samples = manifest.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("diagnostic manifest contains no samples") + slugs: set[str] = set() + for sample in samples: + if not isinstance(sample, dict): + raise ValueError("diagnostic samples must be objects") + slug = str(sample.get("sample_slug") or "").strip() + if not slug or slug in slugs: + raise ValueError(f"missing or duplicate diagnostic sample slug: {slug}") + if sample.get("split") != "calibration": + raise ValueError(f"diagnostic sample is not calibration-only: {slug}") + if sample.get("sample_role") not in {"positive", "background_candidate"}: + raise ValueError(f"invalid diagnostic sample role: {slug}") + slugs.add(slug) + marker_path = manifest_path.parent / "NO_TRAINING.json" + if not marker_path.is_file(): + raise ValueError("diagnostic corpus has no NO_TRAINING.json marker") + marker = json.loads(marker_path.read_text(encoding="utf-8")) + if marker.get("training_allowed") is not False: + raise ValueError("NO_TRAINING marker does not prohibit training") + if marker.get("manifest_sha256") != file_sha256(manifest_path): + raise ValueError("NO_TRAINING marker is not bound to the diagnostic manifest") + return manifest + + +def is_canonical_evaluation_window(window: dict[str, Any], tile_size: int) -> bool: + return bool( + int(window["row_off"]) % tile_size == 0 + and int(window["col_off"]) % tile_size == 0 + and int(window["height"]) == tile_size + and int(window["width"]) == tile_size + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--tile-size", type=int, default=512) + parser.add_argument("--stride", type=int, default=512) + parser.add_argument("--min-label-px", type=float, default=4.0) + parser.add_argument("--min-label-visible-ratio", type=float, default=0.35) + args = parser.parse_args() + if args.output_dir.exists(): + raise SystemExit(f"refusing to overwrite evaluation tiles: {args.output_dir}") + if args.stride != args.tile_size: + raise SystemExit("diagnostic evaluation tiles must be non-overlapping") + manifest_path = args.manifest.expanduser().resolve(strict=True) + try: + manifest = validate_diagnostic_manifest(manifest_path) + except (OSError, ValueError, json.JSONDecodeError) as exc: + raise SystemExit(str(exc)) from exc + + ensure_dependencies() + ensure_yolo_directories(args.output_dir) + exported: list[dict[str, Any]] = [] + for sample in manifest["samples"]: + exported.extend( + export_sample_tiles( + sample=sample, + manifest_path=manifest_path, + output_dir=args.output_dir, + val_slugs={str(sample["sample_slug"]).lower()}, + tile_size=args.tile_size, + stride=args.stride, + negative_keep_ratio=1.0, + min_label_px=args.min_label_px, + min_label_visible_ratio=args.min_label_visible_ratio, + background_negative_repeat=1, + drop_low_variance_negatives=False, + blank_range_threshold=8.0, + reference_source=str(sample["reference_source"]), + reference_layer=str(sample["reference_layer"]), + ) + ) + kept: list[dict[str, Any]] = [] + excluded_edge_cover: list[dict[str, Any]] = [] + for row in exported: + window = row["window"] + is_canonical_grid = bool( + row["kept"] + and is_canonical_evaluation_window(window, args.tile_size) + ) + if is_canonical_grid: + kept.append(row) + continue + if row["kept"]: + excluded_edge_cover.append( + { + "sample_slug": row["sample_slug"], + "tile_index": row["tile_index"], + "window": window, + "reason": "overlapping_edge_cover_tile", + } + ) + Path(row["image_path"]).unlink(missing_ok=True) + Path(row["label_path"]).unlink(missing_ok=True) + if not kept or any(row["split"] != "val" for row in kept): + raise SystemExit("diagnostic exporter produced invalid split membership") + if any((args.output_dir / "images" / "train").iterdir()) or any( + (args.output_dir / "labels" / "train").iterdir() + ): + raise SystemExit("diagnostic exporter wrote training data") + + dataset_yaml = write_dataset_yaml(args.output_dir, "building") + summary = { + "schema_version": 1, + "status": "ok_evaluation_only", + "purpose": "non_protected_diagnostic_evaluation", + "training_allowed": False, + "release_claim_allowed": False, + "dataset_yaml": str(dataset_yaml), + "dataset_yaml_sha256": file_sha256(dataset_yaml), + "source_manifest": str(manifest_path), + "source_manifest_sha256": file_sha256(manifest_path), + "tile_size": args.tile_size, + "stride": args.stride, + "tile_count": len(kept), + "label_count": sum(int(row["label_count"]) for row in kept), + "excluded_overlapping_edge_cover_tile_count": len(excluded_edge_cover), + "excluded_overlapping_edge_cover_tiles": excluded_edge_cover, + "selected_sample_slugs": sorted( + {str(row["sample_slug"]) for row in kept} + ), + "tiles": kept, + } + summary_path = args.output_dir / "yolo_tile_dataset_summary.json" + summary_path.write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + marker = { + "schema_version": 1, + "reason": "non_protected_diagnostic_evaluation_only", + "training_allowed": False, + "release_claim_allowed": False, + "source_manifest_sha256": summary["source_manifest_sha256"], + "summary_sha256": file_sha256(summary_path), + } + (args.output_dir / "NO_TRAINING.json").write_text( + json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(json.dumps({key: value for key, value in summary.items() if key != "tiles"})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/provision_belgium_building_training_portfolio.py b/scripts/provision_belgium_building_training_portfolio.py index 6da265f0..f1280607 100644 --- a/scripts/provision_belgium_building_training_portfolio.py +++ b/scripts/provision_belgium_building_training_portfolio.py @@ -267,6 +267,62 @@ REGION_CONTRACT = { }, } +NON_PROTECTED_SPLITS = {"train", "val", "calibration"} + + +def load_custom_aois(path: Path) -> tuple[Aoi, ...]: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + rows = payload.get("aois") + if not isinstance(rows, list) or not rows: + raise SystemExit("--aoi-spec requires a non-empty aois list") + result: list[Aoi] = [] + seen: set[str] = set() + for row in rows: + if not isinstance(row, dict): + raise SystemExit("--aoi-spec entries must be objects") + slug = str(row.get("slug") or "").strip() + region = str(row.get("region") or "").strip().lower() + split = str(row.get("split") or "").strip().lower() + context = str(row.get("context") or "").strip() + role = str(row.get("sample_role") or "positive").strip() + if not slug or slug in seen: + raise SystemExit(f"Custom AOI slug is missing or duplicated: {slug}") + if region not in REGION_CONTRACT: + raise SystemExit(f"Unsupported custom AOI region for {slug}: {region}") + if split not in NON_PROTECTED_SPLITS: + raise SystemExit( + f"Custom AOI {slug} uses protected/unsupported split {split!r}; " + "fresh provisioning is limited to train, val and calibration" + ) + if not context: + raise SystemExit(f"Custom AOI context is missing: {slug}") + try: + lon = float(row["lon"]) + lat = float(row["lat"]) + except (KeyError, TypeError, ValueError) as exc: + raise SystemExit(f"Custom AOI coordinates are invalid: {slug}") from exc + if not 2.4 <= lon <= 6.5 or not 49.4 <= lat <= 51.7: + raise SystemExit(f"Custom AOI is outside the Belgium workbench: {slug}") + if role not in {"positive", "background_candidate"}: + raise SystemExit(f"Unsupported custom AOI sample_role for {slug}: {role}") + require_empty = bool(row.get("require_empty", False)) + if require_empty and role != "background_candidate": + raise SystemExit(f"Only background candidates may require empty labels: {slug}") + seen.add(slug) + result.append( + Aoi( + slug, + region, + context, + split, + lon, + lat, + role, + require_empty, + ) + ) + return tuple(result) + def bbox_for_center(lon: float, lat: float, side_m: float) -> dict[str, Any]: to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) @@ -298,6 +354,14 @@ def main() -> int: parser.add_argument("--side-m", type=float, default=256.0) parser.add_argument("--resolution-m", type=float, default=0.25) parser.add_argument("--force-refresh", action="store_true") + parser.add_argument( + "--aoi-spec", + type=Path, + help=( + "Optional JSON file containing fresh non-protected AOIs. Custom " + "specs cannot provision test or background-test roles." + ), + ) parser.add_argument( "--only-slug", action="append", @@ -310,11 +374,16 @@ def main() -> int: help="Read an operator session token from a local mode-0600 file instead of exposing it on the command line.", ) args = parser.parse_args() - known_slugs = {aoi.slug for aoi in AOIS} + if args.aoi_spec and args.only_slug: + parser.error("--aoi-spec and --only-slug are mutually exclusive") + available_aois = load_custom_aois(args.aoi_spec) if args.aoi_spec else AOIS + known_slugs = {aoi.slug for aoi in available_aois} unknown_slugs = sorted(set(args.only_slug) - known_slugs) if unknown_slugs: raise SystemExit(f"Unknown --only-slug value(s): {', '.join(unknown_slugs)}") - selected_aois = tuple(aoi for aoi in AOIS if not args.only_slug or aoi.slug in args.only_slug) + selected_aois = tuple( + aoi for aoi in available_aois if not args.only_slug or aoi.slug in args.only_slug + ) session = requests.Session() if args.session_token_file: token = args.session_token_file.read_text(encoding="utf-8").strip() diff --git a/tests/test_evaluate_yolo_checkpoint_matrix.py b/tests/test_evaluate_yolo_checkpoint_matrix.py index 89b13eea..c917baef 100644 --- a/tests/test_evaluate_yolo_checkpoint_matrix.py +++ b/tests/test_evaluate_yolo_checkpoint_matrix.py @@ -6,6 +6,7 @@ from scripts.evaluate_yolo_checkpoint_matrix import ( dataset_overlap_evidence, main, model_lineage_independence_evidence, + validate_pure_background_prefixes, write_blocked_manifest, ) @@ -196,3 +197,32 @@ def test_cli_blocks_lineage_validation_exposure_before_model_resolution( payload = json.loads(output.read_text(encoding="utf-8")) assert payload["status"] == "blocked_model_lineage_sample_exposure" assert payload["model_loading_attempted"] is False + + +def test_pure_background_prefix_rejects_nonempty_labels(tmp_path: Path) -> None: + image_dir = tmp_path / "images" / "val" + label_dir = tmp_path / "labels" / "val" + image_dir.mkdir(parents=True) + label_dir.mkdir(parents=True) + image = image_dir / "sparse_bg_0001.png" + image.write_bytes(b"image") + (label_dir / "sparse_bg_0001.txt").write_text( + "0 0.5 0.5 0.1 0.1\n", encoding="utf-8" + ) + + import pytest + + with pytest.raises(ValueError, match="non-empty labels"): + validate_pure_background_prefixes([image], ("sparse_bg",)) + + +def test_pure_background_prefix_accepts_empty_labels(tmp_path: Path) -> None: + image_dir = tmp_path / "images" / "val" + label_dir = tmp_path / "labels" / "val" + image_dir.mkdir(parents=True) + label_dir.mkdir(parents=True) + image = image_dir / "pure_bg_0001.png" + image.write_bytes(b"image") + (label_dir / "pure_bg_0001.txt").write_text("", encoding="utf-8") + + assert validate_pure_background_prefixes([image], ("pure_bg",)) == [image] diff --git a/tests/test_export_yolo_diagnostic_evaluation_tiles.py b/tests/test_export_yolo_diagnostic_evaluation_tiles.py new file mode 100644 index 00000000..abc783a8 --- /dev/null +++ b/tests/test_export_yolo_diagnostic_evaluation_tiles.py @@ -0,0 +1,74 @@ +import json +from pathlib import Path + +import pytest + +from scripts.export_yolo_diagnostic_evaluation_tiles import ( + is_canonical_evaluation_window, + validate_diagnostic_manifest, +) + + +def _write_manifest(tmp_path: Path, *, split: str = "calibration") -> Path: + manifest = tmp_path / "operator_samples_manifest.json" + manifest.write_text( + json.dumps( + { + "purpose": "non_protected_diagnostic_evaluation", + "training_eligibility": { + "status": "not_eligible_evaluation_only" + }, + "samples": [ + { + "sample_slug": "fresh-aoi", + "split": split, + "sample_role": "positive", + } + ], + } + ), + encoding="utf-8", + ) + import hashlib + + digest = hashlib.sha256(manifest.read_bytes()).hexdigest() + (tmp_path / "NO_TRAINING.json").write_text( + json.dumps({"training_allowed": False, "manifest_sha256": digest}), + encoding="utf-8", + ) + return manifest + + +def test_validate_diagnostic_manifest_accepts_bound_calibration(tmp_path: Path) -> None: + manifest = _write_manifest(tmp_path) + + payload = validate_diagnostic_manifest(manifest) + + assert payload["samples"][0]["sample_slug"] == "fresh-aoi" + + +def test_validate_diagnostic_manifest_rejects_training_split(tmp_path: Path) -> None: + manifest = _write_manifest(tmp_path, split="train") + + with pytest.raises(ValueError, match="calibration-only"): + validate_diagnostic_manifest(manifest) + + +def test_validate_diagnostic_manifest_rejects_unbound_marker(tmp_path: Path) -> None: + manifest = _write_manifest(tmp_path) + (tmp_path / "NO_TRAINING.json").write_text( + json.dumps({"training_allowed": False, "manifest_sha256": "0" * 64}), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="not bound"): + validate_diagnostic_manifest(manifest) + + +def test_canonical_evaluation_window_rejects_overlapping_edge_cover() -> None: + assert is_canonical_evaluation_window( + {"row_off": 512, "col_off": 512, "height": 512, "width": 512}, 512 + ) + assert not is_canonical_evaluation_window( + {"row_off": 521, "col_off": 512, "height": 512, "width": 512}, 512 + ) diff --git a/tests/test_provision_belgium_building_training_portfolio.py b/tests/test_provision_belgium_building_training_portfolio.py index 8294d127..adef13d3 100644 --- a/tests/test_provision_belgium_building_training_portfolio.py +++ b/tests/test_provision_belgium_building_training_portfolio.py @@ -1,4 +1,9 @@ -from scripts.provision_belgium_building_training_portfolio import AOIS +import json +from pathlib import Path + +import pytest + +from scripts.provision_belgium_building_training_portfolio import AOIS, load_custom_aois V63_CANDIDATES = { @@ -32,3 +37,57 @@ def test_v66_lowrise_candidates_are_flemish_train_role_inputs() -> None: assert all(aoi.region == "flanders" for aoi in candidates.values()) assert all(aoi.split == "train" for aoi in candidates.values()) assert all("lowrise" in aoi.context for aoi in candidates.values()) + + +def test_custom_aoi_spec_loads_non_protected_calibration(tmp_path: Path) -> None: + spec = tmp_path / "aois.json" + spec.write_text( + json.dumps( + { + "aois": [ + { + "slug": "fresh-rural-cal", + "region": "wallonia", + "context": "rural-lowrise", + "split": "calibration", + "lon": 5.25, + "lat": 50.25, + } + ] + } + ), + encoding="utf-8", + ) + + result = load_custom_aois(spec) + + assert len(result) == 1 + assert result[0].slug == "fresh-rural-cal" + assert result[0].split == "calibration" + + +@pytest.mark.parametrize("split", ["test", "background-test"]) +def test_custom_aoi_spec_rejects_protected_splits( + tmp_path: Path, split: str +) -> None: + spec = tmp_path / "aois.json" + spec.write_text( + json.dumps( + { + "aois": [ + { + "slug": "forbidden", + "region": "flanders", + "context": "urban", + "split": split, + "lon": 4.5, + "lat": 51.0, + } + ] + } + ), + encoding="utf-8", + ) + + with pytest.raises(SystemExit, match="protected/unsupported split"): + load_custom_aois(spec)