diff --git a/backend/tests/test_failure_driven_yolo_sampling.py b/backend/tests/test_failure_driven_yolo_sampling.py index 8fdd0bac..4a56f854 100644 --- a/backend/tests/test_failure_driven_yolo_sampling.py +++ b/backend/tests/test_failure_driven_yolo_sampling.py @@ -55,7 +55,7 @@ def test_sampling_repeats_only_failed_region_train_tiles() -> None: "background": {"pure_empty_false_positives": 2}, } paths, metadata = MODULE.build_sampling( - summary=summary, manifest=manifest, assessment=assessment + summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0 ) assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 3 assert paths.count(str(Path("/tmp/fl-neg.png").resolve())) == 4 @@ -87,7 +87,7 @@ def test_sampling_can_use_calibration_before_test_is_opened() -> None: "background": None, } paths, metadata = MODULE.build_sampling( - summary=summary, manifest=manifest, assessment=assessment + summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0 ) assert len(paths) == 3 assert metadata["failure_evidence_source"] == "calibration" @@ -120,6 +120,7 @@ def test_precision_correction_can_balance_positive_and_negative_tiles() -> None: assessment=assessment, precision_positive_repeat=2, negative_repeat=3, + max_region_share=1.0, ) assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 2 @@ -157,7 +158,9 @@ def test_sampling_targets_failed_calibration_contexts_without_using_protected_ti }, } - paths, metadata = MODULE.build_sampling(summary=summary, manifest=manifest, assessment=assessment) + paths, metadata = MODULE.build_sampling( + summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0 + ) assert paths.count(str(Path("/tmp/industry-pos.png").resolve())) == 5 assert paths.count(str(Path("/tmp/industry-neg.png").resolve())) == 6 @@ -165,3 +168,42 @@ def test_sampling_targets_failed_calibration_contexts_without_using_protected_ti assert not any("protected" in path for path in paths) assert metadata["weak_recall_contexts"] == ["flanders:industrial"] assert metadata["weak_precision_contexts"] == ["flanders:industrial"] + + +def test_region_cap_drops_only_repeats_and_preserves_every_unique_tile() -> None: + manifest = {"samples": [ + {"sample_slug": "fl", "split": "train", "region": "flanders", "context": "industrial"}, + {"sample_slug": "wa", "split": "train", "region": "wallonia", "context": "rural-town"}, + {"sample_slug": "br", "split": "train", "region": "brussels", "context": "dense-urban"}, + ]} + summary = {"tiles": [ + {"sample_slug": "fl", "split": "train", "label_count": 2, "image_path": f"/tmp/fl-{index}.png"} + for index in range(4) + ] + [ + {"sample_slug": "wa", "split": "train", "label_count": 2, "image_path": f"/tmp/wa-{index}.png"} + for index in range(2) + ] + [ + {"sample_slug": "br", "split": "train", "label_count": 2, "image_path": f"/tmp/br-{index}.png"} + for index in range(2) + ]} + assessment = { + "status": "continue_training_loop", + "gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4, + "max_pure_empty_false_positives": 0}, + "calibration": {"regions": { + "flanders": {"f1": .2, "precision": .3, "recall": .2}, + "wallonia": {"f1": .6, "precision": .6, "recall": .6}, + "brussels": {"f1": .6, "precision": .6, "recall": .6}, + }}, + } + + paths, metadata = MODULE.build_sampling( + summary=summary, manifest=manifest, assessment=assessment, + positive_repeat=5, max_region_share=.65, + ) + + assert all(str(Path(f"/tmp/fl-{index}.png").resolve()) in paths for index in range(4)) + assert metadata["pre_cap_entries_by_region"]["flanders"] == 20 + assert metadata["sampled_entries_by_region"]["flanders"] == 7 + assert metadata["dropped_region_repeat_count"] == 13 + assert metadata["sampled_entries_by_region"]["flanders"] / len(paths) <= .65 diff --git a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md index 87123e3b..e6565c3f 100644 --- a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md +++ b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md @@ -108,6 +108,11 @@ modes without copying a protected AOI into training. The sampling evidence records both context sets and repeat factors. When no matching train context exists, regional sampling remains active and the missing context becomes a concrete input for the next immutable corpus expansion. +Failure weighting may not let one region exceed 65% of the sampled entries. +The deterministic cap removes only repeated entries and retains every unique +train tile at least once; manifests record pre-cap counts, final counts and the +number of dropped repeats. This keeps a weak region prominent without turning +the national detector into a single-region expert. The checkpointed orchestrator invokes this builder after every rejected iteration, stores its checksum in `training-loop-state.json`, and uses the resulting dataset YAML for the next checkpoint. A restart resumes both the diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index d3879446..407ef8ab 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -15,6 +15,10 @@ loop now requires and hashes the train tile-quality report, and refuses missing counters instead of treating absent invalid/blank-label evidence as zero. +- Capped failure-driven regional oversampling at 65% after the first v31 + sampling assigned 75.5% of entries to Flanders. The cap retains every unique + tile, removes repeats only and writes pre/post regional counts into the + checksummed sampling evidence. ## 2026-07-27 - Guest demo and product professionalization diff --git a/scripts/build_failure_driven_yolo_sampling.py b/scripts/build_failure_driven_yolo_sampling.py index b190d905..08eea99e 100644 --- a/scripts/build_failure_driven_yolo_sampling.py +++ b/scripts/build_failure_driven_yolo_sampling.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse import hashlib import json +import math from collections import Counter from pathlib import Path from typing import Any @@ -38,6 +39,7 @@ def build_sampling( precision_positive_repeat: int = 1, context_positive_repeat: int = 5, context_negative_repeat: int = 6, + max_region_share: float = 0.65, ) -> tuple[list[str], dict[str, Any]]: if assessment.get("status") != "continue_training_loop": raise ValueError("Failure-driven sampling requires a failed assessment") @@ -49,6 +51,8 @@ def build_sampling( context_negative_repeat, ) < 1: raise ValueError("Repeat factors must be positive") + if not 0 < max_region_share <= 1: + raise ValueError("max_region_share must be in (0, 1]") samples = {item["sample_slug"]: item for item in manifest["samples"]} gates = assessment["gates"] @@ -91,8 +95,8 @@ def build_sampling( ): weak_precision_contexts.add(key) - image_paths: list[str] = [] - repeat_counts: Counter[str] = Counter() + base_paths_by_region: dict[str, list[str]] = {} + extra_paths_by_region: dict[str, list[str]] = {} selected_samples: set[str] = set() protected_samples: set[str] = set() for tile in summary["tiles"]: @@ -120,10 +124,34 @@ def build_sampling( else negative_repeat ) path = str(Path(tile["image_path"]).resolve()) - image_paths.extend([path] * repeat) - repeat_counts[region] += repeat + base_paths_by_region.setdefault(region, []).append(path) + extra_paths_by_region.setdefault(region, []).extend([path] * (repeat - 1)) selected_samples.add(tile["sample_slug"]) + pre_cap_counts = Counter({ + region: len(paths) + len(extra_paths_by_region[region]) + for region, paths in base_paths_by_region.items() + }) + capped_counts = Counter(pre_cap_counts) + if max_region_share < 1: + while capped_counts: + region, count = max(capped_counts.items(), key=lambda item: (item[1], item[0])) + total = sum(capped_counts.values()) + if count / total <= max_region_share: + break + others = total - count + limit = math.floor(max_region_share / (1 - max_region_share) * others) + limit = max(limit, len(base_paths_by_region[region])) + if limit >= count: + break + capped_counts[region] = limit + image_paths: list[str] = [] + for region in sorted(base_paths_by_region): + base_paths = base_paths_by_region[region] + extra_limit = capped_counts[region] - len(base_paths) + image_paths.extend(base_paths) + image_paths.extend(extra_paths_by_region[region][:extra_limit]) + repeat_counts = Counter({region: capped_counts[region] for region in capped_counts}) if not image_paths: raise ValueError("No train-only tiles selected") metadata = { @@ -141,6 +169,7 @@ def build_sampling( "precision_positive_repeat": precision_positive_repeat, "context_positive_repeat": context_positive_repeat, "context_negative_repeat": context_negative_repeat, + "max_region_share": max_region_share, "source_train_tile_count": sum( 1 for tile in summary["tiles"] @@ -148,6 +177,8 @@ def build_sampling( ), "sampled_train_entry_count": len(image_paths), "sampled_entries_by_region": dict(sorted(repeat_counts.items())), + "pre_cap_entries_by_region": dict(sorted(pre_cap_counts.items())), + "dropped_region_repeat_count": sum(pre_cap_counts.values()) - len(image_paths), "selected_train_sample_count": len(selected_samples), "protected_sample_count": len(protected_samples), "protected_samples_in_training": [], @@ -166,6 +197,7 @@ def main() -> int: parser.add_argument("--precision-positive-repeat", type=int, default=1) parser.add_argument("--context-positive-repeat", type=int, default=5) parser.add_argument("--context-negative-repeat", type=int, default=6) + parser.add_argument("--max-region-share", type=float, default=0.65) args = parser.parse_args() summary = json.loads(args.summary.read_text(encoding="utf-8")) @@ -180,6 +212,7 @@ def main() -> int: precision_positive_repeat=args.precision_positive_repeat, context_positive_repeat=args.context_positive_repeat, context_negative_repeat=args.context_negative_repeat, + max_region_share=args.max_region_share, ) args.output_dir.mkdir(parents=True, exist_ok=True) train_list = args.output_dir / "train-failure-driven.txt"