diff --git a/backend/tests/test_failure_driven_yolo_sampling.py b/backend/tests/test_failure_driven_yolo_sampling.py index 19bdb0f8..ae437379 100644 --- a/backend/tests/test_failure_driven_yolo_sampling.py +++ b/backend/tests/test_failure_driven_yolo_sampling.py @@ -279,6 +279,42 @@ def test_region_cap_rotates_repeats_between_sampling_rounds() -> None: assert second_metadata["sampling_round"] == 2 +def test_region_cap_preserves_failed_context_positive_before_hard_negative() -> None: + manifest = {"samples": [ + {"sample_slug": "target", "split": "train", "region": "flanders", "context": "industrial"}, + {"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"}, + {"sample_slug": "wa", "split": "train", "region": "wallonia", "context": "rural-town"}, + {"sample_slug": "br", "split": "train", "region": "brussels", "context": "dense-urban"}, + ]} + summary = {"tiles": [ + {"sample_slug": "target", "split": "train", "label_count": 2, "image_path": "/tmp/target.png"}, + {"sample_slug": "negative", "split": "train", "label_count": 0, "image_path": "/tmp/negative.png"}, + {"sample_slug": "wa", "split": "train", "label_count": 1, "image_path": "/tmp/wa.png"}, + {"sample_slug": "br", "split": "train", "label_count": 1, "image_path": "/tmp/br.png"}, + ]} + assessment = { + "status": "continue_training_loop", + "gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4, + "max_pure_empty_false_positives": 0}, + "calibration": { + "regions": { + "flanders": {"f1": .2, "precision": .2, "recall": .3}, + "wallonia": {"f1": .6, "precision": .6, "recall": .6}, + "brussels": {"f1": .6, "precision": .6, "recall": .6}, + }, + "samples": {"target": {"f1": .2, "precision": .2, "recall": .3}}, + }, + } + + paths, metadata = MODULE.build_sampling( + summary=summary, manifest=manifest, assessment=assessment, max_region_share=.65, + ) + + assert paths.count(str(Path("/tmp/target.png").resolve())) == 2 + assert paths.count(str(Path("/tmp/negative.png").resolve())) == 1 + assert metadata["priority_positive_repeat_count"] == 4 + + def test_precision_guard_band_keeps_near_gate_region_stabilized() -> None: manifest = {"samples": [ {"sample_slug": "wa-positive", "split": "train", "region": "wallonia", "context": "rural-town"}, diff --git a/scripts/build_failure_driven_yolo_sampling.py b/scripts/build_failure_driven_yolo_sampling.py index 89e701ea..b062fc2d 100644 --- a/scripts/build_failure_driven_yolo_sampling.py +++ b/scripts/build_failure_driven_yolo_sampling.py @@ -39,6 +39,30 @@ def dataset_validation_source(source_yaml: Path) -> str: raise ValueError(f"Source dataset YAML has no validation source: {source_yaml}") +def spread_repeats(paths: list[str], *, sampling_round: int, lane: str) -> list[str]: + """Deterministically spread adjacent repeats before a regional cap.""" + indexed = enumerate(paths) + return [ + path + for _index, path in sorted( + indexed, + key=lambda item: hashlib.sha256( + f"{sampling_round}:{lane}:{item[0]}:{item[1]}".encode() + ).digest(), + ) + ] + + +def interleave(left: list[str], right: list[str]) -> list[str]: + combined: list[str] = [] + for index in range(max(len(left), len(right))): + if index < len(left): + combined.append(left[index]) + if index < len(right): + combined.append(right[index]) + return combined + + def build_sampling( *, summary: dict[str, Any], @@ -128,6 +152,7 @@ def build_sampling( ) base_paths_by_region: dict[str, list[str]] = {} + priority_positive_paths_by_region: dict[str, list[str]] = {} extra_paths_by_region: dict[str, list[str]] = {} selected_samples: set[str] = set() protected_samples: set[str] = set() @@ -161,9 +186,14 @@ def build_sampling( ) path = str(Path(tile["image_path"]).resolve()) base_paths_by_region.setdefault(region, []).append(path) + priority_positives = priority_positive_paths_by_region.setdefault(region, []) extras = extra_paths_by_region.setdefault(region, []) repeated = [path] * (repeat - 1) - if tile["label_count"] == 0 and context_key in targeted_negative_contexts: + if tile["label_count"] > 0 and context_key in weak_recall_contexts: + # Preserve diagnostic positives before precision-oriented + # negatives when the regional cap truncates repeat entries. + priority_positives.extend(repeated) + elif tile["label_count"] == 0 and context_key in targeted_negative_contexts: # Preserve the most diagnostic hard-negative repeats when the # regional cap has to remove lower-priority repetition. extras[:0] = repeated @@ -172,7 +202,11 @@ def build_sampling( selected_samples.add(tile["sample_slug"]) pre_cap_counts = Counter({ - region: len(paths) + len(extra_paths_by_region[region]) + region: ( + len(paths) + + len(priority_positive_paths_by_region[region]) + + len(extra_paths_by_region[region]) + ) for region, paths in base_paths_by_region.items() }) capped_counts = Counter(pre_cap_counts) @@ -193,12 +227,17 @@ def build_sampling( base_paths = base_paths_by_region[region] extra_limit = capped_counts[region] - len(base_paths) image_paths.extend(base_paths) - extras = extra_paths_by_region[region] - if extras and extra_limit < len(extras): - # Rotate the capped repeat window between loop rounds so persistent - # failures cannot yield the exact same training list indefinitely. - offset = sampling_round % len(extras) - extras = extras[offset:] + extras[:offset] + priority_positives = spread_repeats( + priority_positive_paths_by_region[region], + sampling_round=sampling_round, + lane=f"{region}:positive", + ) + corrections = spread_repeats( + extra_paths_by_region[region], + sampling_round=sampling_round, + lane=f"{region}:correction", + ) + extras = interleave(priority_positives, corrections) image_paths.extend(extras[:extra_limit]) repeat_counts = Counter({region: capped_counts[region] for region in capped_counts}) if not image_paths: @@ -233,6 +272,9 @@ def build_sampling( ), "sampled_train_entry_count": len(image_paths), "sampled_entries_by_region": dict(sorted(repeat_counts.items())), + "priority_positive_repeat_count": sum( + len(paths) for paths in priority_positive_paths_by_region.values() + ), "pre_cap_entries_by_region": dict(sorted(pre_cap_counts.items())), "dropped_region_repeat_count": sum(pre_cap_counts.values()) - len(image_paths), "selected_train_sample_count": len(selected_samples), diff --git a/scripts/tower-v50-v47-loop-command.json b/scripts/tower-v50-v47-loop-command.json new file mode 100644 index 00000000..ef7e3be6 --- /dev/null +++ b/scripts/tower-v50-v47-loop-command.json @@ -0,0 +1,76 @@ +[ + "docker", + "exec", + "-d", + "geointel", + "/opt/geointel/venv/bin/python", + "/app/scripts/run_belgium_building_training_loop.py", + "--initial-model", + "/app/storage/training/building-be-v43-v42-closed-loop-r1/iteration-005/candidate.pt", + "--train-yaml", + "/app/storage/operator-data/building-be-v47-merged-rotated-holdouts-r1/train/dataset.yaml", + "--train-summary", + "/app/storage/operator-data/building-be-v47-merged-rotated-holdouts-r1/train/yolo_tile_dataset_summary.json", + "--dataset-audit", + "/app/storage/training/building-be-v47-corpus-audit-r1/belgium-building-corpus-audit.json", + "--train-quality-audit", + "/app/storage/training/building-be-v47-yolo-quality-audit-r1/operator_yolo_dataset_quality_audit.json", + "--calibration-summary", + "/app/storage/operator-data/building-be-v47-merged-rotated-holdouts-r1/calibration/yolo_tile_dataset_summary.json", + "--test-summary", + "/app/storage/operator-data/building-be-v47-merged-rotated-holdouts-r1/test/yolo_tile_dataset_summary.json", + "--background-summary", + "/app/storage/operator-data/building-be-v47-merged-rotated-holdouts-r1/background-test/yolo_tile_dataset_summary.json", + "--corpus-manifest", + "/app/storage/operator-data/building-be-v47-merged-rotated-holdouts-r1/operator_samples_manifest.json", + "--output-dir", + "/app/storage/training/building-be-v50-v47-spread-loop-r1", + "--iterations", + "20", + "--epochs", + "100", + "--patience", + "12", + "--batch", + "8", + "--workers", + "0", + "--max-det", + "1000", + "--imgsz", + "640", + "--optimizer", + "AdamW", + "--lr0", + "0.00005", + "--mosaic", + "0", + "--scale", + "0.15", + "--translate", + "0.05", + "--degrees", + "180", + "--flipud", + "0.5", + "--fliplr", + "0.5", + "--warmup-epochs", + "1", + "--warmup-bias-lr", + "0.01", + "--hsv-h", + "0.01", + "--hsv-s", + "0.2", + "--hsv-v", + "0.15", + "--seed", + "20260950", + "--yolo", + "/opt/geointel/venv/bin/yolo", + "--evaluate-initial-model" +] + + +