diff --git a/backend/tests/test_failure_driven_yolo_sampling.py b/backend/tests/test_failure_driven_yolo_sampling.py index b2f02e32..8fdd0bac 100644 --- a/backend/tests/test_failure_driven_yolo_sampling.py +++ b/backend/tests/test_failure_driven_yolo_sampling.py @@ -125,3 +125,43 @@ def test_precision_correction_can_balance_positive_and_negative_tiles() -> None: assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 2 assert paths.count(str(Path("/tmp/fl-neg.png").resolve())) == 3 assert metadata["precision_positive_repeat"] == 2 + + +def test_sampling_targets_failed_calibration_contexts_without_using_protected_tiles() -> None: + manifest = { + "samples": [ + {"sample_slug": "train-industry", "split": "train", "region": "flanders", "context": "industrial"}, + {"sample_slug": "train-suburban", "split": "train", "region": "flanders", "context": "suburban"}, + {"sample_slug": "cal-industry", "split": "calibration", "region": "flanders", "context": "industrial"}, + ] + } + summary = { + "tiles": [ + {"sample_slug": "train-industry", "split": "train", "label_count": 2, "image_path": "/tmp/industry-pos.png"}, + {"sample_slug": "train-industry", "split": "train", "label_count": 0, "image_path": "/tmp/industry-neg.png"}, + {"sample_slug": "train-suburban", "split": "train", "label_count": 2, "image_path": "/tmp/suburban-pos.png"}, + {"sample_slug": "cal-industry", "split": "val", "label_count": 2, "image_path": "/tmp/protected.png"}, + ] + } + assessment = { + "status": "continue_training_loop", + "gates": { + "min_region_f1": 0.45, + "min_region_precision": 0.5, + "min_region_recall": 0.4, + "max_pure_empty_false_positives": 0, + }, + "calibration": { + "regions": {"flanders": {"f1": 0.3, "precision": 0.35, "recall": 0.27}}, + "samples": {"cal-industry": {"f1": 0.2, "precision": 0.3, "recall": 0.15}}, + }, + } + + paths, metadata = MODULE.build_sampling(summary=summary, manifest=manifest, assessment=assessment) + + assert paths.count(str(Path("/tmp/industry-pos.png").resolve())) == 5 + assert paths.count(str(Path("/tmp/industry-neg.png").resolve())) == 6 + assert paths.count(str(Path("/tmp/suburban-pos.png").resolve())) == 3 + assert not any("protected" in path for path in paths) + assert metadata["weak_recall_contexts"] == ["flanders:industrial"] + assert metadata["weak_precision_contexts"] == ["flanders:industrial"] diff --git a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md index d849647e..4ce203a5 100644 --- a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md +++ b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md @@ -92,6 +92,13 @@ recall are repeated, while true negative train tiles are repeated when a regional precision gate or the pure-background gate fails. Calibration, test, background-test and validation AOIs are excluded by their frozen corpus split; the generated evidence records that no protected sample entered training. +Per-AOI calibration evidence also identifies failed region/context pairs. +Train-only AOIs with the same governed context receive a stronger repeat factor +than the remaining failed region, so correction rounds target distinct failure +modes without copying a protected AOI into training. The sampling evidence +records both context sets and repeat factors. When no matching train context +exists, regional sampling remains active and the missing context becomes a +concrete input for the next immutable corpus expansion. The checkpointed orchestrator invokes this builder after every rejected iteration, stores its checksum in `training-loop-state.json`, and uses the resulting dataset YAML for the next checkpoint. A restart resumes both the diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 6d327adf..eff91821 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,12 @@ +## 2026-07-29 - Context-aware Belgian training correction + +- Audited the active closed-loop calibration trend and confirmed that the + remaining Flemish failure cannot be solved by confidence-threshold selection. +- Extended failure-driven sampling with region-plus-context weighting from + per-AOI calibration evidence. Protected calibration, test, background-test + and validation tiles remain excluded; manifests now record targeted contexts + and their stronger repeat factors for reproducible follow-up iterations. + ## 2026-07-27 - Guest demo and product professionalization - Audited the access experience, workbench information density, responsive diff --git a/scripts/build_failure_driven_yolo_sampling.py b/scripts/build_failure_driven_yolo_sampling.py index 73c54060..b190d905 100644 --- a/scripts/build_failure_driven_yolo_sampling.py +++ b/scripts/build_failure_driven_yolo_sampling.py @@ -36,10 +36,18 @@ def build_sampling( positive_repeat: int = 3, negative_repeat: int = 4, precision_positive_repeat: int = 1, + context_positive_repeat: int = 5, + context_negative_repeat: int = 6, ) -> tuple[list[str], dict[str, Any]]: if assessment.get("status") != "continue_training_loop": raise ValueError("Failure-driven sampling requires a failed assessment") - if positive_repeat < 1 or negative_repeat < 1 or precision_positive_repeat < 1: + if min( + positive_repeat, + negative_repeat, + precision_positive_repeat, + context_positive_repeat, + context_negative_repeat, + ) < 1: raise ValueError("Repeat factors must be positive") samples = {item["sample_slug"]: item for item in manifest["samples"]} @@ -65,6 +73,23 @@ def build_sampling( and background["pure_empty_false_positives"] > gates["max_pure_empty_false_positives"] ) + weak_recall_contexts: set[tuple[str, str]] = set() + weak_precision_contexts: set[tuple[str, str]] = set() + for sample_slug, metrics in evaluation.get("samples", {}).items(): + sample = samples.get(sample_slug) + if not sample: + continue + key = (sample["region"], sample.get("context", "unknown")) + if sample["region"] in weak_recall_regions and ( + metrics["f1"] < gates["min_region_f1"] + or metrics["recall"] < gates["min_region_recall"] + ): + weak_recall_contexts.add(key) + if ( + sample["region"] in weak_precision_regions + and metrics["precision"] < gates["min_region_precision"] + ): + weak_precision_contexts.add(key) image_paths: list[str] = [] repeat_counts: Counter[str] = Counter() @@ -76,15 +101,24 @@ def build_sampling( protected_samples.add(tile["sample_slug"]) continue region = sample["region"] + context_key = (region, sample.get("context", "unknown")) repeat = 1 if tile["label_count"] > 0 and region in weak_recall_regions: - repeat = positive_repeat + repeat = ( + context_positive_repeat + if context_key in weak_recall_contexts + else positive_repeat + ) elif tile["label_count"] > 0 and region in weak_precision_regions: # Precision-only correction still needs positive examples to avoid # shifting the classifier toward background and sacrificing recall. repeat = precision_positive_repeat if tile["label_count"] == 0 and (background_failed or region in weak_precision_regions): - repeat = negative_repeat + repeat = ( + context_negative_repeat + if context_key in weak_precision_contexts + else negative_repeat + ) path = str(Path(tile["image_path"]).resolve()) image_paths.extend([path] * repeat) repeat_counts[region] += repeat @@ -99,10 +133,14 @@ def build_sampling( "failure_evidence_source": "test" if assessment.get("test") else "calibration", "weak_recall_regions": sorted(weak_recall_regions), "weak_precision_regions": sorted(weak_precision_regions), + "weak_recall_contexts": [f"{region}:{context}" for region, context in sorted(weak_recall_contexts)], + "weak_precision_contexts": [f"{region}:{context}" for region, context in sorted(weak_precision_contexts)], "background_gate_failed": background_failed, "positive_repeat": positive_repeat, "negative_repeat": negative_repeat, "precision_positive_repeat": precision_positive_repeat, + "context_positive_repeat": context_positive_repeat, + "context_negative_repeat": context_negative_repeat, "source_train_tile_count": sum( 1 for tile in summary["tiles"] @@ -126,6 +164,8 @@ def main() -> int: parser.add_argument("--positive-repeat", type=int, default=3) parser.add_argument("--negative-repeat", type=int, default=4) parser.add_argument("--precision-positive-repeat", type=int, default=1) + parser.add_argument("--context-positive-repeat", type=int, default=5) + parser.add_argument("--context-negative-repeat", type=int, default=6) args = parser.parse_args() summary = json.loads(args.summary.read_text(encoding="utf-8")) @@ -138,6 +178,8 @@ def main() -> int: positive_repeat=args.positive_repeat, negative_repeat=args.negative_repeat, precision_positive_repeat=args.precision_positive_repeat, + context_positive_repeat=args.context_positive_repeat, + context_negative_repeat=args.context_negative_repeat, ) args.output_dir.mkdir(parents=True, exist_ok=True) train_list = args.output_dir / "train-failure-driven.txt"