From 0f49c980baf2371a528cd76bb2663b4264e4e911 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 12 Jul 2026 23:42:29 +0200 Subject: [PATCH] Expand YOLO training AOIs safely --- CHANGELOG.md | 8 +++ ...st_sprint130_operator_yolo_tile_dataset.py | 25 ++++++++ ...est_sprint131_operator_sample_expansion.py | 35 +++++++++++ ...int156_background_corpus_classification.py | 1 + ...print170_detection_false_negative_audit.py | 59 +++++++++++++++++++ docs/CODEX_EXECUTION_LOG.md | 25 ++++++++ docs/TODO.md | 11 ++++ scripts/README.md | 37 ++++++++++++ ...audit_detection_false_negative_evidence.py | 23 +++++++- scripts/export_operator_yolo_tile_dataset.py | 49 ++++++++++++++- scripts/prepare_operator_real_data_samples.py | 44 +++++++++++++- 11 files changed, 310 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc592403..72706aac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ # Changelog +## Sprint 171 Positive AOI expansion and split safety (2026-07-12) + +- Added four explicit, real-reference Kempen training AOIs: Olen, Lille, Oud-Turnhout and Kasterlee center. +- Preserved Turnhout, Retie, Westerlo and Arendonk-heide as manifest-backed validation holdouts and made the tile exporter reject unknown samples or holdout leakage. +- Added `recommended_split` provenance to generated sample/reference/tile metadata and recorded the validation split in dataset summaries. +- Hardened persistent false-negative comparison so portfolios with different reference feature identities cannot be compared. +- Documented the expanded low-minimum-label dataset flow; no model was activated and no product API or migration changed. + ## Sprint 170 Persistent false-negative evidence audit (2026-07-12) - Added fixed-threshold evidence portfolio input generation so model comparisons use exactly one matching model/tile/threshold run per AOI. diff --git a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py index 16bcb8d4..76924031 100644 --- a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py +++ b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py @@ -6,6 +6,7 @@ import subprocess import sys import numpy as np +import pytest ROOT = Path(__file__).resolve().parents[2] @@ -75,6 +76,30 @@ def test_operator_yolo_tile_dataset_export_help_does_not_require_gis_dependencie assert "--blank-range-threshold" in result.stdout +def test_default_validation_split_is_explicit_and_rejects_holdout_leakage() -> None: + module = load_tile_exporter() + samples = [ + {"sample_slug": "geel", "recommended_split": "train"}, + {"sample_slug": "turnhout", "recommended_split": "val"}, + {"sample_slug": "retie", "recommended_split": "val"}, + {"sample_slug": "westerlo", "recommended_split": "val"}, + {"sample_slug": "arendonk_heide", "recommended_split": "val"}, + ] + + assert module.DEFAULT_VALIDATION_SAMPLE_SLUGS == frozenset( + {"turnhout", "retie", "westerlo", "arendonk_heide"} + ) + assert module.validate_validation_split( + samples, + set(module.DEFAULT_VALIDATION_SAMPLE_SLUGS), + ) == set(module.DEFAULT_VALIDATION_SAMPLE_SLUGS) + + with pytest.raises(SystemExit, match="recommended validation holdouts"): + module.validate_validation_split(samples, {"turnhout"}) + with pytest.raises(SystemExit, match="unknown samples"): + module.validate_validation_split(samples, {"turnhout", "missing"}) + + def test_iter_tile_windows_covers_edges_without_duplicates() -> None: module = load_tile_exporter() diff --git a/backend/tests/test_sprint131_operator_sample_expansion.py b/backend/tests/test_sprint131_operator_sample_expansion.py index 0bb93095..b245b750 100644 --- a/backend/tests/test_sprint131_operator_sample_expansion.py +++ b/backend/tests/test_sprint131_operator_sample_expansion.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.util +import math from pathlib import Path import sys @@ -43,6 +44,40 @@ def test_operator_sample_registry_includes_kempen_reference_and_background_candi assert all(module.SAMPLES[slug].sample_role == "background_candidate" for slug in expected_background_slugs) +def test_operator_training_expansion_preserves_geographically_separate_holdouts() -> None: + module = load_sample_preparer() + + expected_expansion = {"olen_center", "lille_center", "oud_turnhout_center", "kasterlee_center"} + expected_holdouts = {"turnhout", "retie", "westerlo", "arendonk_heide"} + + assert module.TRAINING_EXPANSION_SAMPLE_SLUGS == frozenset(expected_expansion) + assert module.DEFAULT_VALIDATION_SAMPLE_SLUGS == frozenset(expected_holdouts) + assert all(module.SAMPLES[slug].sample_role == "reference" for slug in expected_expansion) + assert all(not module.SAMPLES[slug].allow_empty_reference for slug in expected_expansion) + assert all(module.recommended_split_for_sample(module.SAMPLES[slug]) == "train" for slug in expected_expansion) + assert all(module.recommended_split_for_sample(module.SAMPLES[slug]) == "val" for slug in expected_holdouts) + + def distance_m(left, right) -> float: + radius_m = 6_371_008.8 + left_lat = math.radians(left.center_lat) + right_lat = math.radians(right.center_lat) + delta_lat = right_lat - left_lat + delta_lon = math.radians(right.center_lon - left.center_lon) + haversine = ( + math.sin(delta_lat / 2) ** 2 + + math.cos(left_lat) * math.cos(right_lat) * math.sin(delta_lon / 2) ** 2 + ) + return 2 * radius_m * math.asin(math.sqrt(haversine)) + + reference_holdouts = expected_holdouts - {"arendonk_heide"} + for expansion_slug in expected_expansion: + expansion = module.SAMPLES[expansion_slug] + assert min( + distance_m(expansion, module.SAMPLES[holdout_slug]) + for holdout_slug in reference_holdouts + ) >= 2_000 + + def test_operator_background_candidates_are_unique_enough_for_hard_negative_training() -> None: module = load_sample_preparer() diff --git a/backend/tests/test_sprint156_background_corpus_classification.py b/backend/tests/test_sprint156_background_corpus_classification.py index c362fff0..be8fde20 100644 --- a/backend/tests/test_sprint156_background_corpus_classification.py +++ b/backend/tests/test_sprint156_background_corpus_classification.py @@ -90,6 +90,7 @@ def test_prepare_sample_manifest_records_background_category_from_cached_referen prepared = module.prepare_sample(sample, tmp_path, force=False) assert prepared["background_category"] == "sparse_building_context" + assert prepared["recommended_split"] == "train" assert prepared["reference_feature_count"] == 1 diff --git a/backend/tests/test_sprint170_detection_false_negative_audit.py b/backend/tests/test_sprint170_detection_false_negative_audit.py index 83515ca8..6817c9ef 100644 --- a/backend/tests/test_sprint170_detection_false_negative_audit.py +++ b/backend/tests/test_sprint170_detection_false_negative_audit.py @@ -201,3 +201,62 @@ def test_false_negative_audit_finds_persistent_reference_misses(tmp_path: Path) assert active["false_negative_area_m2"]["median"] > 0 assert report["recommendations"] assert (output_dir / "detection_false_negative_audit.md").is_file() + + +def test_false_negative_audit_rejects_mismatched_reference_populations(tmp_path: Path) -> None: + script = ROOT / "scripts" / "audit_detection_false_negative_evidence.py" + portfolio_args = [] + for label, source_ids in (("active", ("one", "two")), ("candidate", ("one",))): + portfolio_dir = tmp_path / label + evidence_dir = portfolio_dir / "samples" / "geel" / "evidence" + evidence_dir.mkdir(parents=True) + evidence_path = evidence_dir / "calibration_evidence.geojson" + evidence_path.write_text( + json.dumps( + { + "type": "FeatureCollection", + "features": [ + _evidence_feature( + "false_negative", + source_id, + _polygon(5.0 + index * 0.001, 51.2, 0.0001), + ) + for index, source_id in enumerate(source_ids) + ], + } + ), + encoding="utf-8", + ) + portfolio_path = portfolio_dir / "calibration_evidence_portfolio.json" + portfolio_path.write_text( + json.dumps( + { + "model_asset_id": f"model-{label}", + "samples": [ + { + "sample_slug": "geel", + "evidence_geojson_path": str(evidence_path), + } + ], + } + ), + encoding="utf-8", + ) + portfolio_args.extend(("--portfolio", f"{label}={portfolio_path}")) + + result = subprocess.run( + [ + sys.executable, + str(script), + *portfolio_args, + "--output-dir", + str(tmp_path / "audit"), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "different reference populations" in result.stderr diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 8ed8888e..d62fae03 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -6974,3 +6974,28 @@ Open: ## Next recommended pass - Train one inactive candidate from the filtered AOI1024 cleanpx dataset, then run the existing positive-AOI matrix and split-background promotion workflow before considering default activation. +# Sprint 171 - Positive AOI expansion and split safety + +## What changed + +- Converted the Sprint 170 false-negative evidence into a guarded data action instead of another blind training run. +- Added Olen, Lille, Oud-Turnhout and Kasterlee center as explicit real-reference training AOIs. +- Kept Turnhout, Retie, Westerlo and Arendonk-heide as the documented validation holdouts. +- Added generated `recommended_split` provenance and tile-export validation that rejects unknown samples and manifest-backed holdout leakage. +- Hardened false-negative portfolio comparison to require identical reference feature identities, not only matching AOI names. + +## Local validation + +- RED tests proved the expansion/split constants and validation guard were absent before implementation. +- `python -m pytest backend/tests/test_sprint131_operator_sample_expansion.py backend/tests/test_sprint130_operator_yolo_tile_dataset.py backend/tests/test_sprint156_background_corpus_classification.py backend/tests/test_sprint170_detection_false_negative_audit.py -q`: 23 passed. +- Official GRB OGC API probes returned building features at all four new AOI centers. +- Local sample generation was attempted but correctly stopped because the workstation Python lacks the existing GIS runtime extras; the all-in-one Tower runtime is the supported execution environment. + +## Known limitations + +- No new model has been trained or activated. +- The four AOIs still require full orthophoto/reference preparation, tile export, structural audit and visual contact-sheet review on Tower. + +## Next recommended pass + +- Redeploy Tower, refresh the existing AOI1024 manifest so only missing AOIs are downloaded, then compare a `min-label-px=4` expanded dataset against the rejected `min-label-px=12` baseline before deciding whether training is justified. diff --git a/docs/TODO.md b/docs/TODO.md index 881fc836..d0ceb7e0 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -494,3 +494,14 @@ This file now starts with the current implementation status. Older preparation/b - [x] Improve AOI1024 label quality before retraining: `yolo-building-aoi1024-cleanpx12vis035` now audits `ok` with 14,632 labels, `min_label_px=12`, `min_label_visible_ratio=0.35`, median normalized box area `0.001373291015625` and small-box share `0.0`. - [x] Train and reject `geointel-building-yolov8s-aoi1024cleanpx12vis035e50-pt` through the positive/background promotion gate. - [ ] Keep every local YOLO candidate inactive until positive-AOI and hard-negative promotion reports recommend default activation. + +# Sprint 171 - Positive AOI expansion and small-building recovery + +- [x] Reject cross-model false-negative comparisons when reference populations differ. +- [x] Add explicit Olen, Lille, Oud-Turnhout and Kasterlee positive training AOIs. +- [x] Preserve Turnhout, Retie, Westerlo and Arendonk-heide as manifest-backed validation holdouts. +- [x] Make the tile exporter reject unknown validation samples and holdout leakage. +- [ ] Refresh the full AOI1024 operator manifest on Tower and fetch only missing AOIs. +- [ ] Export and audit a low-minimum-label dataset without changing the active model. +- [ ] Render and inspect a label contact sheet before training. +- [ ] Train only when the expanded dataset passes structural and visual review. diff --git a/scripts/README.md b/scripts/README.md index a9f83f5a..032d732d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -405,6 +405,43 @@ This refreshed cleanpx dataset is the minimum pre-training baseline after the visual contact-sheet pass found six blank-looking `arendonk_heide` validation negatives in the older export. +The persisted false-negative audit subsequently showed that the cleanpx12 +candidate still misses about 79-92% of the comparable reference population and +misses every reference building below 25 m2 in the seven-AOI review. Do not +train another candidate from the same four positive training AOIs. Refresh the +existing AOI1024 sample directory after pulling Sprint 171; existing files are +reused and only the four new explicit positive AOIs need to be fetched: + +```bash +docker exec -it geointel python3 /app/scripts/prepare_operator_real_data_samples.py \ + --output-dir /app/storage/operator-data/operator-samples-1024 \ + --width 1024 \ + --height 1024 \ + --half-size-scale 2 +``` + +The expansion adds Olen, Lille, Oud-Turnhout and Kasterlee center as training +samples. Turnhout, Retie, Westerlo and Arendonk-heide remain explicit validation +holdouts in generated manifest provenance. The tile exporter defaults to those +four holdouts and rejects a manifest-aware split that leaks one into training. +Use the lower `min-label-px=4` profile first to measure small-building retention; +it remains subject to dataset audit and visual contact-sheet review before any +training: + +```bash +docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \ + --manifest-path /app/storage/operator-data/operator-samples-1024/operator_samples_manifest.json \ + --output-dir /app/storage/operator-data/yolo-building-aoi1024-expanded-minpx4vis035 \ + --tile-size 512 \ + --stride 256 \ + --negative-keep-ratio 1.0 \ + --min-label-px 4 \ + --min-label-visible-ratio 0.35 \ + --drop-low-variance-negatives \ + --blank-range-threshold 3 \ + --force +``` + Then audit with stricter small-box gates: ```bash diff --git a/scripts/audit_detection_false_negative_evidence.py b/scripts/audit_detection_false_negative_evidence.py index 7ac089d6..a5e656c5 100644 --- a/scripts/audit_detection_false_negative_evidence.py +++ b/scripts/audit_detection_false_negative_evidence.py @@ -116,6 +116,7 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) -> raise SystemExit("Evidence GeoJSON must be a FeatureCollection") features = payload.get("features") or [] false_negative_ids: set[str] = set() + reference_ids: set[str] = set() false_negative_areas: list[float] = [] matched_reference_areas: list[float] = [] bucket_counts = { @@ -141,11 +142,13 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) -> ) area_m2 = abs(float(geod.geometry_area_perimeter(geometry)[0])) bucket = area_bucket(area_m2) + reference_id = stable_reference_id(feature, geometry) + reference_ids.add(reference_id) bucket_role = "false_negative" if role == "false_negative" else "matched_reference" bucket_counts[bucket][bucket_role] += 1 bucket_counts[bucket]["total_reference"] += 1 if role == "false_negative": - false_negative_ids.add(stable_reference_id(feature, geometry)) + false_negative_ids.add(reference_id) false_negative_areas.append(area_m2) else: matched_reference_areas.append(area_m2) @@ -157,6 +160,7 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) -> total_reference = len(false_negative_areas) + len(matched_reference_areas) return { "false_negative_ids": false_negative_ids, + "reference_ids": reference_ids, "false_negative_count": len(false_negative_areas), "matched_reference_count": len(matched_reference_areas), "total_reference_count": total_reference, @@ -284,16 +288,31 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]: for sample_slug in sorted(expected_slugs): portfolio_rows = [] false_negative_sets = [] + reference_sets = [] for label, _ in parsed_portfolios: raw = portfolio_samples[label][sample_slug] false_negative_sets.append(raw["false_negative_ids"]) + reference_sets.append(raw["reference_ids"]) portfolio_rows.append( - {key: value for key, value in raw.items() if key != "false_negative_ids"} + { + key: value + for key, value in raw.items() + if key not in {"false_negative_ids", "reference_ids"} + } + ) + if any(reference_ids != reference_sets[0] for reference_ids in reference_sets[1:]): + counts = ", ".join( + f"{label}={len(reference_ids)}" + for (label, _), reference_ids in zip(parsed_portfolios, reference_sets, strict=True) + ) + raise SystemExit( + f"Sample {sample_slug} has different reference populations across portfolios ({counts})" ) persistent_ids = sorted(set.intersection(*false_negative_sets)) sample_reports.append( { "sample_slug": sample_slug, + "reference_population_count": len(reference_sets[0]), "persistent_false_negative_count": len(persistent_ids), "persistent_reference_ids": persistent_ids, "portfolios": portfolio_rows, diff --git a/scripts/export_operator_yolo_tile_dataset.py b/scripts/export_operator_yolo_tile_dataset.py index e591a68d..aa4e8bce 100644 --- a/scripts/export_operator_yolo_tile_dataset.py +++ b/scripts/export_operator_yolo_tile_dataset.py @@ -25,6 +25,10 @@ REFERENCE_AOI_CATEGORY = "reference_aoi" PURE_EMPTY_BACKGROUND_CATEGORY = "pure_empty_negative" SPARSE_BACKGROUND_CATEGORY = "sparse_building_context" LOW_VARIANCE_NEGATIVE_SKIP_REASON = "low_visual_variance_negative" +DEFAULT_VALIDATION_SAMPLE_SLUGS = frozenset( + {"turnhout", "retie", "westerlo", "arendonk_heide"} +) +DEFAULT_VALIDATION_SAMPLES = ",".join(sorted(DEFAULT_VALIDATION_SAMPLE_SLUGS)) rasterio: Any = None Window: Any = None Transformer: Any = None @@ -67,8 +71,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--stride", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_STRIDE", "128"))) parser.add_argument( "--val-samples", - default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"), - help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.", + default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", DEFAULT_VALIDATION_SAMPLES), + help=( + "Comma/space separated sample slugs assigned to validation. " + "Defaults to the documented Turnhout, Retie, Westerlo and Arendonk-heide holdouts." + ), ) parser.add_argument( "--negative-keep-ratio", @@ -145,6 +152,35 @@ def split_slugs(raw: str) -> set[str]: return {value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()} +def validate_validation_split(samples: list[dict[str, Any]], val_slugs: set[str]) -> set[str]: + sample_slugs = { + str(sample.get("sample_slug") or "").strip().lower() + for sample in samples + if str(sample.get("sample_slug") or "").strip() + } + if not val_slugs: + raise SystemExit("YOLO validation split must include at least one sample") + unknown = val_slugs - sample_slugs + if unknown: + raise SystemExit( + "YOLO validation split references unknown samples: " + ", ".join(sorted(unknown)) + ) + recommended_holdouts = { + str(sample.get("sample_slug") or "").strip().lower() + for sample in samples + if str(sample.get("recommended_split") or "").strip().lower() == "val" + } + missing_holdouts = recommended_holdouts - val_slugs + if missing_holdouts: + raise SystemExit( + "YOLO validation split omits recommended validation holdouts: " + + ", ".join(sorted(missing_holdouts)) + ) + if not sample_slugs - val_slugs: + raise SystemExit("YOLO validation split leaves no training samples") + return val_slugs + + def edge_starts(length: int, tile_size: int, stride: int) -> list[int]: if tile_size <= 0: raise ValueError("tile_size must be positive") @@ -367,6 +403,7 @@ def export_sample_tiles( sample_slug = str(sample["sample_slug"]) sample_role = str(sample.get("sample_role") or "reference") background_category = background_category_for_sample(sample) + recommended_split = str(sample.get("recommended_split") or "") split = "val" if sample_slug.lower() in val_slugs else "train" raster_path = resolve_manifest_path(str(sample["raster_path"]), manifest_path) reference_path = resolve_manifest_path(str(sample["reference_path"]), manifest_path) @@ -390,6 +427,9 @@ def export_sample_tiles( exported.append( { "sample_slug": sample_slug, + "sample_role": sample_role, + "background_category": background_category, + "recommended_split": recommended_split, "split": split, "tile_index": tile_index, "kept": False, @@ -410,6 +450,7 @@ def export_sample_tiles( "sample_slug": sample_slug, "sample_role": sample_role, "background_category": background_category, + "recommended_split": recommended_split, "split": split, "tile_index": tile_index, "kept": False, @@ -446,6 +487,7 @@ def export_sample_tiles( "sample_slug": sample_slug, "sample_role": sample_role, "background_category": background_category, + "recommended_split": recommended_split, "split": split, "tile_index": tile_index, "repeat_index": repeat_index, @@ -479,7 +521,7 @@ def main() -> int: samples = manifest.get("samples") or [] if not samples: raise SystemExit("Operator sample manifest contains no samples") - val_slugs = split_slugs(args.val_samples) + val_slugs = validate_validation_split(samples, split_slugs(args.val_samples)) exported_tiles: list[dict[str, Any]] = [] for sample in samples: exported_tiles.extend( @@ -527,6 +569,7 @@ def main() -> int: "drop_low_variance_negatives": args.drop_low_variance_negatives, "blank_range_threshold": args.blank_range_threshold, "source_sample_count": len(samples), + "validation_sample_slugs": sorted(val_slugs), "tile_count": len(kept_tiles), "positive_tile_count": len(positive_tiles), "negative_tile_count": len(negative_tiles), diff --git a/scripts/prepare_operator_real_data_samples.py b/scripts/prepare_operator_real_data_samples.py index 3b7de9c1..0301af68 100644 --- a/scripts/prepare_operator_real_data_samples.py +++ b/scripts/prepare_operator_real_data_samples.py @@ -25,6 +25,12 @@ DEFAULT_GRB_MAX_FEATURES = 100000 REFERENCE_AOI_CATEGORY = "reference_aoi" PURE_EMPTY_BACKGROUND_CATEGORY = "pure_empty_negative" SPARSE_BACKGROUND_CATEGORY = "sparse_building_context" +TRAINING_EXPANSION_SAMPLE_SLUGS = frozenset( + {"olen_center", "lille_center", "oud_turnhout_center", "kasterlee_center"} +) +DEFAULT_VALIDATION_SAMPLE_SLUGS = frozenset( + {"turnhout", "retie", "westerlo", "arendonk_heide"} +) requests: Any = None rasterio: Any = None Transformer: Any = None @@ -92,6 +98,30 @@ SAMPLES: dict[str, OperatorSample] = { center_lat=51.0909, half_size_m=220.0, ), + "olen_center": OperatorSample( + slug="olen_center", + display_name="Olen center training expansion", + center_lon=4.8597257, + center_lat=51.1438611, + ), + "lille_center": OperatorSample( + slug="lille_center", + display_name="Lille center training expansion", + center_lon=4.8242404, + center_lat=51.2382180, + ), + "oud_turnhout_center": OperatorSample( + slug="oud_turnhout_center", + display_name="Oud-Turnhout center training expansion", + center_lon=4.9817086, + center_lat=51.3178319, + ), + "kasterlee_center": OperatorSample( + slug="kasterlee_center", + display_name="Kasterlee center training expansion", + center_lon=4.9678120, + center_lat=51.2407915, + ), "postel_bos": OperatorSample( slug="postel_bos", display_name="Postel forest background candidate", @@ -270,6 +300,10 @@ def background_category_for_sample(sample: OperatorSample, reference_feature_cou return PURE_EMPTY_BACKGROUND_CATEGORY if reference_feature_count <= 0 else SPARSE_BACKGROUND_CATEGORY +def recommended_split_for_sample(sample: OperatorSample) -> str: + return "val" if sample.slug in DEFAULT_VALIDATION_SAMPLE_SLUGS else "train" + + def apply_sample_overrides( sample: OperatorSample, *, @@ -481,6 +515,7 @@ def fetch_reference( reference["sample_role"] = sample.sample_role reference["allow_empty_reference"] = sample.allow_empty_reference reference["background_category"] = background_category_for_sample(sample, len(features)) + reference["recommended_split"] = recommended_split_for_sample(sample) reference["reference_page_limit"] = page_limit reference["reference_max_features"] = max_features reference["reference_pages_fetched"] = len(pages) @@ -495,6 +530,7 @@ def fetch_reference( props.setdefault("sample_slug", sample.slug) props.setdefault("sample_role", sample.sample_role) props.setdefault("background_category", background_category_for_sample(sample, len(features))) + props.setdefault("recommended_split", recommended_split_for_sample(sample)) reference_path.write_text(json.dumps(reference, ensure_ascii=False), encoding="utf-8") return prepared_url(GRB_GBG_URL, ogc_params), len(features) @@ -537,6 +573,7 @@ def prepare_sample( "sample_role": sample.sample_role, "allow_empty_reference": sample.allow_empty_reference, "background_category": background_category, + "recommended_split": recommended_split_for_sample(sample), "raster_path": str(ortho_path), "reference_path": str(reference_path), "reference_feature_count": reference_feature_count, @@ -572,7 +609,8 @@ def write_readme(output_dir: Path, samples: list[dict[str, Any]]) -> None: f"- `{sample['sample_slug']}`: `{Path(sample['raster_path']).name}` and " f"`{Path(sample['reference_path']).name}`, " f"{sample['reference_feature_count']} reference features, role `{sample['sample_role']}`, " - f"background category `{sample['background_category']}`." + f"background category `{sample['background_category']}`, " + f"recommended split `{sample['recommended_split']}`." ) lines.append("") lines.append("Purpose: configured-YOLO detection + persisted QA/QC validation with operator-provided files.") @@ -602,7 +640,7 @@ def main() -> int: write_readme(output_dir, samples) manifest = { - "schema_version": 1, + "schema_version": 2, "description": "GeoIntel operator real-data samples for configured-YOLO QA validation.", "output_dir": str(output_dir), "sample_width": args.width, @@ -610,6 +648,8 @@ def main() -> int: "half_size_scale": args.half_size_scale, "reference_page_limit": args.reference_page_limit, "reference_max_features": args.reference_max_features, + "default_validation_sample_slugs": sorted(DEFAULT_VALIDATION_SAMPLE_SLUGS), + "training_expansion_sample_slugs": sorted(TRAINING_EXPANSION_SAMPLE_SLUGS), "samples": samples, } manifest_path = output_dir / args.manifest_name