Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make building training loop calibration-gated

Browse Source
This commit is contained in:
Jens
2026-07-27 09:29:07 +02:00
parent 8155acf969
commit 1c6cf6a6a8
3 changed files with 168 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build_regional_yolo_expert_dataset.py
+20 -8
View File
@@ -25,28 +25,38 @@ def main() -> int:
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--positive-repeat", type=int, default=2)
parser.add_argument("--negative-repeat", type=int, default=1)
parser.add_argument(
"--other-region-repeat",
type=int,
default=0,
help="Include each train tile outside the expert region this many times.",
)
args = parser.parse_args()
if args.positive_repeat < 1 or args.negative_repeat < 1:
raise SystemExit("repeat factors must be positive")
if args.positive_repeat < 1 or args.negative_repeat < 1 or args.other_region_repeat < 0:
raise SystemExit("regional repeats must be positive and other-region repeat non-negative")
summary = json.loads(args.summary.read_text(encoding="utf-8"))
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
samples = {item["sample_slug"]: item for item in manifest["samples"]}
paths: list[str] = []
selected_samples: set[str] = set()
positive_tiles = negative_tiles = 0
positive_tiles = negative_tiles = other_region_tiles = 0
for tile in summary["tiles"]:
sample = samples[tile["sample_slug"]]
if sample["split"] != "train" or tile["split"] != "train":
continue
if sample["region"] != args.region:
continue
positive = int(tile.get("label_count") or 0) > 0
repeat = args.positive_repeat if positive else args.negative_repeat
if sample["region"] == args.region:
repeat = args.positive_repeat if positive else args.negative_repeat
positive_tiles += int(positive)
negative_tiles += int(not positive)
else:
repeat = args.other_region_repeat
other_region_tiles += int(repeat > 0)
if repeat == 0:
continue
paths.extend([str(Path(tile["image_path"]).resolve())] * repeat)
selected_samples.add(tile["sample_slug"])
positive_tiles += int(positive)
negative_tiles += int(not positive)
if not paths or not positive_tiles:
raise SystemExit(f"no positive train tiles found for region {args.region!r}")
@@ -69,8 +79,10 @@ def main() -> int:
"corpus_manifest_sha256": sha256(args.corpus_manifest),
"positive_repeat": args.positive_repeat,
"negative_repeat": args.negative_repeat,
"other_region_repeat": args.other_region_repeat,
"source_positive_tile_count": positive_tiles,
"source_negative_tile_count": negative_tiles,
"source_other_region_tile_count": other_region_tiles,
"sampled_train_entry_count": len(paths),
"selected_train_samples": sorted(selected_samples),
"protected_samples_in_training": [],