Initial public release

scripts/export_experimental_yolo_train_shard.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""Export an isolated train-only YOLO shard without making a release claim.
+
+This path exists for evidence-generating model experiments when source contracts
+are eligible but accepted human label review is still pending.  It deliberately
+cannot create a training release and must never be used for model promotion.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import shutil
+import sys
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+
+from export_operator_yolo_tile_dataset import (  # noqa: E402
+    ensure_dependencies,
+    ensure_yolo_directories,
+    export_sample_tiles,
+    file_sha256,
+    write_dataset_yaml,
+)
+from training_dataset_eligibility import (  # noqa: E402
+    TrainingEligibilityError,
+    assert_frozen_manifest_training_eligible,
+)
+
+EXPERIMENTAL_ROOT = Path("/app/storage/training/experimental")
+
+
+def _inside(path: Path, root: Path) -> bool:
+    try:
+        path.resolve().relative_to(root.resolve())
+        return True
+    except ValueError:
+        return False
+
+
+def validate_experimental_request(manifest_path: Path, output_dir: Path) -> dict:
+    if not _inside(output_dir, EXPERIMENTAL_ROOT):
+        raise ValueError(f"output must remain below {EXPERIMENTAL_ROOT}")
+    if (manifest_path.parent / "NO_TRAINING.json").exists():
+        raise ValueError("source corpus explicitly prohibits training")
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
+    if manifest.get("purpose") != "training_corpus":
+        raise ValueError("source manifest is not a training corpus")
+    samples = manifest.get("samples") or []
+    if not samples:
+        raise ValueError("source manifest contains no samples")
+    invalid_splits = sorted(
+        {
+            str(sample.get("split") or "").strip().lower()
+            for sample in samples
+            if str(sample.get("split") or "").strip().lower() != "train"
+        }
+    )
+    if invalid_splits:
+        raise ValueError(f"experimental shard accepts train samples only: {invalid_splits}")
+    return manifest
+
+
+def is_canonical_train_window(window: dict, tile_size: int) -> bool:
+    return bool(
+        int(window["row_off"]) % tile_size == 0
+        and int(window["col_off"]) % tile_size == 0
+        and int(window["height"]) == tile_size
+        and int(window["width"]) == tile_size
+    )
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--manifest-path", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--tile-size", type=int, default=512)
+    parser.add_argument("--stride", type=int, default=512)
+    parser.add_argument("--negative-keep-ratio", type=float, default=1.0)
+    parser.add_argument("--min-label-px", type=float, default=4.0)
+    parser.add_argument("--min-label-visible-ratio", type=float, default=0.25)
+    parser.add_argument("--force", action="store_true")
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    try:
+        manifest = validate_experimental_request(args.manifest_path, args.output_dir)
+        assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
+    except (ValueError, TrainingEligibilityError) as exc:
+        raise SystemExit(str(exc)) from exc
+    ensure_dependencies()
+    if args.force and args.output_dir.exists():
+        shutil.rmtree(args.output_dir)
+    if args.output_dir.exists() and any(args.output_dir.iterdir()):
+        raise SystemExit("experimental output directory already exists and is not empty; use --force")
+    ensure_yolo_directories(args.output_dir)
+    tiles: list[dict] = []
+    for sample in manifest["samples"]:
+        tiles.extend(
+            export_sample_tiles(
+                sample=sample,
+                manifest_path=args.manifest_path,
+                output_dir=args.output_dir,
+                val_slugs=set(),
+                tile_size=args.tile_size,
+                stride=args.stride,
+                negative_keep_ratio=args.negative_keep_ratio,
+                min_label_px=args.min_label_px,
+                min_label_visible_ratio=args.min_label_visible_ratio,
+                background_negative_repeat=1,
+                drop_low_variance_negatives=False,
+                blank_range_threshold=3,
+                reference_source="grb",
+                reference_layer="buildings",
+            )
+        )
+    kept: list[dict] = []
+    excluded_edge_cover: list[dict] = []
+    for tile in tiles:
+        if not tile["kept"]:
+            continue
+        if is_canonical_train_window(tile["window"], args.tile_size):
+            kept.append(tile)
+            continue
+        excluded_edge_cover.append(
+            {
+                "sample_slug": tile["sample_slug"],
+                "tile_index": tile["tile_index"],
+                "window": tile["window"],
+                "reason": "overlapping_edge_cover_tile",
+            }
+        )
+        Path(tile["image_path"]).unlink(missing_ok=True)
+        Path(tile["label_path"]).unlink(missing_ok=True)
+    if not kept:
+        raise SystemExit("experimental export produced no tiles")
+    dataset_yaml = write_dataset_yaml(args.output_dir, "building")
+    # Ultralytics requires a val key while training. Its score is explicitly
+    # invalid for model selection; independent V72 evaluation is mandatory.
+    dataset_yaml.write_text(
+        dataset_yaml.read_text(encoding="utf-8").replace("val: images/val", "val: images/train"),
+        encoding="utf-8",
+    )
+    asset_hash = hashlib.sha256()
+    for tile in sorted(kept, key=lambda item: str(item["image_path"])):
+        asset_hash.update(file_sha256(Path(tile["image_path"])).encode())
+        asset_hash.update(file_sha256(Path(tile["label_path"])).encode())
+    summary = {
+        "schema_version": 1,
+        "status": "experimental_only_human_review_pending",
+        "promotion_allowed": False,
+        "release_claim_allowed": False,
+        "internal_validation_valid_for_selection": False,
+        "required_independent_evaluation": "frozen V72 calibration portfolio",
+        "source_manifest": str(args.manifest_path),
+        "source_manifest_sha256": file_sha256(args.manifest_path),
+        "dataset_yaml": str(dataset_yaml),
+        "dataset_yaml_sha256": file_sha256(dataset_yaml),
+        "tile_asset_chain_sha256": asset_hash.hexdigest(),
+        "sample_count": len(manifest["samples"]),
+        "tile_count": len(kept),
+        "positive_tile_count": sum(not tile["is_negative"] for tile in kept),
+        "negative_tile_count": sum(tile["is_negative"] for tile in kept),
+        "excluded_overlapping_edge_cover_tile_count": len(excluded_edge_cover),
+        "excluded_overlapping_edge_cover_tiles": excluded_edge_cover,
+        "label_count": sum(int(tile["label_count"]) for tile in kept),
+        "tile_size": args.tile_size,
+        "stride": args.stride,
+        "min_label_px": args.min_label_px,
+        "min_label_visible_ratio": args.min_label_visible_ratio,
+        "tiles": kept,
+    }
+    summary_path = args.output_dir / "experimental_dataset_summary.json"
+    summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
+    marker = {
+        key: summary[key]
+        for key in (
+            "schema_version",
+            "status",
+            "promotion_allowed",
+            "release_claim_allowed",
+            "source_manifest_sha256",
+            "dataset_yaml_sha256",
+            "tile_asset_chain_sha256",
+        )
+    }
+    (args.output_dir / "EXPERIMENTAL_ONLY.json").write_text(
+        json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    print(json.dumps({key: value for key, value in summary.items() if key != "tiles"}, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())