202 lines
7.8 KiB
Python
202 lines
7.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Export an isolated train-only YOLO shard without making a release claim.
|
|
|
|
This path exists for evidence-generating model experiments when source contracts
|
|
are eligible but accepted human label review is still pending. It deliberately
|
|
cannot create a training release and must never be used for model promotion.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
if str(SCRIPT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPT_DIR))
|
|
|
|
from export_operator_yolo_tile_dataset import ( # noqa: E402
|
|
ensure_dependencies,
|
|
ensure_yolo_directories,
|
|
export_sample_tiles,
|
|
file_sha256,
|
|
write_dataset_yaml,
|
|
)
|
|
from training_dataset_eligibility import ( # noqa: E402
|
|
TrainingEligibilityError,
|
|
assert_frozen_manifest_training_eligible,
|
|
)
|
|
|
|
EXPERIMENTAL_ROOT = Path("/app/storage/training/experimental")
|
|
|
|
|
|
def _inside(path: Path, root: Path) -> bool:
|
|
try:
|
|
path.resolve().relative_to(root.resolve())
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def validate_experimental_request(manifest_path: Path, output_dir: Path) -> dict:
|
|
if not _inside(output_dir, EXPERIMENTAL_ROOT):
|
|
raise ValueError(f"output must remain below {EXPERIMENTAL_ROOT}")
|
|
if (manifest_path.parent / "NO_TRAINING.json").exists():
|
|
raise ValueError("source corpus explicitly prohibits training")
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
|
|
if manifest.get("purpose") != "training_corpus":
|
|
raise ValueError("source manifest is not a training corpus")
|
|
samples = manifest.get("samples") or []
|
|
if not samples:
|
|
raise ValueError("source manifest contains no samples")
|
|
invalid_splits = sorted(
|
|
{
|
|
str(sample.get("split") or "").strip().lower()
|
|
for sample in samples
|
|
if str(sample.get("split") or "").strip().lower() != "train"
|
|
}
|
|
)
|
|
if invalid_splits:
|
|
raise ValueError(f"experimental shard accepts train samples only: {invalid_splits}")
|
|
return manifest
|
|
|
|
|
|
def is_canonical_train_window(window: dict, tile_size: int) -> bool:
|
|
return bool(
|
|
int(window["row_off"]) % tile_size == 0
|
|
and int(window["col_off"]) % tile_size == 0
|
|
and int(window["height"]) == tile_size
|
|
and int(window["width"]) == tile_size
|
|
)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--manifest-path", type=Path, required=True)
|
|
parser.add_argument("--output-dir", type=Path, required=True)
|
|
parser.add_argument("--tile-size", type=int, default=512)
|
|
parser.add_argument("--stride", type=int, default=512)
|
|
parser.add_argument("--negative-keep-ratio", type=float, default=1.0)
|
|
parser.add_argument("--min-label-px", type=float, default=4.0)
|
|
parser.add_argument("--min-label-visible-ratio", type=float, default=0.25)
|
|
parser.add_argument("--force", action="store_true")
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
try:
|
|
manifest = validate_experimental_request(args.manifest_path, args.output_dir)
|
|
assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
|
|
except (ValueError, TrainingEligibilityError) as exc:
|
|
raise SystemExit(str(exc)) from exc
|
|
ensure_dependencies()
|
|
if args.force and args.output_dir.exists():
|
|
shutil.rmtree(args.output_dir)
|
|
if args.output_dir.exists() and any(args.output_dir.iterdir()):
|
|
raise SystemExit("experimental output directory already exists and is not empty; use --force")
|
|
ensure_yolo_directories(args.output_dir)
|
|
tiles: list[dict] = []
|
|
for sample in manifest["samples"]:
|
|
tiles.extend(
|
|
export_sample_tiles(
|
|
sample=sample,
|
|
manifest_path=args.manifest_path,
|
|
output_dir=args.output_dir,
|
|
val_slugs=set(),
|
|
tile_size=args.tile_size,
|
|
stride=args.stride,
|
|
negative_keep_ratio=args.negative_keep_ratio,
|
|
min_label_px=args.min_label_px,
|
|
min_label_visible_ratio=args.min_label_visible_ratio,
|
|
background_negative_repeat=1,
|
|
drop_low_variance_negatives=False,
|
|
blank_range_threshold=3,
|
|
reference_source="grb",
|
|
reference_layer="buildings",
|
|
)
|
|
)
|
|
kept: list[dict] = []
|
|
excluded_edge_cover: list[dict] = []
|
|
for tile in tiles:
|
|
if not tile["kept"]:
|
|
continue
|
|
if is_canonical_train_window(tile["window"], args.tile_size):
|
|
kept.append(tile)
|
|
continue
|
|
excluded_edge_cover.append(
|
|
{
|
|
"sample_slug": tile["sample_slug"],
|
|
"tile_index": tile["tile_index"],
|
|
"window": tile["window"],
|
|
"reason": "overlapping_edge_cover_tile",
|
|
}
|
|
)
|
|
Path(tile["image_path"]).unlink(missing_ok=True)
|
|
Path(tile["label_path"]).unlink(missing_ok=True)
|
|
if not kept:
|
|
raise SystemExit("experimental export produced no tiles")
|
|
dataset_yaml = write_dataset_yaml(args.output_dir, "building")
|
|
# Ultralytics requires a val key while training. Its score is explicitly
|
|
# invalid for model selection; independent V72 evaluation is mandatory.
|
|
dataset_yaml.write_text(
|
|
dataset_yaml.read_text(encoding="utf-8").replace("val: images/val", "val: images/train"),
|
|
encoding="utf-8",
|
|
)
|
|
asset_hash = hashlib.sha256()
|
|
for tile in sorted(kept, key=lambda item: str(item["image_path"])):
|
|
asset_hash.update(file_sha256(Path(tile["image_path"])).encode())
|
|
asset_hash.update(file_sha256(Path(tile["label_path"])).encode())
|
|
summary = {
|
|
"schema_version": 1,
|
|
"status": "experimental_only_human_review_pending",
|
|
"promotion_allowed": False,
|
|
"release_claim_allowed": False,
|
|
"internal_validation_valid_for_selection": False,
|
|
"required_independent_evaluation": "frozen V72 calibration portfolio",
|
|
"source_manifest": str(args.manifest_path),
|
|
"source_manifest_sha256": file_sha256(args.manifest_path),
|
|
"dataset_yaml": str(dataset_yaml),
|
|
"dataset_yaml_sha256": file_sha256(dataset_yaml),
|
|
"tile_asset_chain_sha256": asset_hash.hexdigest(),
|
|
"sample_count": len(manifest["samples"]),
|
|
"tile_count": len(kept),
|
|
"positive_tile_count": sum(not tile["is_negative"] for tile in kept),
|
|
"negative_tile_count": sum(tile["is_negative"] for tile in kept),
|
|
"excluded_overlapping_edge_cover_tile_count": len(excluded_edge_cover),
|
|
"excluded_overlapping_edge_cover_tiles": excluded_edge_cover,
|
|
"label_count": sum(int(tile["label_count"]) for tile in kept),
|
|
"tile_size": args.tile_size,
|
|
"stride": args.stride,
|
|
"min_label_px": args.min_label_px,
|
|
"min_label_visible_ratio": args.min_label_visible_ratio,
|
|
"tiles": kept,
|
|
}
|
|
summary_path = args.output_dir / "experimental_dataset_summary.json"
|
|
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
|
|
marker = {
|
|
key: summary[key]
|
|
for key in (
|
|
"schema_version",
|
|
"status",
|
|
"promotion_allowed",
|
|
"release_claim_allowed",
|
|
"source_manifest_sha256",
|
|
"dataset_yaml_sha256",
|
|
"tile_asset_chain_sha256",
|
|
)
|
|
}
|
|
(args.output_dir / "EXPERIMENTAL_ONLY.json").write_text(
|
|
json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
|
)
|
|
print(json.dumps({key: value for key, value in summary.items() if key != "tiles"}, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|