evaluate fresh Flemish remediation training

2026-08-10 03:47:51 +02:00
parent b068a5e065
commit 98188e0a44
20 changed files with 5120 additions and 0 deletions
@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""Export an isolated train-only YOLO shard without making a release claim.
+
+This path exists for evidence-generating model experiments when source contracts
+are eligible but accepted human label review is still pending.  It deliberately
+cannot create a training release and must never be used for model promotion.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import shutil
+import sys
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+
+from export_operator_yolo_tile_dataset import (  # noqa: E402
+    ensure_dependencies,
+    ensure_yolo_directories,
+    export_sample_tiles,
+    file_sha256,
+    write_dataset_yaml,
+)
+from training_dataset_eligibility import (  # noqa: E402
+    TrainingEligibilityError,
+    assert_frozen_manifest_training_eligible,
+)
+
+EXPERIMENTAL_ROOT = Path("/app/storage/training/experimental")
+
+
+def _inside(path: Path, root: Path) -> bool:
+    try:
+        path.resolve().relative_to(root.resolve())
+        return True
+    except ValueError:
+        return False
+
+
+def validate_experimental_request(manifest_path: Path, output_dir: Path) -> dict:
+    if not _inside(output_dir, EXPERIMENTAL_ROOT):
+        raise ValueError(f"output must remain below {EXPERIMENTAL_ROOT}")
+    if (manifest_path.parent / "NO_TRAINING.json").exists():
+        raise ValueError("source corpus explicitly prohibits training")
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
+    if manifest.get("purpose") != "training_corpus":
+        raise ValueError("source manifest is not a training corpus")
+    samples = manifest.get("samples") or []
+    if not samples:
+        raise ValueError("source manifest contains no samples")
+    invalid_splits = sorted(
+        {
+            str(sample.get("split") or "").strip().lower()
+            for sample in samples
+            if str(sample.get("split") or "").strip().lower() != "train"
+        }
+    )
+    if invalid_splits:
+        raise ValueError(f"experimental shard accepts train samples only: {invalid_splits}")
+    return manifest
+
+
+def is_canonical_train_window(window: dict, tile_size: int) -> bool:
+    return bool(
+        int(window["row_off"]) % tile_size == 0
+        and int(window["col_off"]) % tile_size == 0
+        and int(window["height"]) == tile_size
+        and int(window["width"]) == tile_size
+    )
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--manifest-path", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--tile-size", type=int, default=512)
+    parser.add_argument("--stride", type=int, default=512)
+    parser.add_argument("--negative-keep-ratio", type=float, default=1.0)
+    parser.add_argument("--min-label-px", type=float, default=4.0)
+    parser.add_argument("--min-label-visible-ratio", type=float, default=0.25)
+    parser.add_argument("--force", action="store_true")
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    try:
+        manifest = validate_experimental_request(args.manifest_path, args.output_dir)
+        assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
+    except (ValueError, TrainingEligibilityError) as exc:
+        raise SystemExit(str(exc)) from exc
+    ensure_dependencies()
+    if args.force and args.output_dir.exists():
+        shutil.rmtree(args.output_dir)
+    if args.output_dir.exists() and any(args.output_dir.iterdir()):
+        raise SystemExit("experimental output directory already exists and is not empty; use --force")
+    ensure_yolo_directories(args.output_dir)
+    tiles: list[dict] = []
+    for sample in manifest["samples"]:
+        tiles.extend(
+            export_sample_tiles(
+                sample=sample,
+                manifest_path=args.manifest_path,
+                output_dir=args.output_dir,
+                val_slugs=set(),
+                tile_size=args.tile_size,
+                stride=args.stride,
+                negative_keep_ratio=args.negative_keep_ratio,
+                min_label_px=args.min_label_px,
+                min_label_visible_ratio=args.min_label_visible_ratio,
+                background_negative_repeat=1,
+                drop_low_variance_negatives=False,
+                blank_range_threshold=3,
+                reference_source="grb",
+                reference_layer="buildings",
+            )
+        )
+    kept: list[dict] = []
+    excluded_edge_cover: list[dict] = []
+    for tile in tiles:
+        if not tile["kept"]:
+            continue
+        if is_canonical_train_window(tile["window"], args.tile_size):
+            kept.append(tile)
+            continue
+        excluded_edge_cover.append(
+            {
+                "sample_slug": tile["sample_slug"],
+                "tile_index": tile["tile_index"],
+                "window": tile["window"],
+                "reason": "overlapping_edge_cover_tile",
+            }
+        )
+        Path(tile["image_path"]).unlink(missing_ok=True)
+        Path(tile["label_path"]).unlink(missing_ok=True)
+    if not kept:
+        raise SystemExit("experimental export produced no tiles")
+    dataset_yaml = write_dataset_yaml(args.output_dir, "building")
+    # Ultralytics requires a val key while training. Its score is explicitly
+    # invalid for model selection; independent V72 evaluation is mandatory.
+    dataset_yaml.write_text(
+        dataset_yaml.read_text(encoding="utf-8").replace("val: images/val", "val: images/train"),
+        encoding="utf-8",
+    )
+    asset_hash = hashlib.sha256()
+    for tile in sorted(kept, key=lambda item: str(item["image_path"])):
+        asset_hash.update(file_sha256(Path(tile["image_path"])).encode())
+        asset_hash.update(file_sha256(Path(tile["label_path"])).encode())
+    summary = {
+        "schema_version": 1,
+        "status": "experimental_only_human_review_pending",
+        "promotion_allowed": False,
+        "release_claim_allowed": False,
+        "internal_validation_valid_for_selection": False,
+        "required_independent_evaluation": "frozen V72 calibration portfolio",
+        "source_manifest": str(args.manifest_path),
+        "source_manifest_sha256": file_sha256(args.manifest_path),
+        "dataset_yaml": str(dataset_yaml),
+        "dataset_yaml_sha256": file_sha256(dataset_yaml),
+        "tile_asset_chain_sha256": asset_hash.hexdigest(),
+        "sample_count": len(manifest["samples"]),
+        "tile_count": len(kept),
+        "positive_tile_count": sum(not tile["is_negative"] for tile in kept),
+        "negative_tile_count": sum(tile["is_negative"] for tile in kept),
+        "excluded_overlapping_edge_cover_tile_count": len(excluded_edge_cover),
+        "excluded_overlapping_edge_cover_tiles": excluded_edge_cover,
+        "label_count": sum(int(tile["label_count"]) for tile in kept),
+        "tile_size": args.tile_size,
+        "stride": args.stride,
+        "min_label_px": args.min_label_px,
+        "min_label_visible_ratio": args.min_label_visible_ratio,
+        "tiles": kept,
+    }
+    summary_path = args.output_dir / "experimental_dataset_summary.json"
+    summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
+    marker = {
+        key: summary[key]
+        for key in (
+            "schema_version",
+            "status",
+            "promotion_allowed",
+            "release_claim_allowed",
+            "source_manifest_sha256",
+            "dataset_yaml_sha256",
+            "tile_asset_chain_sha256",
+        )
+    }
+    (args.output_dir / "EXPERIMENTAL_ONLY.json").write_text(
+        json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    print(json.dumps({key: value for key, value in summary.items() if key != "tiles"}, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""CUDA-train an isolated, non-promotable YOLO candidate."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import shutil
+from pathlib import Path
+
+EXPERIMENTAL_ROOT = Path("/app/storage/training/experimental")
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _inside(path: Path, root: Path) -> bool:
+    try:
+        path.resolve().relative_to(root.resolve())
+        return True
+    except ValueError:
+        return False
+
+
+def validate_paths(dataset_dir: Path, output_dir: Path) -> dict:
+    if not _inside(dataset_dir, EXPERIMENTAL_ROOT) or not _inside(output_dir, EXPERIMENTAL_ROOT):
+        raise ValueError(f"dataset and output must remain below {EXPERIMENTAL_ROOT}")
+    marker_path = dataset_dir / "EXPERIMENTAL_ONLY.json"
+    yaml_path = dataset_dir / "dataset.yaml"
+    if not marker_path.is_file() or not yaml_path.is_file():
+        raise ValueError("experimental marker or dataset.yaml is missing")
+    marker = json.loads(marker_path.read_text(encoding="utf-8"))
+    if marker.get("promotion_allowed") is not False or marker.get("release_claim_allowed") is not False:
+        raise ValueError("experimental marker does not prohibit promotion and release claims")
+    if marker.get("dataset_yaml_sha256") != sha256(yaml_path):
+        raise ValueError("experimental dataset.yaml checksum mismatch")
+    return marker
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--dataset-dir", type=Path, required=True)
+    parser.add_argument("--base-model", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--run-name", default="v73-flanders-remediation")
+    parser.add_argument("--epochs", type=int, default=20)
+    parser.add_argument("--imgsz", type=int, default=640)
+    parser.add_argument("--batch", type=int, default=8)
+    parser.add_argument("--learning-rate", type=float, default=0.0001)
+    parser.add_argument("--freeze", type=int, default=10)
+    parser.add_argument("--workers", type=int, default=0)
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    validate_paths(args.dataset_dir, args.output_dir)
+    if not args.base_model.is_file():
+        raise SystemExit(f"base model missing: {args.base_model}")
+    import torch
+    from ultralytics import YOLO
+
+    if not torch.cuda.is_available():
+        raise SystemExit("CUDA is required for GeoIntel experimental training")
+    model = YOLO(str(args.base_model))
+    result = model.train(
+        data=str(args.dataset_dir / "dataset.yaml"),
+        epochs=args.epochs,
+        imgsz=args.imgsz,
+        batch=args.batch,
+        workers=args.workers,
+        device=0,
+        project=str(args.output_dir),
+        name=args.run_name,
+        exist_ok=False,
+        pretrained=True,
+        deterministic=True,
+        seed=73,
+        amp=False,
+        val=False,
+        lr0=args.learning_rate,
+        lrf=0.1,
+        freeze=args.freeze,
+        mosaic=0.0,
+        translate=0.05,
+        scale=0.1,
+        plots=False,
+        verbose=True,
+    )
+    run_dir = Path(result.save_dir)
+    best = run_dir / "weights" / "best.pt"
+    if not best.is_file():
+        best = run_dir / "weights" / "last.pt"
+    candidate = run_dir / "candidate.experimental.pt"
+    shutil.copy2(best, candidate)
+    summary = {
+        "schema_version": 1,
+        "status": "trained_experimental_only",
+        "promotion_allowed": False,
+        "release_claim_allowed": False,
+        "human_review_pending": True,
+        "dataset_yaml_sha256": sha256(args.dataset_dir / "dataset.yaml"),
+        "base_model": str(args.base_model),
+        "base_model_sha256": sha256(args.base_model),
+        "candidate_model": str(candidate),
+        "candidate_model_sha256": sha256(candidate),
+        "epochs": args.epochs,
+        "imgsz": args.imgsz,
+        "batch": args.batch,
+        "learning_rate": args.learning_rate,
+        "freeze": args.freeze,
+        "workers": args.workers,
+        "device": torch.cuda.get_device_name(0),
+        "required_next_gate": "independent frozen V72 calibration evaluation",
+    }
+    (run_dir / "experimental_training_summary.json").write_text(
+        json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    print(json.dumps(summary, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())