make building results authority-first after V74 evaluation

scripts/compose_experimental_yolo_rehearsal_dataset.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""Compose legacy rehearsal and fresh remediation tiles for an experiment."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+
+from export_experimental_yolo_train_shard import EXPERIMENTAL_ROOT, file_sha256  # noqa: E402
+
+
+def _inside(path: Path, root: Path) -> bool:
+    try:
+        path.resolve().relative_to(root.resolve())
+        return True
+    except ValueError:
+        return False
+
+
+def validate_sources(rehearsal_summary: Path, remediation_summary: Path, output_dir: Path) -> tuple[dict, dict]:
+    if not _inside(output_dir, EXPERIMENTAL_ROOT):
+        raise ValueError(f"output must remain below {EXPERIMENTAL_ROOT}")
+    rehearsal = json.loads(rehearsal_summary.read_text(encoding="utf-8"))
+    remediation = json.loads(remediation_summary.read_text(encoding="utf-8"))
+    marker_path = remediation_summary.parent / "EXPERIMENTAL_ONLY.json"
+    if not marker_path.is_file():
+        raise ValueError("fresh remediation source has no EXPERIMENTAL_ONLY marker")
+    marker = json.loads(marker_path.read_text(encoding="utf-8"))
+    if marker.get("promotion_allowed") is not False:
+        raise ValueError("fresh remediation marker does not prohibit promotion")
+    if remediation.get("source_manifest_sha256") != marker.get("source_manifest_sha256"):
+        raise ValueError("fresh remediation marker/summary binding mismatch")
+    rehearsal_tiles = [tile for tile in rehearsal.get("tiles") or [] if tile.get("kept", True)]
+    remediation_tiles = [tile for tile in remediation.get("tiles") or [] if tile.get("kept", True)]
+    if not rehearsal_tiles or not remediation_tiles:
+        raise ValueError("both rehearsal and remediation summaries require retained tiles")
+    if any(tile.get("split") not in {"train", "val"} for tile in rehearsal_tiles):
+        raise ValueError("rehearsal source contains a protected or unsupported split")
+    if any(tile.get("split") != "train" for tile in remediation_tiles):
+        raise ValueError("fresh remediation source must be train-only")
+    return rehearsal, remediation
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--rehearsal-summary", type=Path, required=True)
+    parser.add_argument("--remediation-summary", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--remediation-repeat", type=int, default=2)
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    if args.remediation_repeat < 1 or args.remediation_repeat > 4:
+        raise SystemExit("remediation-repeat must be between 1 and 4")
+    if args.output_dir.exists():
+        raise SystemExit(f"refusing to overwrite rehearsal dataset: {args.output_dir}")
+    try:
+        rehearsal, remediation = validate_sources(
+            args.rehearsal_summary, args.remediation_summary, args.output_dir
+        )
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        raise SystemExit(str(exc)) from exc
+    rehearsal_yaml = Path(str(rehearsal["dataset_yaml"]))
+    remediation_yaml = Path(str(remediation["dataset_yaml"]))
+    rehearsal_root = rehearsal_yaml.parent
+    remediation_root = remediation_yaml.parent
+    for directory in (
+        rehearsal_root / "images" / "train",
+        rehearsal_root / "images" / "val",
+        remediation_root / "images" / "train",
+    ):
+        if not directory.is_dir():
+            raise SystemExit(f"rehearsal dataset directory is missing: {directory}")
+    args.output_dir.mkdir(parents=True)
+    train_sources = [str(rehearsal_root / "images" / "train")] + [
+        str(remediation_root / "images" / "train")
+    ] * args.remediation_repeat
+    yaml_path = args.output_dir / "dataset.yaml"
+    yaml_path.write_text(
+        "\n".join(
+            [
+                "path: /",
+                "train:",
+                *[f"  - {path}" for path in train_sources],
+                f"val: {rehearsal_root / 'images' / 'val'}",
+                "names:",
+                "  0: building",
+                "",
+            ]
+        ),
+        encoding="utf-8",
+    )
+    summary = {
+        "schema_version": 1,
+        "status": "experimental_rehearsal_only",
+        "promotion_allowed": False,
+        "release_claim_allowed": False,
+        "human_review_pending": True,
+        "dataset_yaml": str(yaml_path),
+        "dataset_yaml_sha256": file_sha256(yaml_path),
+        "rehearsal_summary": str(args.rehearsal_summary),
+        "rehearsal_summary_sha256": file_sha256(args.rehearsal_summary),
+        "remediation_summary": str(args.remediation_summary),
+        "remediation_summary_sha256": file_sha256(args.remediation_summary),
+        "remediation_repeat": args.remediation_repeat,
+        "rehearsal_train_tile_count": int(rehearsal.get("train_tile_count") or 0),
+        "remediation_unique_train_tile_count": int(remediation.get("tile_count") or 0),
+        "protected_v72_used": False,
+    }
+    (args.output_dir / "experimental_rehearsal_summary.json").write_text(
+        json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    marker = {
+        key: summary[key]
+        for key in (
+            "schema_version",
+            "status",
+            "promotion_allowed",
+            "release_claim_allowed",
+            "dataset_yaml_sha256",
+            "rehearsal_summary_sha256",
+            "remediation_summary_sha256",
+        )
+    }
+    (args.output_dir / "EXPERIMENTAL_ONLY.json").write_text(
+        json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    print(json.dumps(summary, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/train_experimental_building_proposal_filter.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+"""Train a non-promotable second-stage building/background proposal filter."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import shutil
+import sys
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+
+from train_building_proposal_filter import (  # noqa: E402
+    choose_threshold,
+    materialize_split,
+    sha256,
+)
+
+EXPERIMENTAL_ROOT = Path("/app/storage/training/experimental")
+
+
+def _inside(path: Path, root: Path = EXPERIMENTAL_ROOT) -> bool:
+    try:
+        path.resolve().relative_to(root.resolve())
+        return True
+    except ValueError:
+        return False
+
+
+def validate_inputs(summary_path: Path, output_dir: Path) -> dict:
+    if not _inside(summary_path) or not _inside(output_dir):
+        raise ValueError(f"summary and output must remain below {EXPERIMENTAL_ROOT}")
+    marker_path = summary_path.parent / "EXPERIMENTAL_ONLY.json"
+    if not marker_path.is_file():
+        raise ValueError("experimental source marker is missing")
+    marker = json.loads(marker_path.read_text(encoding="utf-8"))
+    if marker.get("promotion_allowed") is not False or marker.get("release_claim_allowed") is not False:
+        raise ValueError("experimental marker does not prohibit promotion")
+    summary = json.loads(summary_path.read_text(encoding="utf-8"))
+    if marker.get("dataset_yaml_sha256") != summary.get("dataset_yaml_sha256"):
+        raise ValueError("experimental marker/summary binding mismatch")
+    if summary.get("status") != "experimental_only_human_review_pending":
+        raise ValueError("source summary is not an experimental train shard")
+    return summary
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--summary", type=Path, required=True)
+    parser.add_argument("--proposal-model", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--val-sample", action="append", required=True)
+    parser.add_argument("--device", default="cuda:0")
+    parser.add_argument("--proposal-confidence", type=float, default=0.01)
+    parser.add_argument("--crop-scale", type=float, default=1.4)
+    parser.add_argument("--negative-match-iou", type=float, default=0.05)
+    parser.add_argument("--max-positive-per-tile", type=int, default=40)
+    parser.add_argument("--max-negative-per-tile", type=int, default=40)
+    parser.add_argument("--epochs", type=int, default=8)
+    parser.add_argument("--batch", type=int, default=64)
+    parser.add_argument("--seed", type=int, default=74)
+    parser.add_argument("--force", action="store_true")
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    try:
+        summary = validate_inputs(args.summary, args.output_dir)
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        raise SystemExit(str(exc)) from exc
+    if not args.proposal_model.is_file():
+        raise SystemExit(f"proposal model is unavailable: {args.proposal_model}")
+    val_samples = {slug.strip() for slug in args.val_sample if slug.strip()}
+    sample_slugs = {str(tile["sample_slug"]) for tile in summary["tiles"]}
+    unknown = val_samples - sample_slugs
+    if unknown:
+        raise SystemExit(f"unknown validation samples: {', '.join(sorted(unknown))}")
+    if not val_samples or val_samples == sample_slugs:
+        raise SystemExit("proposal filter requires disjoint non-empty train and validation AOIs")
+    if args.output_dir.exists():
+        if not args.force:
+            raise SystemExit(f"output already exists: {args.output_dir}")
+        shutil.rmtree(args.output_dir)
+
+    random.seed(args.seed)
+    import torch
+    from torch import nn
+    from torch.utils.data import DataLoader
+    from torchvision import datasets
+    from torchvision.models import ResNet18_Weights, resnet18
+    from ultralytics import YOLO
+
+    if not torch.cuda.is_available() or not args.device.startswith("cuda"):
+        raise SystemExit("CUDA is required for experimental proposal-filter training")
+    torch.manual_seed(args.seed)
+    tiles = [tile for tile in summary["tiles"] if tile.get("kept", True)]
+    split_tiles = {
+        "val": [dict(tile, split="val") for tile in tiles if tile["sample_slug"] in val_samples],
+        "train": [dict(tile, split="train") for tile in tiles if tile["sample_slug"] not in val_samples],
+    }
+    proposal_model = YOLO(str(args.proposal_model))
+    counts: dict[str, dict[str, int]] = {}
+    for split, selected in split_tiles.items():
+        results = proposal_model.predict(
+            [tile["image_path"] for tile in selected],
+            conf=args.proposal_confidence,
+            imgsz=640,
+            max_det=1000,
+            device=args.device,
+            verbose=False,
+        )
+        counts[split] = materialize_split(
+            tiles=selected,
+            results=results,
+            output_dir=args.output_dir / "crops",
+            split=split,
+            crop_scale=args.crop_scale,
+            max_positives_per_tile=args.max_positive_per_tile,
+            max_negatives_per_tile=args.max_negative_per_tile,
+            negative_match_iou=args.negative_match_iou,
+        )
+        del results
+        torch.cuda.empty_cache()
+    del proposal_model
+    torch.cuda.empty_cache()
+    if any(counts[split][label] == 0 for split in ("train", "val") for label in ("positive", "negative")):
+        raise SystemExit(f"proposal crop class is empty: {counts}")
+
+    weights = ResNet18_Weights.DEFAULT
+    transform = weights.transforms()
+    data = {
+        split: datasets.ImageFolder(args.output_dir / "crops" / split, transform=transform)
+        for split in ("train", "val")
+    }
+    loaders = {
+        split: DataLoader(dataset, batch_size=args.batch, shuffle=split == "train", num_workers=0)
+        for split, dataset in data.items()
+    }
+    model = resnet18(weights=weights)
+    model.fc = nn.Linear(model.fc.in_features, 1)
+    model.to(args.device)
+    positives = counts["train"]["positive"]
+    negatives = counts["train"]["negative"]
+    criterion = nn.BCEWithLogitsLoss(
+        pos_weight=torch.tensor([negatives / positives], device=args.device)
+    )
+    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=1e-4)
+    history = []
+    for epoch in range(1, args.epochs + 1):
+        model.train()
+        loss_sum = 0.0
+        for images, labels in loaders["train"]:
+            images = images.to(args.device)
+            labels = labels.float().to(args.device)
+            optimizer.zero_grad(set_to_none=True)
+            loss = criterion(model(images).flatten(), labels)
+            loss.backward()
+            optimizer.step()
+            loss_sum += float(loss.detach()) * len(images)
+        row = {"epoch": epoch, "train_loss": loss_sum / len(data["train"])}
+        history.append(row)
+        print(json.dumps(row), flush=True)
+
+    model.eval()
+    probabilities: list[float] = []
+    labels: list[int] = []
+    with torch.inference_mode():
+        for images, batch_labels in loaders["val"]:
+            probabilities.extend(torch.sigmoid(model(images.to(args.device)).flatten()).cpu().tolist())
+            labels.extend(batch_labels.tolist())
+    threshold, validation_metrics = choose_threshold(probabilities, labels)
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    model_path = args.output_dir / "proposal-filter.experimental.torchscript.pt"
+    torch.jit.script(model.cpu()).save(str(model_path))
+    evidence = {
+        "schema_version": 1,
+        "status": "trained_experimental_only",
+        "promotion_allowed": False,
+        "release_claim_allowed": False,
+        "human_review_pending": True,
+        "architecture": "resnet18_binary_proposal_filter",
+        "source_summary": str(args.summary),
+        "source_summary_sha256": sha256(args.summary),
+        "proposal_model": str(args.proposal_model),
+        "proposal_model_sha256": sha256(args.proposal_model),
+        "train_samples": sorted(sample_slugs - val_samples),
+        "validation_samples": sorted(val_samples),
+        "counts": counts,
+        "epochs": args.epochs,
+        "history": history,
+        "selected_threshold_source": "internal_v73_validation_only",
+        "selected_threshold": threshold,
+        "validation_metrics": validation_metrics,
+        "model": str(model_path),
+        "model_sha256": sha256(model_path),
+        "required_next_gate": "independent V72 detector-plus-filter calibration",
+    }
+    (args.output_dir / "proposal-filter.experimental.json").write_text(
+        json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    print(json.dumps(evidence, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/train_experimental_yolo_candidate.py

@@ -84,6 +84,7 @@ def main() -> int:
         seed=73,
         amp=False,
         val=False,
+        optimizer="AdamW",
         lr0=args.learning_rate,
         lrf=0.1,
         freeze=args.freeze,
@@ -114,6 +115,7 @@ def main() -> int:
         "imgsz": args.imgsz,
         "batch": args.batch,
         "learning_rate": args.learning_rate,
+        "optimizer": "AdamW",
         "freeze": args.freeze,
         "workers": args.workers,
         "device": torch.cuda.get_device_name(0),