Supervise checkpointed Tower training

backend/tests/test_yolo_training_supervisor.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).parents[2] / "scripts" / "supervise_container_yolo_training.py"
+SPEC = importlib.util.spec_from_file_location("yolo_supervisor", SCRIPT)
+assert SPEC and SPEC.loader
+MODULE = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MODULE)
+
+
+def test_training_active_requires_yolo_train_and_exact_marker(monkeypatch) -> None:
+    class Result:
+        returncode = 0
+        stdout = "python api.py\npython3 /opt/venv/bin/yolo train resume=/runs/v37/weights/last.pt device=0\n"
+
+    monkeypatch.setattr(MODULE.subprocess, "run", lambda *args, **kwargs: Result())
+    assert MODULE.training_active("geointel", "/runs/v37") is True
+    assert MODULE.training_active("geointel", "/runs/v38") is False
+
+
+def test_container_running_fails_closed_on_inspect_error(monkeypatch) -> None:
+    class Result:
+        returncode = 1
+        stdout = ""
+
+    monkeypatch.setattr(MODULE.subprocess, "run", lambda *args, **kwargs: Result())
+    assert MODULE.container_running("missing") is False

docs/CODEX_EXECUTION_LOG.md

@@ -11782,6 +11782,18 @@ Deployment evidence:
   supplied weights, runs calibration first, and rejoins the same automatic
   sampling/training path after rejection. Protected evidence remains closed
   until calibration passes.
+- The all-in-one container was externally recreated after v37 epoch 1. Both
+  456 MB checkpoints remained intact and training resumed from `last.pt` on
+  the RTX 4080 instead of restarting the experiment.
+- Added and activated a host-side YOLO supervisor. It requires the exact run
+  marker, a valid checkpoint larger than 1 MB, a running target container and
+  absence of `results.png` before issuing a bounded resume. It exits on a
+  completed training artifact or a missing/incomplete checkpoint.
+- The first live probe exposed that `docker top -eo args` is rejected by the
+  daemon and could misclassify an active Python-launched YOLO process. Two
+  transient duplicate resume processes were detected and terminated before
+  another epoch completed. Detection now uses `docker top -eo pid,args`; a
+  live one-shot check returned `monitoring` with exactly one GPU process.
 - Confirmed v37 epoch 1 completed on CUDA with validation precision `0.601`,
   recall `0.455`, mAP50 `0.474` and mAP50-95 `0.205`; the run remains inactive
   and these internal-validation metrics are not release evidence.
@@ -11792,6 +11804,8 @@ Verified in this pass:
   (`12 passed`).
 - `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_failure_driven_yolo_sampling.py backend/tests/test_belgium_training_iteration_assessment.py`
   (`16 passed` after adding the completed-checkpoint entry contract).
+- `py -3 -m pytest -q backend/tests/test_yolo_training_supervisor.py backend/tests/test_belgium_training_loop.py`
+  (`11 passed`), plus a live supervisor one-shot and single-process GPU audit.
 
 Open:
 

docs/TODO.md

@@ -974,6 +974,7 @@ This file now starts with the current implementation status. Older preparation/b
 - [x] Allow the objective CUDA loop to consume an automatically clean `needs_human_review` corpus while keeping final human sign-off as a separate, mandatory promotion gate.
 - [x] Persist checksummed train-only failure-driven sampling after every rejected loop iteration and resume the next checkpoint from that exact dataset YAML.
 - [x] Add a guarded calibration-first entry point for completed checkpoints so v37 and future externally interrupted runs can rejoin the automated loop without redundant retraining.
+- [x] Add and activate a host-side, exact-run-marker supervisor that resumes the v37 CUDA checkpoint after container recreation without launching concurrent trainers.
 - [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed.
 - [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes.
 

scripts/supervise_container_yolo_training.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Resume one checkpointed YOLO run after container recreation, fail closed."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+
+
+def container_running(container: str) -> bool:
+    result = subprocess.run(
+        ["docker", "inspect", "--format", "{{.State.Running}}", container],
+        capture_output=True, text=True, check=False,
+    )
+    return result.returncode == 0 and result.stdout.strip() == "true"
+
+
+def training_active(container: str, run_marker: str) -> bool:
+    result = subprocess.run(
+        ["docker", "top", container, "-eo", "pid,args"],
+        capture_output=True, text=True, check=False,
+    )
+    return result.returncode == 0 and any(
+        "train" in line and run_marker in line
+        for line in result.stdout.splitlines()
+    )
+
+
+def write_state(path: Path, payload: dict) -> None:
+    payload["updated_at"] = datetime.now(UTC).isoformat()
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    temporary.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    temporary.replace(path)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--container", required=True)
+    parser.add_argument("--host-run-dir", type=Path, required=True)
+    parser.add_argument("--container-checkpoint", required=True)
+    parser.add_argument("--run-marker", required=True)
+    parser.add_argument("--yolo", default="/opt/geointel/venv/bin/yolo")
+    parser.add_argument("--poll-seconds", type=int, default=30)
+    parser.add_argument("--max-resumes", type=int, default=20)
+    parser.add_argument("--once", action="store_true")
+    args = parser.parse_args()
+    if args.poll_seconds < 1 or args.max_resumes < 1:
+        raise SystemExit("poll-seconds and max-resumes must be positive")
+
+    state_path = args.host_run_dir / "supervisor-state.json"
+    state = {"schema_version": 1, "status": "monitoring", "resume_count": 0}
+    if state_path.is_file():
+        state.update(json.loads(state_path.read_text(encoding="utf-8")))
+
+    while True:
+        if (args.host_run_dir / "results.png").is_file():
+            state["status"] = "training_finished"
+            write_state(state_path, state)
+            return 0
+        checkpoint = args.host_run_dir / "weights" / "last.pt"
+        if not checkpoint.is_file() or checkpoint.stat().st_size < 1024 * 1024:
+            state["status"] = "checkpoint_missing_or_incomplete"
+            write_state(state_path, state)
+            return 2
+        if container_running(args.container) and not training_active(args.container, args.run_marker):
+            if int(state["resume_count"]) >= args.max_resumes:
+                state["status"] = "resume_budget_exhausted"
+                write_state(state_path, state)
+                return 3
+            result = subprocess.run(
+                ["docker", "exec", "-d", args.container, args.yolo, "train",
+                 f"resume={args.container_checkpoint}", "device=0"],
+                check=False,
+            )
+            if result.returncode == 0:
+                state["resume_count"] = int(state["resume_count"]) + 1
+                state["status"] = "resumed"
+            else:
+                state["status"] = "resume_command_failed"
+            write_state(state_path, state)
+        else:
+            state["status"] = "monitoring"
+            write_state(state_path, state)
+        if args.once:
+            return 0
+        time.sleep(args.poll_seconds)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())