diff --git a/backend/tests/test_yolo_training_supervisor.py b/backend/tests/test_yolo_training_supervisor.py new file mode 100644 index 00000000..5238381d --- /dev/null +++ b/backend/tests/test_yolo_training_supervisor.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +SCRIPT = Path(__file__).parents[2] / "scripts" / "supervise_container_yolo_training.py" +SPEC = importlib.util.spec_from_file_location("yolo_supervisor", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_training_active_requires_yolo_train_and_exact_marker(monkeypatch) -> None: + class Result: + returncode = 0 + stdout = "python api.py\npython3 /opt/venv/bin/yolo train resume=/runs/v37/weights/last.pt device=0\n" + + monkeypatch.setattr(MODULE.subprocess, "run", lambda *args, **kwargs: Result()) + assert MODULE.training_active("geointel", "/runs/v37") is True + assert MODULE.training_active("geointel", "/runs/v38") is False + + +def test_container_running_fails_closed_on_inspect_error(monkeypatch) -> None: + class Result: + returncode = 1 + stdout = "" + + monkeypatch.setattr(MODULE.subprocess, "run", lambda *args, **kwargs: Result()) + assert MODULE.container_running("missing") is False diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 5f549738..8ae5fa26 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11782,6 +11782,18 @@ Deployment evidence: supplied weights, runs calibration first, and rejoins the same automatic sampling/training path after rejection. Protected evidence remains closed until calibration passes. +- The all-in-one container was externally recreated after v37 epoch 1. Both + 456 MB checkpoints remained intact and training resumed from `last.pt` on + the RTX 4080 instead of restarting the experiment. +- Added and activated a host-side YOLO supervisor. It requires the exact run + marker, a valid checkpoint larger than 1 MB, a running target container and + absence of `results.png` before issuing a bounded resume. It exits on a + completed training artifact or a missing/incomplete checkpoint. +- The first live probe exposed that `docker top -eo args` is rejected by the + daemon and could misclassify an active Python-launched YOLO process. Two + transient duplicate resume processes were detected and terminated before + another epoch completed. Detection now uses `docker top -eo pid,args`; a + live one-shot check returned `monitoring` with exactly one GPU process. - Confirmed v37 epoch 1 completed on CUDA with validation precision `0.601`, recall `0.455`, mAP50 `0.474` and mAP50-95 `0.205`; the run remains inactive and these internal-validation metrics are not release evidence. @@ -11792,6 +11804,8 @@ Verified in this pass: (`12 passed`). - `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_failure_driven_yolo_sampling.py backend/tests/test_belgium_training_iteration_assessment.py` (`16 passed` after adding the completed-checkpoint entry contract). +- `py -3 -m pytest -q backend/tests/test_yolo_training_supervisor.py backend/tests/test_belgium_training_loop.py` + (`11 passed`), plus a live supervisor one-shot and single-process GPU audit. Open: diff --git a/docs/TODO.md b/docs/TODO.md index bf40db20..e03fd824 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -974,6 +974,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Allow the objective CUDA loop to consume an automatically clean `needs_human_review` corpus while keeping final human sign-off as a separate, mandatory promotion gate. - [x] Persist checksummed train-only failure-driven sampling after every rejected loop iteration and resume the next checkpoint from that exact dataset YAML. - [x] Add a guarded calibration-first entry point for completed checkpoints so v37 and future externally interrupted runs can rejoin the automated loop without redundant retraining. +- [x] Add and activate a host-side, exact-run-marker supervisor that resumes the v37 CUDA checkpoint after container recreation without launching concurrent trainers. - [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed. - [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes. diff --git a/scripts/supervise_container_yolo_training.py b/scripts/supervise_container_yolo_training.py new file mode 100644 index 00000000..b876f376 --- /dev/null +++ b/scripts/supervise_container_yolo_training.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Resume one checkpointed YOLO run after container recreation, fail closed.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import time +from datetime import UTC, datetime +from pathlib import Path + + +def container_running(container: str) -> bool: + result = subprocess.run( + ["docker", "inspect", "--format", "{{.State.Running}}", container], + capture_output=True, text=True, check=False, + ) + return result.returncode == 0 and result.stdout.strip() == "true" + + +def training_active(container: str, run_marker: str) -> bool: + result = subprocess.run( + ["docker", "top", container, "-eo", "pid,args"], + capture_output=True, text=True, check=False, + ) + return result.returncode == 0 and any( + "train" in line and run_marker in line + for line in result.stdout.splitlines() + ) + + +def write_state(path: Path, payload: dict) -> None: + payload["updated_at"] = datetime.now(UTC).isoformat() + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2), encoding="utf-8") + temporary.replace(path) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--container", required=True) + parser.add_argument("--host-run-dir", type=Path, required=True) + parser.add_argument("--container-checkpoint", required=True) + parser.add_argument("--run-marker", required=True) + parser.add_argument("--yolo", default="/opt/geointel/venv/bin/yolo") + parser.add_argument("--poll-seconds", type=int, default=30) + parser.add_argument("--max-resumes", type=int, default=20) + parser.add_argument("--once", action="store_true") + args = parser.parse_args() + if args.poll_seconds < 1 or args.max_resumes < 1: + raise SystemExit("poll-seconds and max-resumes must be positive") + + state_path = args.host_run_dir / "supervisor-state.json" + state = {"schema_version": 1, "status": "monitoring", "resume_count": 0} + if state_path.is_file(): + state.update(json.loads(state_path.read_text(encoding="utf-8"))) + + while True: + if (args.host_run_dir / "results.png").is_file(): + state["status"] = "training_finished" + write_state(state_path, state) + return 0 + checkpoint = args.host_run_dir / "weights" / "last.pt" + if not checkpoint.is_file() or checkpoint.stat().st_size < 1024 * 1024: + state["status"] = "checkpoint_missing_or_incomplete" + write_state(state_path, state) + return 2 + if container_running(args.container) and not training_active(args.container, args.run_marker): + if int(state["resume_count"]) >= args.max_resumes: + state["status"] = "resume_budget_exhausted" + write_state(state_path, state) + return 3 + result = subprocess.run( + ["docker", "exec", "-d", args.container, args.yolo, "train", + f"resume={args.container_checkpoint}", "device=0"], + check=False, + ) + if result.returncode == 0: + state["resume_count"] = int(state["resume_count"]) + 1 + state["status"] = "resumed" + else: + state["status"] = "resume_command_failed" + write_state(state_path, state) + else: + state["status"] = "monitoring" + write_state(state_path, state) + if args.once: + return 0 + time.sleep(args.poll_seconds) + + +if __name__ == "__main__": + raise SystemExit(main())