Recover active training loop checkpoints

backend/tests/test_belgium_training_loop.py

@@ -85,6 +85,13 @@ def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_pat
     assert command[command.index("--output-dir") + 1].endswith("failure-driven-training")
 
 
+def test_partial_iteration_resume_uses_exact_checkpoint_and_cuda(tmp_path: Path) -> None:
+    checkpoint = tmp_path / "runs" / "iteration-002" / "weights" / "last.pt"
+    assert MODULE.resumable_training_command("yolo", checkpoint) == [
+        "yolo", "train", f"resume={checkpoint}", "device=0"
+    ]
+
+
 def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -> None:
     audit = tmp_path / "audit.json"
     audit.write_text(json.dumps({
@@ -107,7 +114,9 @@ def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -
         ], capture_output=True, text=True, check=False,
     )
     assert result.returncode == 0
-    assert json.loads(result.stdout) == {"training_command": None, "evaluate_existing": True}
+    assert json.loads(result.stdout) == {
+        "training_command": None, "evaluate_existing": True, "resume_partial": False
+    }
 
 
 def test_existing_checkpoint_iteration_directory_can_be_created_without_yolo(tmp_path: Path) -> None:

backend/tests/test_training_loop_supervisor.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).parents[2] / "scripts" / "supervise_container_training_loop.py"
+SPEC = importlib.util.spec_from_file_location("loop_supervisor", SCRIPT)
+assert SPEC and SPEC.loader
+MODULE = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MODULE)
+
+
+def test_loop_status_is_fail_closed_and_recognises_completion(tmp_path: Path) -> None:
+    state = tmp_path / "state.json"
+    assert MODULE.read_loop_status(state) is None
+    state.write_text("not-json", encoding="utf-8")
+    assert MODULE.read_loop_status(state) == "invalid"
+    state.write_text(json.dumps({"status": "training_complete"}), encoding="utf-8")
+    assert MODULE.read_loop_status(state) == "training_complete"
+
+
+def test_process_detection_requires_exact_marker(monkeypatch) -> None:
+    class Result:
+        returncode = 0
+        stdout = "12 python api.py\n13 python run_belgium_building_training_loop.py --output-dir /runs/v37\n"
+
+    monkeypatch.setattr(MODULE.subprocess, "run", lambda *args, **kwargs: Result())
+    assert MODULE.process_active("geointel", "run_belgium_building_training_loop.py")
+    assert not MODULE.process_active("geointel", "other_loop.py")