From 72ee6233a61ae3f9ea60e2b2ef0b71b7d44e8fa6 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 6 Jul 2026 16:43:53 +0200 Subject: [PATCH] Add safe YOLO model env configurator --- CHANGELOG.md | 7 + backend/README.md | 9 + ...test_sprint119_yolo_model_configuration.py | 97 ++++++++ deploy/unraid/README.md | 30 +++ docs/CODEX_EXECUTION_LOG.md | 32 +++ docs/TODO.md | 1 + scripts/README.md | 24 ++ scripts/configure_yolo_model.py | 214 ++++++++++++++++++ 8 files changed, 414 insertions(+) create mode 100644 backend/tests/test_sprint119_yolo_model_configuration.py create mode 100644 scripts/configure_yolo_model.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d52a20c..f14b56c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 117 Safe local YOLO model activation (2026-07-06) + +- Added `scripts/configure_yolo_model.py` to configure an existing local YOLO model into the Unraid/Tower `.env` file without downloading weights, loading a model or running inference. +- The helper refuses no-model and ambiguous multi-model states, and only applies env changes when `--apply` is provided. +- Documented the Tower flow for placing model files under `/mnt/user/appdata/geointel/models`, applying the env update and restarting/redeploying the all-in-one container. +- Added regression coverage for no-model, multi-model, dry-run and env-file apply behavior. + ## Sprint 116 Operational GIS map workflow (2026-07-04) - Switched the default MapLibre basemap from demo tiles to an OpenStreetMap road raster basemap with visible attribution while keeping `VITE_MAP_STYLE_URL` as the production override. diff --git a/backend/README.md b/backend/README.md index 8b9e4dd0..ffd7137d 100644 --- a/backend/README.md +++ b/backend/README.md @@ -263,6 +263,15 @@ YOLO_ENABLED=true YOLO_MODEL_PATH=/app/models/local-model.pt ``` +The root helper can write those values safely after a local model is placed: + +```bash +python scripts/configure_yolo_model.py \ + --models-dir /mnt/user/appdata/geointel/models \ + --env-file /mnt/user/appdata/geointel/.env \ + --apply +``` + The smoke loads only the supplied local model file, does not run inference and does not download weights. diff --git a/backend/tests/test_sprint119_yolo_model_configuration.py b/backend/tests/test_sprint119_yolo_model_configuration.py new file mode 100644 index 00000000..ec1e9c2a --- /dev/null +++ b/backend/tests/test_sprint119_yolo_model_configuration.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "configure_yolo_model.py" + + +def _run_configure(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), *args, "--json"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + +def test_configure_yolo_model_reports_no_local_model(tmp_path: Path) -> None: + result = _run_configure("--models-dir", str(tmp_path), "--env-file", str(tmp_path / ".env")) + payload = json.loads(result.stdout) + + assert result.returncode == 2 + assert payload["status"] == "no_model_found" + assert payload["will_download_models"] is False + assert payload["env_updates"] == {} + assert not (tmp_path / ".env").exists() + + +def test_configure_yolo_model_refuses_ambiguous_model_selection(tmp_path: Path) -> None: + (tmp_path / "a.pt").write_bytes(b"model-a") + (tmp_path / "b.onnx").write_bytes(b"model-b") + + result = _run_configure("--models-dir", str(tmp_path), "--env-file", str(tmp_path / ".env")) + payload = json.loads(result.stdout) + + assert result.returncode == 3 + assert payload["status"] == "multiple_models_found" + assert len(payload["candidates"]) == 2 + assert payload["env_updates"] == {} + assert not (tmp_path / ".env").exists() + + +def test_configure_yolo_model_dry_run_selects_single_model(tmp_path: Path) -> None: + model_path = tmp_path / "nested" / "detector.pt" + model_path.parent.mkdir() + model_path.write_bytes(b"model") + + result = _run_configure( + "--models-dir", + str(tmp_path), + "--container-model-dir", + "/app/models", + "--env-file", + str(tmp_path / ".env"), + ) + payload = json.loads(result.stdout) + + assert result.returncode == 0 + assert payload["status"] == "ready_to_apply" + assert payload["selected_host_model_path"] == str(model_path) + assert payload["selected_container_model_path"] == "/app/models/nested/detector.pt" + assert payload["env_updates"]["GEOINTEL_INSTALL_AI"] == "true" + assert payload["env_updates"]["YOLO_ENABLED"] == "true" + assert payload["env_updates"]["YOLO_MODEL_PATH"] == "/app/models/nested/detector.pt" + assert payload["will_download_models"] is False + assert not (tmp_path / ".env").exists() + + +def test_configure_yolo_model_apply_updates_existing_env_file(tmp_path: Path) -> None: + model_path = tmp_path / "detector.engine" + model_path.write_bytes(b"model") + env_file = tmp_path / ".env" + env_file.write_text("GEOINTEL_FRONTEND_PORT=1202\nYOLO_ENABLED=false\n", encoding="utf-8") + + result = _run_configure( + "--models-dir", + str(tmp_path), + "--container-model-dir", + "/app/models", + "--env-file", + str(env_file), + "--apply", + ) + payload = json.loads(result.stdout) + + assert result.returncode == 0 + assert payload["status"] == "applied" + contents = env_file.read_text(encoding="utf-8") + assert "GEOINTEL_FRONTEND_PORT=1202" in contents + assert "GEOINTEL_INSTALL_AI=true" in contents + assert "YOLO_ENABLED=true" in contents + assert "YOLO_MODEL_PATH=/app/models/detector.engine" in contents diff --git a/deploy/unraid/README.md b/deploy/unraid/README.md index 556e1555..59d38e3a 100644 --- a/deploy/unraid/README.md +++ b/deploy/unraid/README.md @@ -91,6 +91,36 @@ libraries needed for Ultralytics imports; it still never downloads model weights `YOLO_CONFIG_DIR` defaults to `/app/storage/ultralytics`, a writable persistent path, so Ultralytics settings do not fall back to root user config directories. +To safely configure an existing local YOLO model on Tower, place one supported +model file (`.pt`, `.onnx` or `.engine`) under: + +```text +/mnt/user/appdata/geointel/models +``` + +Then run a dry-run first: + +```bash +cd /mnt/user/appdata/geointel +python scripts/configure_yolo_model.py \ + --models-dir /mnt/user/appdata/geointel/models \ + --env-file .env +``` + +Apply only after the selected host and container paths are correct: + +```bash +python scripts/configure_yolo_model.py \ + --models-dir /mnt/user/appdata/geointel/models \ + --env-file .env \ + --apply +bash deploy/unraid/run-dockerman-container.sh +``` + +If multiple model files are present, add `--model-file /mnt/user/appdata/geointel/models/.pt`. +The helper does not download weights, does not load a model and does not run +inference; it only updates the env file for the mounted local model. + Validate: ```bash diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 49503af2..fe258be0 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -4279,6 +4279,38 @@ Limitations: Next recommended pass: - Continue with V1 usability work that reduces operator confusion without expanding frozen product scope. +## Sprint 117 Safe local YOLO model activation (2026-07-06) + +Changed: +- Added `scripts/configure_yolo_model.py` to configure an existing local YOLO model into the deployment `.env` file without downloading model weights, loading a model or running inference. +- The helper scans a mounted model directory for `.pt`, `.onnx` and `.engine` files, refuses no-model and ambiguous multi-model states, and writes env updates only when `--apply` is provided. +- Added regression coverage in `backend/tests/test_sprint119_yolo_model_configuration.py` for no local model, ambiguous model selection, dry-run single model selection and env-file apply behavior. +- Updated `scripts/README.md`, `deploy/unraid/README.md`, `backend/README.md`, `docs/TODO.md` and `CHANGELOG.md`. + +Tested: +- Red step: `python -m pytest backend\tests\test_sprint119_yolo_model_configuration.py -q` failed while `scripts/configure_yolo_model.py` was absent. +- `python -m pytest backend\tests\test_sprint119_yolo_model_configuration.py -q` (`4 passed`) +- `python -m py_compile scripts\configure_yolo_model.py` +- `python -m compileall backend/app` +- `cd backend && python -m pytest -q` (`375 passed`, existing Pydantic protected-namespace warnings remain) +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` +- `bash scripts/run_readiness_check.sh` (`Run readiness check passed`) +- `cd backend && python -m alembic heads` (`202606120900 (head)`) +- `cd backend && python -m alembic upgrade head --sql` +- `bash -n scripts/live_migration_smoke.sh` +- `bash -n scripts/deploy_tower.sh` + +Open: +- No local YOLO model file is currently present on Tower under `/mnt/user/appdata/geointel/models`, so runtime YOLO activation remains intentionally not configured until the operator places a real model file. + +Limitations: +- Operational configuration helper only; no AI inference behavior, model download behavior, backend API contract, database migration, provider fetching or frontend product workflow changed. +- If multiple local model files are present, the operator must choose one with `--model-file` so GeoIntel does not silently activate the wrong model. + +Next recommended pass: +- Place a real local model under `/mnt/user/appdata/geointel/models`, run the configurator with `--apply`, redeploy/restart the all-in-one container, then run the YOLO preflight with `--check-model-load` before any detection test run. + ## Sprint 104 AI Lab action guardrails (2026-06-24) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index ba24c733..9a3d9cea 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -388,3 +388,4 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add Map workspace QA/QC evidence drilldown handoff for saved selection comparisons. - [x] Persist QA/QC feature-level evidence for matches, false positives and false negatives. - [x] Render persisted QA/QC feature-level evidence as Map workspace overlays. +- [x] Add safe local YOLO model env configuration helper for Unraid/Tower runtime activation. diff --git a/scripts/README.md b/scripts/README.md index 216709c3..ef46f385 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -147,6 +147,30 @@ For Unraid/all-in-one deployments, place model files under `GEOINTEL_MODELS_PATH` so they appear in the container under `/app/models`, then set `YOLO_ENABLED=true` and `YOLO_MODEL_PATH=/app/models/.pt`. +Configure the Unraid/Tower env file from an existing local model without +downloading weights or running inference: + +```bash +python scripts/configure_yolo_model.py \ + --models-dir /mnt/user/appdata/geointel/models \ + --env-file /mnt/user/appdata/geointel/.env +``` + +If exactly one supported model file (`.pt`, `.onnx` or `.engine`) is present, +apply the env update explicitly: + +```bash +python scripts/configure_yolo_model.py \ + --models-dir /mnt/user/appdata/geointel/models \ + --env-file /mnt/user/appdata/geointel/.env \ + --apply +``` + +The configurator refuses to proceed when no model exists or when multiple model +files are present without `--model-file`. It writes only +`GEOINTEL_INSTALL_AI=true`, `YOLO_ENABLED=true` and the mounted +`YOLO_MODEL_PATH`. + Clean old offline demo export artifacts without touching uploaded source data: ```bash diff --git a/scripts/configure_yolo_model.py b/scripts/configure_yolo_model.py new file mode 100644 index 00000000..09844e07 --- /dev/null +++ b/scripts/configure_yolo_model.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Iterable + + +SUPPORTED_MODEL_SUFFIXES = {".pt", ".onnx", ".engine"} +ENV_KEYS = ("GEOINTEL_INSTALL_AI", "YOLO_ENABLED", "YOLO_MODEL_PATH") + + +def _candidate_paths(models_dir: Path) -> list[Path]: + if not models_dir.exists(): + return [] + return sorted( + path.resolve() + for path in models_dir.rglob("*") + if path.is_file() and path.suffix.lower() in SUPPORTED_MODEL_SUFFIXES + ) + + +def _container_path(host_model_path: Path, models_dir: Path, container_model_dir: str) -> str: + relative = host_model_path.resolve().relative_to(models_dir.resolve()) + base = container_model_dir.rstrip("/") + return f"{base}/{relative.as_posix()}" if relative.as_posix() else base + + +def _read_env_lines(env_file: Path) -> list[str]: + if not env_file.exists(): + return [] + return env_file.read_text(encoding="utf-8").splitlines() + + +def _update_env_file(env_file: Path, updates: dict[str, str]) -> None: + existing_lines = _read_env_lines(env_file) + seen: set[str] = set() + next_lines: list[str] = [] + + for line in existing_lines: + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in line: + next_lines.append(line) + continue + + key = line.split("=", 1)[0].strip() + if key in updates: + next_lines.append(f"{key}={updates[key]}") + seen.add(key) + else: + next_lines.append(line) + + for key, value in updates.items(): + if key not in seen: + next_lines.append(f"{key}={value}") + + env_file.parent.mkdir(parents=True, exist_ok=True) + env_file.write_text("\n".join(next_lines).rstrip() + "\n", encoding="utf-8") + + +def _base_payload(args: argparse.Namespace, candidates: Iterable[Path]) -> dict[str, object]: + return { + "models_dir": str(Path(args.models_dir).resolve()), + "env_file": str(Path(args.env_file).resolve()), + "candidates": [str(path) for path in candidates], + "selected_host_model_path": None, + "selected_container_model_path": None, + "env_updates": {}, + "apply": args.apply, + "applied": False, + "docker_restart_required": False, + "will_download_models": False, + "will_run_inference": False, + } + + +def _emit(payload: dict[str, object], *, as_json: bool) -> None: + if as_json: + print(json.dumps(payload, indent=2, sort_keys=True)) + return + + print(f"status: {payload['status']}") + print(f"message: {payload['message']}") + if payload.get("selected_host_model_path"): + print(f"host model: {payload['selected_host_model_path']}") + print(f"container model: {payload['selected_container_model_path']}") + if payload.get("env_updates"): + print("env updates:") + for key, value in payload["env_updates"].items(): # type: ignore[union-attr] + print(f" {key}={value}") + + +def configure(args: argparse.Namespace) -> tuple[int, dict[str, object]]: + models_dir = Path(args.models_dir).resolve() + env_file = Path(args.env_file).resolve() + candidates = _candidate_paths(models_dir) + payload = _base_payload(args, candidates) + + if args.model_file: + selected = Path(args.model_file).resolve() + if not selected.exists() or not selected.is_file(): + payload.update( + { + "status": "model_not_found", + "message": "The requested YOLO model file does not exist.", + } + ) + return 2, payload + if selected.suffix.lower() not in SUPPORTED_MODEL_SUFFIXES: + payload.update( + { + "status": "unsupported_model_file", + "message": "The requested YOLO model file type is not supported.", + } + ) + return 4, payload + try: + selected.relative_to(models_dir) + except ValueError: + payload.update( + { + "status": "model_outside_models_dir", + "message": "The requested model must be inside the mounted models directory.", + } + ) + return 4, payload + elif not candidates: + payload.update( + { + "status": "no_model_found", + "message": "No local YOLO model file was found. Place a .pt, .onnx or .engine file in the models directory first.", + } + ) + return 2, payload + elif len(candidates) > 1: + payload.update( + { + "status": "multiple_models_found", + "message": "Multiple model files were found. Re-run with --model-file to select one explicitly.", + } + ) + return 3, payload + else: + selected = candidates[0] + + selected_container_path = _container_path(selected, models_dir, args.container_model_dir) + updates = { + "GEOINTEL_INSTALL_AI": "true", + "YOLO_ENABLED": "true", + "YOLO_MODEL_PATH": selected_container_path, + } + payload.update( + { + "status": "ready_to_apply", + "message": "A local YOLO model was selected. Re-run with --apply to update the environment file.", + "selected_host_model_path": str(selected), + "selected_container_model_path": selected_container_path, + "env_updates": updates, + } + ) + + if args.apply: + _update_env_file(env_file, updates) + payload.update( + { + "status": "applied", + "message": "Environment file updated. Rebuild or restart the GeoIntel container to activate YOLO.", + "applied": True, + "docker_restart_required": True, + } + ) + + return 0, payload + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Configure GeoIntel to use an existing local YOLO model without downloading or running inference." + ) + parser.add_argument( + "--models-dir", + default=os.environ.get("GEOINTEL_MODELS_PATH", "models"), + help="Host directory that is mounted into the container as the model directory.", + ) + parser.add_argument( + "--container-model-dir", + default="/app/models", + help="Container path where --models-dir is mounted.", + ) + parser.add_argument( + "--env-file", + default=".env", + help="Environment file to update when --apply is provided.", + ) + parser.add_argument( + "--model-file", + help="Explicit model file to select when multiple local models exist.", + ) + parser.add_argument("--apply", action="store_true", help="Write the required YOLO env values to --env-file.") + parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv or sys.argv[1:]) + exit_code, payload = configure(args) + _emit(payload, as_json=args.json) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main())