Add guarded promoted YOLO activation
This commit is contained in:
Codex
@@ -7,6 +7,14 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 163 Guarded YOLO candidate activation (2026-07-11)
|
||||
|
||||
- Added `scripts/activate_promoted_yolo_candidate.py` to validate a promotion report and exact candidate key before emitting YOLO `.env` activation updates.
|
||||
- The helper supports dry-run by default and writes `.env` only with `--apply`; it does not download weights, load models or run inference.
|
||||
- Updated Detection Lab operator profiles: balanced `0.15` remains candidate-only, while conservative `0.35` is marked as the promoted profile backed by the split-background pure-empty gate.
|
||||
- Added tests for dry-run activation, `.env` apply behavior, rejected report handling and promoted UI profile status.
|
||||
- No API contract, database migration, provider fetching, fake detections, model file mutation or automatic runtime activation was introduced.
|
||||
|
||||
## Sprint 162 Split-background promotion runtime pass (2026-07-11)
|
||||
|
||||
- Hardened split-background preflight compatibility for legacy operator manifests by deriving missing background categories from `reference_feature_count`.
|
||||
|
||||
+17
-2
@@ -275,8 +275,23 @@ python scripts/configure_yolo_model.py \
|
||||
--apply
|
||||
```
|
||||
|
||||
The smoke loads only the supplied local model file, does not run inference and
|
||||
does not download weights.
|
||||
When a promotion report recommends an exact model/tile/threshold candidate,
|
||||
prefer the guarded activation helper. It validates the report, checks the local
|
||||
model asset and writes `.env` only when `--apply` is supplied:
|
||||
|
||||
```bash
|
||||
python scripts/activate_promoted_yolo_candidate.py \
|
||||
--promotion-report /mnt/user/appdata/geointel/artifacts/detection-model-promotion/split-aware/aoi1024bg512r3e50-high-threshold-split-20260710T222934Z/detection_model_promotion_report.json \
|
||||
--candidate-key 'geointel-building-yolov8s-aoi1024bg512r3e50-pt|512|64|0.35' \
|
||||
--models-dir /mnt/user/appdata/geointel/models \
|
||||
--env-file /mnt/user/appdata/geointel/.env \
|
||||
--json
|
||||
```
|
||||
|
||||
Add `--apply` only after reviewing the emitted env updates. The smoke and
|
||||
activation helpers load no model by default, run no inference and do not
|
||||
download weights. Restart or rebuild the runtime after applying because the
|
||||
active model is read from `YOLO_MODEL_PATH`.
|
||||
|
||||
Operator-only local training preparation is available when real public model
|
||||
candidates are too weak for the target imagery. It is not a browser feature and
|
||||
|
||||
@@ -4,7 +4,7 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_detection_operator_profiles_define_explicit_non_default_yolo_candidates() -> None:
|
||||
def test_detection_operator_profiles_define_explicit_yolo_candidates_and_promoted_profile() -> None:
|
||||
profiles = ROOT / "frontend" / "src" / "components" / "detection" / "detectionProfiles.ts"
|
||||
source = profiles.read_text(encoding="utf-8")
|
||||
|
||||
@@ -15,7 +15,10 @@ def test_detection_operator_profiles_define_explicit_non_default_yolo_candidates
|
||||
assert "confidenceThreshold: 0.15" in source
|
||||
assert "confidenceThreshold: 0.35" in source
|
||||
assert "defaultApproved: false" in source
|
||||
assert "defaultApproved: true" in source
|
||||
assert "promotionRecommendation: 'none'" in source
|
||||
assert "promotionRecommendation: 'promote_candidate'" in source
|
||||
assert "pure-empty gate passed" in source
|
||||
assert "false-positive pressure" in source
|
||||
|
||||
|
||||
@@ -29,6 +32,7 @@ def test_detection_lab_surfaces_profiles_as_deliberate_operator_actions() -> Non
|
||||
assert "profile.displayName" in lab
|
||||
assert "profile.confidenceThreshold" in lab
|
||||
assert "Candidate only - not default-approved" in lab
|
||||
assert "default-approved" in lab
|
||||
assert "Apply profile" in lab
|
||||
assert "onApplyOperatorProfile(profile)" in lab
|
||||
assert "Recommended starting threshold: 0.25" not in lab
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "activate_promoted_yolo_candidate.py"
|
||||
CANDIDATE_KEY = "geointel-building-yolov8s-aoi1024bg512r3e50-pt|512|64|0.35"
|
||||
|
||||
|
||||
def _write_model(models_dir: Path) -> Path:
|
||||
model_file = models_dir / "geointel-building-yolov8s-aoi1024bg512r3e50.pt"
|
||||
model_file.parent.mkdir(parents=True)
|
||||
model_file.write_bytes(b"local promoted model")
|
||||
return model_file
|
||||
|
||||
|
||||
def _write_report(path: Path, *, promotion_status: str = "promote_candidate") -> None:
|
||||
rejection_reasons = [] if promotion_status == "promote_candidate" else ["background_false_positive_pressure"]
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"gates": {
|
||||
"max_background_detections_per_sample": 0,
|
||||
"min_background_samples": 2,
|
||||
"min_mean_f1": 0.25,
|
||||
"min_positive_samples": 7,
|
||||
},
|
||||
"recommended_candidate": {
|
||||
"background_sample_count": 3,
|
||||
"background_samples": [["arendonk_heide", 0], ["lommel_heide", 0], ["postel_bos", 0]],
|
||||
"candidate_key": CANDIDATE_KEY,
|
||||
"max_background_detections": 0,
|
||||
"mean_f1": 0.32086574003576274,
|
||||
"mean_precision": 0.8400057773951873,
|
||||
"mean_recall": 0.20213514285308795,
|
||||
"model_asset_id": "geointel-building-yolov8s-aoi1024bg512r3e50-pt",
|
||||
"positive_sample_count": 7,
|
||||
"promotion_status": promotion_status,
|
||||
"rejection_reasons": rejection_reasons,
|
||||
"threshold": 0.35,
|
||||
"tile_overlap": 64,
|
||||
"tile_size": 512,
|
||||
"total_background_detections": 0,
|
||||
},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _run_activation(tmp_path: Path, *extra_args: str) -> subprocess.CompletedProcess[str]:
|
||||
models_dir = tmp_path / "models"
|
||||
_write_model(models_dir)
|
||||
report_path = tmp_path / "promotion_report.json"
|
||||
_write_report(report_path)
|
||||
env_file = tmp_path / ".env"
|
||||
env_file.write_text("GEOINTEL_ENV=production\nYOLO_ENABLED=false\n", encoding="utf-8")
|
||||
|
||||
return subprocess.run(
|
||||
[
|
||||
"python",
|
||||
str(SCRIPT),
|
||||
"--promotion-report",
|
||||
str(report_path),
|
||||
"--candidate-key",
|
||||
CANDIDATE_KEY,
|
||||
"--models-dir",
|
||||
str(models_dir),
|
||||
"--container-model-dir",
|
||||
"/app/models",
|
||||
"--env-file",
|
||||
str(env_file),
|
||||
"--json",
|
||||
*extra_args,
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def test_promoted_yolo_activation_dry_run_validates_report_and_model(tmp_path: Path) -> None:
|
||||
result = _run_activation(tmp_path)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["status"] == "ready_to_apply"
|
||||
assert payload["applied"] is False
|
||||
assert payload["will_download_models"] is False
|
||||
assert payload["candidate"]["candidate_key"] == CANDIDATE_KEY
|
||||
assert payload["candidate"]["threshold"] == 0.35
|
||||
assert payload["env_updates"]["YOLO_ENABLED"] == "true"
|
||||
assert payload["env_updates"]["YOLO_MODEL_PATH"] == "/app/models/geointel-building-yolov8s-aoi1024bg512r3e50.pt"
|
||||
|
||||
|
||||
def test_promoted_yolo_activation_apply_updates_env_file(tmp_path: Path) -> None:
|
||||
result = _run_activation(tmp_path, "--apply")
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["status"] == "applied"
|
||||
env_text = (tmp_path / ".env").read_text(encoding="utf-8")
|
||||
assert "GEOINTEL_ENV=production" in env_text
|
||||
assert "GEOINTEL_INSTALL_AI=true" in env_text
|
||||
assert "YOLO_ENABLED=true" in env_text
|
||||
assert "YOLO_MODELS_DIR=/app/models" in env_text
|
||||
assert "YOLO_MODEL_PATH=/app/models/geointel-building-yolov8s-aoi1024bg512r3e50.pt" in env_text
|
||||
|
||||
|
||||
def test_promoted_yolo_activation_rejects_non_promoted_report(tmp_path: Path) -> None:
|
||||
models_dir = tmp_path / "models"
|
||||
_write_model(models_dir)
|
||||
report_path = tmp_path / "promotion_report.json"
|
||||
_write_report(report_path, promotion_status="reject")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"python",
|
||||
str(SCRIPT),
|
||||
"--promotion-report",
|
||||
str(report_path),
|
||||
"--candidate-key",
|
||||
CANDIDATE_KEY,
|
||||
"--models-dir",
|
||||
str(models_dir),
|
||||
"--env-file",
|
||||
str(tmp_path / ".env"),
|
||||
"--json",
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 3
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["status"] == "candidate_not_promoted"
|
||||
assert "rejection_reasons" in payload
|
||||
|
||||
|
||||
def test_readiness_gate_compiles_promoted_activation_script() -> None:
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "-m py_compile scripts/activate_promoted_yolo_candidate.py" in readiness
|
||||
+27
-5
@@ -335,13 +335,35 @@ hard-negative gates, then run `sparse_building_context` as a separate review
|
||||
matrix. The first expanded local model improved dense AOI F1, but Kasterlee-bos
|
||||
false positives block default promotion.
|
||||
|
||||
The current inactive AOI1024 background-aware local model asset,
|
||||
The AOI1024 background-aware local model asset,
|
||||
`geointel-building-yolov8s-aoi1024bg512r3e50-pt`, is exposed in Detection Lab
|
||||
only through deliberate operator profiles. `balanced-review` applies threshold
|
||||
`0.15` for the strongest positive-AOI F1 observed so far; `conservative-review`
|
||||
applies threshold `0.35` for higher precision review. Both profiles remain
|
||||
candidate-only, not default-approved, because the promotion recommendation is
|
||||
still `none` and background false-positive pressure has not passed the gate.
|
||||
`0.15` for the strongest positive-AOI F1 observed so far, but remains
|
||||
candidate-only because pure-empty false-positive pressure failed at that
|
||||
threshold. `conservative-review` applies threshold `0.35` and is marked as the
|
||||
promoted candidate after the split-background report passed the strict
|
||||
pure-empty gate. Sparse-context detections remain review-only evidence, not a
|
||||
default-promotion blocker.
|
||||
|
||||
To update a Tower/Unraid `.env` from a promoted report, use the guarded
|
||||
activation helper. It validates the exact report candidate key, verifies that
|
||||
the candidate has `promotion_status=promote_candidate`, resolves the local model
|
||||
asset under the mounted models directory, and writes environment updates only
|
||||
when `--apply` is supplied:
|
||||
|
||||
```bash
|
||||
python scripts/activate_promoted_yolo_candidate.py \
|
||||
--promotion-report artifacts/detection-model-promotion/split-aware/aoi1024bg512r3e50-high-threshold-split-20260710T222934Z/detection_model_promotion_report.json \
|
||||
--candidate-key 'geointel-building-yolov8s-aoi1024bg512r3e50-pt|512|64|0.35' \
|
||||
--models-dir /mnt/user/appdata/geointel/models \
|
||||
--env-file /mnt/user/appdata/geointel/.env \
|
||||
--json
|
||||
```
|
||||
|
||||
Re-run with `--apply` only after reviewing the emitted env updates. The helper
|
||||
does not download weights, load a model or run inference. Restart or rebuild the
|
||||
runtime after applying because `YOLO_MODEL_PATH` is read from environment
|
||||
configuration.
|
||||
|
||||
To compare the same model/tile/threshold grid across all prepared operator
|
||||
samples, use:
|
||||
|
||||
@@ -6532,3 +6532,44 @@ Open:
|
||||
## Next recommended pass
|
||||
|
||||
- Add a guarded model activation/operator-selection workflow that can mark a promoted candidate as active only after the report artifact and candidate key are explicitly supplied.
|
||||
|
||||
# Sprint 163 - Guarded promoted YOLO activation workflow
|
||||
|
||||
## What changed
|
||||
|
||||
- Added `scripts/activate_promoted_yolo_candidate.py`.
|
||||
- The helper validates:
|
||||
- the promotion report file exists and is valid JSON;
|
||||
- the exact supplied `candidate_key` matches the report recommended candidate;
|
||||
- the candidate has `promotion_status=promote_candidate`;
|
||||
- positive sample count, background sample count, mean F1 and max background detections still satisfy report gates;
|
||||
- the candidate `model_asset_id` resolves to an existing local model file under the mounted models directory.
|
||||
- The helper emits `.env` updates in dry-run mode by default and writes them only when `--apply` is supplied.
|
||||
- Updated Detection Lab operator profiles:
|
||||
- `balanced-review` at threshold `0.15` remains candidate-only because pure-empty false-positive pressure failed.
|
||||
- `conservative-review` at threshold `0.35` is marked as promoted/default-approved based on the split-background pure-empty gate.
|
||||
- Added docs for the guarded activation command in `docs/AI_PIPELINES.md`, `scripts/README.md`, `backend/README.md` and `frontend/README.md`.
|
||||
- Added readiness coverage for compiling the new helper.
|
||||
- No API contract, database migration, provider fetching, fake detection path, model file mutation, model download or automatic runtime activation was introduced in code.
|
||||
|
||||
## What was tested locally
|
||||
|
||||
- RED: `python -m pytest tests/test_sprint162_promoted_model_activation.py -q` failed while `scripts/activate_promoted_yolo_candidate.py` was absent.
|
||||
- RED: `python -m pytest tests/test_sprint155_detection_operator_profiles.py -q` failed before `conservative-review` was marked promoted.
|
||||
- RED: `python -m pytest tests/test_sprint162_promoted_model_activation.py::test_readiness_gate_compiles_promoted_activation_script -q` failed before readiness compiled the helper.
|
||||
- Ran `python -m pytest tests/test_sprint162_promoted_model_activation.py tests/test_sprint155_detection_operator_profiles.py -q`: 7 passed.
|
||||
- Ran `python -m py_compile scripts/activate_promoted_yolo_candidate.py`.
|
||||
- Ran `python -m compileall backend/app`.
|
||||
- Ran `python -m pytest` in `backend`: 454 passed, 17 existing Pydantic protected-namespace warnings.
|
||||
- Ran `npm run typecheck` in `frontend`.
|
||||
- Ran `npm run build` in `frontend`.
|
||||
- Ran `bash scripts/run_readiness_check.sh`: passed.
|
||||
|
||||
## Known limitations
|
||||
|
||||
- The helper updates runtime environment only; a container restart or rebuild is still required for `YOLO_MODEL_PATH` changes to take effect.
|
||||
- The promoted threshold is represented in the operator profile and promotion report. The backend detection endpoint still requires clients to submit the intended confidence threshold explicitly.
|
||||
|
||||
## Next recommended pass
|
||||
|
||||
- Push this helper to Tower, run it first as dry-run against the high-threshold promotion report, then apply and redeploy/restart only if the emitted `YOLO_MODEL_PATH` matches the promoted local asset.
|
||||
|
||||
+4
-3
@@ -121,14 +121,15 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
- [x] Export and audit AOI1024 clean-label variants; select `yolo-building-aoi1024-visible050-minpx8` as the first audit-passing 512px training candidate.
|
||||
- [x] Train and gate `geointel-building-yolov8s-aoi1024clean512e50-pt` through seven positive AOIs and nine hard-negative/background AOIs.
|
||||
- [x] Train and gate background-aware `geointel-building-yolov8s-aoi1024bg512r3e50-pt`; it is the strongest positive-AOI candidate so far but remains inactive because full background-candidate false-positive pressure still blocks default promotion.
|
||||
- [x] Add explicit operator detection profiles for local model assets: balanced review around threshold `0.15` and conservative high-precision review around threshold `0.35`, both clearly marked as non-default-approved until promotion gates pass.
|
||||
- [x] Add explicit operator detection profiles for local model assets: balanced review around threshold `0.15` remains candidate-only, while conservative high-precision review around threshold `0.35` is marked promoted after the pure-empty split-background gate passed.
|
||||
- [x] Add pure-empty versus sparse-building contextual background corpus classification to operator manifests, hard-negative matrix filters and YOLO tile provenance.
|
||||
- [x] Add a split background-corpus matrix runner and report builder that runs pure-empty and sparse-context matrices separately.
|
||||
- [x] Teach the model promotion report to consume split background summaries so only `pure_empty_negative` blocks default promotion and `sparse_building_context` stays review-only.
|
||||
- [x] Add one-command operator workflow to run split background matrices and immediately build the split-aware promotion report.
|
||||
- [x] Add preflight-only validation for the split-background promotion workflow before long runtime matrices.
|
||||
- [ ] Rerun split background matrices on Tower after rebuild, then retrain or recalibrate against the cleaner pure-empty gate plus separate sparse-context inspection matrix.
|
||||
- [ ] Promote a V1 default building detector only after it passes seven positive AOIs, clean hard-negative/background gates and persisted QA/QC evidence without fake detections or model downloads.
|
||||
- [x] Rerun split background matrices on Tower after rebuild, then recalibrate against the cleaner pure-empty gate plus separate sparse-context inspection matrix.
|
||||
- [x] Add guarded promoted-candidate activation helper requiring a promotion report path and exact candidate key before `.env` can be changed.
|
||||
- [ ] Apply promoted V1 default building detector only after explicit operator review of the emitted `.env` updates, followed by rebuild/restart and browser/runtime smoke.
|
||||
|
||||
## Sprint 8 status
|
||||
|
||||
|
||||
+2
-1
@@ -124,7 +124,8 @@ AI Lab run controls explicitly explain when no raster dataset is available, inst
|
||||
- Detection Lab now exposes the `yolo-configured` capability reported by the backend.
|
||||
- When `yolo-configured` is selected, users can provide an existing raster tile manifest path.
|
||||
- Detection Lab lists local model assets from `GET /api/v1/detection/model-assets` so operators can choose an existing mounted model file instead of editing only one hidden `YOLO_MODEL_PATH` slot.
|
||||
- Detection Lab exposes explicit operator profiles for the current inactive local AOI1024 building detector: balanced review at threshold `0.15` and conservative review at threshold `0.35`. Applying a profile deliberately selects the local model asset and threshold; it does not approve or promote a default model.
|
||||
- Detection Lab exposes explicit operator profiles for the local AOI1024 building detector: balanced review at threshold `0.15` remains candidate-only, while conservative review at threshold `0.35` is marked as the promoted profile after the split-background pure-empty gate passed.
|
||||
- Applying a profile deliberately selects the local model asset and threshold for the browser-run request; runtime default activation remains a separate guarded `.env` operation through `scripts/activate_promoted_yolo_candidate.py`.
|
||||
- Detection Lab includes a read-only YOLO runtime preflight panel with backend status, dependency visibility, local model configuration, `torch`/`ultralytics` versions, CUDA state and `YOLO_CONFIG_DIR`.
|
||||
- The UI still does not download models or create fake detections; backend status and error codes remain the source of truth.
|
||||
|
||||
|
||||
@@ -255,7 +255,7 @@ export function DetectionLab({
|
||||
<strong>Operator profiles</strong>
|
||||
<p>
|
||||
Candidate profiles apply a local model asset and confidence threshold only after an explicit click.
|
||||
Candidate only - not default-approved while the promotion recommendation remains none.
|
||||
Promoted profiles still require explicit operator action and do not mutate the runtime environment.
|
||||
</p>
|
||||
</div>
|
||||
<div className="operator-profile-grid" aria-label="Configured YOLO operator profiles">
|
||||
@@ -305,8 +305,8 @@ export function DetectionLab({
|
||||
<div className="model-asset-guidance">
|
||||
<strong>Selected model asset status</strong>
|
||||
<p>
|
||||
{selectedModelAsset.display_name} is operator-selected. Keep local candidates inactive until persisted
|
||||
promotion evidence explicitly recommends default activation.
|
||||
{selectedModelAsset.display_name} is selected for this browser-run request. Runtime default activation
|
||||
remains a separate guarded operator action backed by a promotion report.
|
||||
</p>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
@@ -24,24 +24,24 @@ export const DETECTION_OPERATOR_PROFILES: DetectionOperatorProfile[] = [
|
||||
precision: 0.636639,
|
||||
recall: 0.424258,
|
||||
f1: 0.5074022485589402,
|
||||
maxBackgroundDetections: 103,
|
||||
maxBackgroundDetections: 46,
|
||||
description: 'Best positive-AOI F1 profile for deliberate operator review of the inactive AOI1024 model asset.',
|
||||
limitationMessage:
|
||||
'Candidate only because false-positive pressure still blocks default promotion on the background/hard-negative gate.',
|
||||
'Candidate only because pure-empty false-positive pressure still blocks default promotion on the background gate.',
|
||||
},
|
||||
{
|
||||
id: 'conservative-review',
|
||||
displayName: 'Conservative review',
|
||||
displayName: 'Promoted conservative review',
|
||||
modelAssetId: 'geointel-building-yolov8s-aoi1024bg512r3e50-pt',
|
||||
confidenceThreshold: 0.35,
|
||||
defaultApproved: false,
|
||||
promotionRecommendation: 'none',
|
||||
defaultApproved: true,
|
||||
promotionRecommendation: 'promote_candidate',
|
||||
precision: 0.840006,
|
||||
recall: 0.202135,
|
||||
f1: 0.32086574003576274,
|
||||
maxBackgroundDetections: 55,
|
||||
description: 'Higher-precision profile for demos or review sessions where fewer false positives matter more than recall.',
|
||||
maxBackgroundDetections: 0,
|
||||
description: 'Promoted high-precision profile for demos or review sessions where fewer false positives matter more than recall.',
|
||||
limitationMessage:
|
||||
'Candidate only because false-positive pressure remains visible; use it deliberately and inspect persisted QA evidence.',
|
||||
'Default-approved after the split-background pure-empty gate passed; sparse-context detections remain review-only evidence.',
|
||||
},
|
||||
]
|
||||
|
||||
@@ -648,6 +648,24 @@ files are present without `--model-file`. It writes only
|
||||
`GEOINTEL_INSTALL_AI=true`, `YOLO_ENABLED=true`, `YOLO_MODELS_DIR=/app/models`
|
||||
and the mounted `YOLO_MODEL_PATH`.
|
||||
|
||||
When a split-background promotion report recommends a specific candidate, use
|
||||
the guarded activation helper instead of choosing a model path manually. The
|
||||
helper validates the exact `candidate_key`, promotion status and local model
|
||||
asset before writing anything, and it mutates `.env` only with `--apply`:
|
||||
|
||||
```bash
|
||||
python scripts/activate_promoted_yolo_candidate.py \
|
||||
--promotion-report /mnt/user/appdata/geointel/artifacts/detection-model-promotion/split-aware/aoi1024bg512r3e50-high-threshold-split-20260710T222934Z/detection_model_promotion_report.json \
|
||||
--candidate-key 'geointel-building-yolov8s-aoi1024bg512r3e50-pt|512|64|0.35' \
|
||||
--models-dir /mnt/user/appdata/geointel/models \
|
||||
--env-file /mnt/user/appdata/geointel/.env \
|
||||
--json
|
||||
```
|
||||
|
||||
Add `--apply` only after reviewing the emitted updates. The helper never
|
||||
downloads weights, loads the model or runs inference; restart or rebuild the
|
||||
container after applying because `YOLO_MODEL_PATH` is read from the environment.
|
||||
|
||||
Tower-local model evaluation status:
|
||||
|
||||
- `geointel-building-yolov8s-hardneg160r4e50.pt` is available as an evaluated
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SUPPORTED_MODEL_SUFFIXES = {".pt", ".onnx", ".engine"}
|
||||
ENV_KEYS = ("GEOINTEL_INSTALL_AI", "YOLO_ENABLED", "YOLO_MODELS_DIR", "YOLO_MODEL_PATH")
|
||||
|
||||
|
||||
def _asset_id(path: Path) -> str:
|
||||
raw = f"{path.stem}-{path.suffix.lower().lstrip('.')}"
|
||||
normalized = re.sub(r"[^a-z0-9]+", "-", raw.lower()).strip("-")
|
||||
return normalized or "model-asset"
|
||||
|
||||
|
||||
def _candidate_paths(models_dir: Path) -> list[Path]:
|
||||
if not models_dir.exists() or not models_dir.is_dir():
|
||||
return []
|
||||
return sorted(
|
||||
path.resolve()
|
||||
for path in models_dir.rglob("*")
|
||||
if path.is_file() and path.suffix.lower() in SUPPORTED_MODEL_SUFFIXES
|
||||
)
|
||||
|
||||
|
||||
def _container_path(host_model_path: Path, models_dir: Path, container_model_dir: str) -> str:
|
||||
relative = host_model_path.resolve().relative_to(models_dir.resolve())
|
||||
base = container_model_dir.rstrip("/")
|
||||
return f"{base}/{relative.as_posix()}" if relative.as_posix() else base
|
||||
|
||||
|
||||
def _read_env_lines(env_file: Path) -> list[str]:
|
||||
if not env_file.exists():
|
||||
return []
|
||||
return env_file.read_text(encoding="utf-8").splitlines()
|
||||
|
||||
|
||||
def _update_env_file(env_file: Path, updates: dict[str, str]) -> None:
|
||||
existing_lines = _read_env_lines(env_file)
|
||||
seen: set[str] = set()
|
||||
next_lines: list[str] = []
|
||||
|
||||
for line in existing_lines:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#") or "=" not in line:
|
||||
next_lines.append(line)
|
||||
continue
|
||||
|
||||
key = line.split("=", 1)[0].strip()
|
||||
if key in updates:
|
||||
next_lines.append(f"{key}={updates[key]}")
|
||||
seen.add(key)
|
||||
else:
|
||||
next_lines.append(line)
|
||||
|
||||
for key, value in updates.items():
|
||||
if key not in seen:
|
||||
next_lines.append(f"{key}={value}")
|
||||
|
||||
env_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
env_file.write_text("\n".join(next_lines).rstrip() + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _load_report(path: Path) -> dict[str, Any]:
|
||||
if not path.exists() or not path.is_file():
|
||||
raise ValueError("Promotion report file does not exist")
|
||||
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("Promotion report must be a JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
def _candidate_from_report(report: dict[str, Any], candidate_key: str) -> dict[str, Any] | None:
|
||||
recommended = report.get("recommended_candidate")
|
||||
if isinstance(recommended, dict):
|
||||
if recommended.get("candidate_key") == candidate_key:
|
||||
return recommended
|
||||
return None
|
||||
if isinstance(recommended, str) and recommended == candidate_key:
|
||||
for item in report.get("candidate_decisions") or []:
|
||||
if isinstance(item, dict) and item.get("candidate_key") == candidate_key:
|
||||
return item
|
||||
for item in report.get("candidate_decisions") or []:
|
||||
if isinstance(item, dict) and item.get("candidate_key") == candidate_key:
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
def _gate_failures(report: dict[str, Any], candidate: dict[str, Any]) -> list[str]:
|
||||
gates = report.get("gates") if isinstance(report.get("gates"), dict) else {}
|
||||
failures: list[str] = []
|
||||
if candidate.get("promotion_status") != "promote_candidate":
|
||||
failures.append("candidate_not_promoted")
|
||||
if candidate.get("rejection_reasons"):
|
||||
failures.append("candidate_has_rejection_reasons")
|
||||
|
||||
min_positive_samples = int(gates.get("min_positive_samples") or 0)
|
||||
min_background_samples = int(gates.get("min_background_samples") or 0)
|
||||
min_mean_f1 = float(gates.get("min_mean_f1") or 0)
|
||||
max_background_detections = int(gates.get("max_background_detections_per_sample") or 0)
|
||||
|
||||
if int(candidate.get("positive_sample_count") or 0) < min_positive_samples:
|
||||
failures.append("insufficient_positive_samples")
|
||||
if int(candidate.get("background_sample_count") or 0) < min_background_samples:
|
||||
failures.append("insufficient_background_samples")
|
||||
if float(candidate.get("mean_f1") or 0) < min_mean_f1:
|
||||
failures.append("positive_mean_f1_below_gate")
|
||||
if int(candidate.get("max_background_detections") or 0) > max_background_detections:
|
||||
failures.append("background_false_positive_pressure")
|
||||
return failures
|
||||
|
||||
|
||||
def _resolve_model_asset(models_dir: Path, model_asset_id: str) -> Path | None:
|
||||
for path in _candidate_paths(models_dir):
|
||||
if _asset_id(path) == model_asset_id:
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def _base_payload(args: argparse.Namespace) -> dict[str, Any]:
|
||||
return {
|
||||
"promotion_report": str(Path(args.promotion_report).resolve()),
|
||||
"candidate_key": args.candidate_key,
|
||||
"models_dir": str(Path(args.models_dir).resolve()),
|
||||
"env_file": str(Path(args.env_file).resolve()),
|
||||
"candidate": None,
|
||||
"selected_host_model_path": None,
|
||||
"selected_container_model_path": None,
|
||||
"env_updates": {},
|
||||
"apply": args.apply,
|
||||
"applied": False,
|
||||
"docker_restart_required": False,
|
||||
"will_download_models": False,
|
||||
"will_run_inference": False,
|
||||
}
|
||||
|
||||
|
||||
def activate(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:
|
||||
payload = _base_payload(args)
|
||||
try:
|
||||
report = _load_report(Path(args.promotion_report))
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
payload.update({"status": "invalid_promotion_report", "message": str(exc)})
|
||||
return 2, payload
|
||||
|
||||
candidate = _candidate_from_report(report, args.candidate_key)
|
||||
if not candidate:
|
||||
payload.update(
|
||||
{
|
||||
"status": "candidate_not_recommended",
|
||||
"message": "Candidate key does not match the report recommended candidate.",
|
||||
}
|
||||
)
|
||||
return 3, payload
|
||||
|
||||
payload["candidate"] = candidate
|
||||
failures = _gate_failures(report, candidate)
|
||||
if failures:
|
||||
payload.update(
|
||||
{
|
||||
"status": "candidate_not_promoted",
|
||||
"message": "Candidate did not pass the promotion gates.",
|
||||
"rejection_reasons": failures,
|
||||
}
|
||||
)
|
||||
return 3, payload
|
||||
|
||||
model_asset_id = str(candidate.get("model_asset_id") or "").strip()
|
||||
if not model_asset_id:
|
||||
payload.update({"status": "model_asset_missing", "message": "Candidate does not record model_asset_id."})
|
||||
return 2, payload
|
||||
|
||||
models_dir = Path(args.models_dir).resolve()
|
||||
selected_model = _resolve_model_asset(models_dir, model_asset_id)
|
||||
if selected_model is None:
|
||||
payload.update(
|
||||
{
|
||||
"status": "model_asset_not_found",
|
||||
"message": "Promoted candidate model asset was not found in the local models directory.",
|
||||
"model_asset_id": model_asset_id,
|
||||
}
|
||||
)
|
||||
return 2, payload
|
||||
|
||||
selected_container_path = _container_path(selected_model, models_dir, args.container_model_dir)
|
||||
updates = {
|
||||
"GEOINTEL_INSTALL_AI": "true",
|
||||
"YOLO_ENABLED": "true",
|
||||
"YOLO_MODELS_DIR": args.container_model_dir.rstrip("/"),
|
||||
"YOLO_MODEL_PATH": selected_container_path,
|
||||
}
|
||||
payload.update(
|
||||
{
|
||||
"status": "ready_to_apply",
|
||||
"message": "Promoted YOLO candidate validated. Re-run with --apply to update the environment file.",
|
||||
"selected_host_model_path": str(selected_model),
|
||||
"selected_container_model_path": selected_container_path,
|
||||
"env_updates": updates,
|
||||
"docker_restart_required": True,
|
||||
}
|
||||
)
|
||||
|
||||
if args.apply:
|
||||
_update_env_file(Path(args.env_file).resolve(), updates)
|
||||
payload.update(
|
||||
{
|
||||
"status": "applied",
|
||||
"message": "Environment file updated. Rebuild or restart the GeoIntel container to use the promoted model path.",
|
||||
"applied": True,
|
||||
}
|
||||
)
|
||||
|
||||
return 0, payload
|
||||
|
||||
|
||||
def _emit(payload: dict[str, Any], *, as_json: bool) -> None:
|
||||
if as_json:
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
return
|
||||
|
||||
print(f"status: {payload['status']}")
|
||||
print(f"message: {payload['message']}")
|
||||
if payload.get("selected_host_model_path"):
|
||||
print(f"host model: {payload['selected_host_model_path']}")
|
||||
print(f"container model: {payload['selected_container_model_path']}")
|
||||
if payload.get("env_updates"):
|
||||
print("env updates:")
|
||||
for key in ENV_KEYS:
|
||||
if key in payload["env_updates"]:
|
||||
print(f" {key}={payload['env_updates'][key]}")
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Activate an existing local YOLO model only after a promotion report recommends the exact candidate key."
|
||||
)
|
||||
parser.add_argument("--promotion-report", required=True, help="Path to detection_model_promotion_report.json.")
|
||||
parser.add_argument("--candidate-key", required=True, help="Exact promoted candidate key from the report.")
|
||||
parser.add_argument(
|
||||
"--models-dir",
|
||||
default=os.environ.get("GEOINTEL_MODELS_PATH", "models"),
|
||||
help="Host directory containing mounted local model files.",
|
||||
)
|
||||
parser.add_argument("--container-model-dir", default="/app/models", help="Container path where --models-dir is mounted.")
|
||||
parser.add_argument("--env-file", default=".env", help="Environment file to update when --apply is supplied.")
|
||||
parser.add_argument("--apply", action="store_true", help="Write YOLO env updates after all promotion gates pass.")
|
||||
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv or sys.argv[1:])
|
||||
exit_code, payload = activate(args)
|
||||
_emit(payload, as_json=args.json)
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -47,6 +47,7 @@ ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
||||
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
|
||||
${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
|
||||
${PYTHON_BIN} -m py_compile scripts/build_background_corpus_split_report.py
|
||||
${PYTHON_BIN} -m py_compile scripts/activate_promoted_yolo_candidate.py
|
||||
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
||||
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
||||
${PYTHON_BIN} -m compileall backend/app
|
||||
|
||||
Reference in New Issue
Block a user