diff --git a/CHANGELOG.md b/CHANGELOG.md index ca5af232..19dfc8b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 160 Split-background promotion preflight (2026-07-10) + +- Added `--preflight-only` to `scripts/run_split_background_promotion_workflow.sh`. +- Preflight validates local positive portfolio and operator manifest files, confirms required `pure_empty_negative` and `sparse_building_context` background categories, checks Python/curl availability and verifies the runtime API proxy returns the canonical envelope. +- Documented the quick post-redeploy preflight command before starting long configured-YOLO matrix inference. +- No model default, backend API, database migration, provider fetching, fake detection output or model download behavior changed. + ## Sprint 159 Split-background promotion workflow wrapper (2026-07-10) - Added `scripts/run_split_background_promotion_workflow.sh` to run the split background matrix and split-aware promotion report from one operator command. diff --git a/backend/tests/test_sprint159_split_promotion_workflow.py b/backend/tests/test_sprint159_split_promotion_workflow.py index 8dc7b0bc..c47c7fd7 100644 --- a/backend/tests/test_sprint159_split_promotion_workflow.py +++ b/backend/tests/test_sprint159_split_promotion_workflow.py @@ -23,6 +23,18 @@ def test_split_background_promotion_workflow_runs_split_then_split_aware_report( assert "promote model default" not in source.lower() +def test_split_background_promotion_workflow_has_safe_preflight_mode() -> None: + source = (ROOT / "scripts" / "run_split_background_promotion_workflow.sh").read_text(encoding="utf-8") + + assert "--preflight-only" in source + assert "PREFLIGHT_ONLY" in source + assert "curl -fsS" in source + assert "OPERATOR_SAMPLE_MANIFEST_PATH is required" in source + assert "pure_empty_negative" in source + assert "sparse_building_context" in source + assert "Split-background promotion preflight passed" in source + + def test_readiness_checks_split_background_promotion_workflow_syntax() -> None: readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 6315fd00..56a99673 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -296,6 +296,17 @@ The wrapper keeps the same safety boundary: existing dataset upload, configured YOLO detection and report tooling only. It does not change model configuration or bypass the persisted QA/QC evidence requirement. +For a quick post-redeploy check before the long matrix starts, use +`--preflight-only` with the same positive portfolio and operator manifest. This +checks local paths, required background categories and the runtime API envelope +without running inference: + +```bash +PROMOTION_POSITIVE_PORTFOLIO_PATH=artifacts/detection-quality-matrix/multi-sample/aoi1024bg512r3e50-positive/multi_sample_quality_summary.json \ +OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +bash scripts/run_split_background_promotion_workflow.sh --preflight-only http://192.168.10.150:1202 +``` + The underlying single-category matrix remains available: ```bash diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 55da0686..ff110b1b 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -6350,3 +6350,43 @@ Open: - Run: - `PROMOTION_POSITIVE_PORTFOLIO_PATH=... bash scripts/run_split_background_promotion_workflow.sh http://192.168.10.150:1202` - Review the generated split-aware promotion report before any retraining, recalibration or default-model decision. + +# Sprint 160 - Split-background promotion preflight + +## What changed + +- Added `--preflight-only` to `scripts/run_split_background_promotion_workflow.sh`. +- Preflight now checks: + - `PROMOTION_POSITIVE_PORTFOLIO_PATH` exists and is readable. + - `OPERATOR_SAMPLE_MANIFEST_PATH` exists and is readable. + - the manifest contains both `pure_empty_negative` and `sparse_building_context` background categories. + - Python and `curl` are available. + - the runtime frontend API proxy returns the canonical `data` envelope from `/api/v1/projects`. +- Updated operator docs, TODO and changelog with the quick post-redeploy preflight command. + +## What was tested + +- Red step: `python -m pytest tests/test_sprint159_split_promotion_workflow.py -q` failed because `--preflight-only` was not yet present. +- Ran `python -m pytest tests/test_sprint159_split_promotion_workflow.py -q`: 3 passed. +- Ran `bash -n scripts/run_split_background_promotion_workflow.sh`. +- Ran `python -m pytest tests/test_sprint159_split_promotion_workflow.py tests/test_sprint158_promotion_report_split_background.py -q`: 4 passed. +- Ran `curl.exe -fsS http://192.168.10.150:1202/api/v1/projects`: runtime API proxy returned a canonical `data` envelope. +- Ran `python -m compileall backend/app`. +- Ran `python -m pytest` in `backend`: 447 passed, 17 existing Pydantic namespace warnings. +- Ran `cd frontend && npm run typecheck`. +- Ran `cd frontend && npm run build`. +- Ran `cd backend && python -m alembic heads`: `202606120900 (head)`. +- Ran `cd backend && python -m alembic upgrade head --sql`. +- Ran `bash -n scripts/live_migration_smoke.sh`. +- Ran `bash scripts/run_readiness_check.sh`: passed. + +## Known limitations + +- Preflight is intentionally non-mutating and does not run inference, create datasets, retrain models, activate defaults, fetch providers, fetch weights or change API/database behavior. +- Full split-background matrix execution still requires redeploying this commit to the Tower/Unraid runtime and running the wrapper in the runtime checkout where the positive portfolio and operator manifest artifacts exist. + +## Next recommended pass + +- Redeploy/pull on Tower, then run: + - `PROMOTION_POSITIVE_PORTFOLIO_PATH=... OPERATOR_SAMPLE_MANIFEST_PATH=... bash scripts/run_split_background_promotion_workflow.sh --preflight-only http://192.168.10.150:1202` +- If preflight passes, rerun without `--preflight-only` to generate the split summary and split-aware promotion report. diff --git a/docs/TODO.md b/docs/TODO.md index 0e94f0f7..a5ef0cbd 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -125,6 +125,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add a split background-corpus matrix runner and report builder that runs pure-empty and sparse-context matrices separately. - [x] Teach the model promotion report to consume split background summaries so only `pure_empty_negative` blocks default promotion and `sparse_building_context` stays review-only. - [x] Add one-command operator workflow to run split background matrices and immediately build the split-aware promotion report. +- [x] Add preflight-only validation for the split-background promotion workflow before long runtime matrices. - [ ] Rerun split background matrices on Tower after rebuild, then retrain or recalibrate against the cleaner pure-empty gate plus separate sparse-context inspection matrix. - [ ] Promote a V1 default building detector only after it passes seven positive AOIs, clean hard-negative/background gates and persisted QA/QC evidence without fake detections or model downloads. diff --git a/scripts/README.md b/scripts/README.md index 0735009e..bb844302 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -718,6 +718,18 @@ generated `background_corpus_split_summary.json` into the split-aware promotion report. It still uses only existing upload, detection and report paths; it does not fetch providers, fetch model weights or activate a default. +Run the same command with `--preflight-only` first when checking a fresh +redeploy. Preflight validates the positive portfolio path, operator manifest +presence, required `pure_empty_negative` and `sparse_building_context` +background categories, Python/curl availability and the frontend API proxy +envelope without starting inference: + +```bash +PROMOTION_POSITIVE_PORTFOLIO_PATH=/mnt/user/appdata/geointel/artifacts/detection-quality-matrix/multi-sample/aoi1024bg512r3e50-positive/multi_sample_quality_summary.json \ +OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +bash scripts/run_split_background_promotion_workflow.sh --preflight-only http://192.168.10.150:1202 +``` + If a legacy positive evidence portfolio records `model_asset_id` at portfolio level but does not include per-run tile size/overlap, pass explicit tile defaults instead of letting the report guess: diff --git a/scripts/run_split_background_promotion_workflow.sh b/scripts/run_split_background_promotion_workflow.sh index 8365f950..c67c60ec 100644 --- a/scripts/run_split_background_promotion_workflow.sh +++ b/scripts/run_split_background_promotion_workflow.sh @@ -10,7 +10,7 @@ Usage: QUALITY_TILE_SIZES="512" \ QUALITY_TILE_OVERLAPS="64" \ QUALITY_THRESHOLDS="0.35 0.15" \ - bash scripts/run_split_background_promotion_workflow.sh [base_url] + bash scripts/run_split_background_promotion_workflow.sh [--preflight-only] [base_url] Optional environment: BACKGROUND_SPLIT_OUTPUT_DIR Split matrix output directory. @@ -22,6 +22,7 @@ Optional environment: PROMOTION_MAX_BACKGROUND_DETECTIONS_PER_SAMPLE Default: 0. PROMOTION_DEFAULT_POSITIVE_TILE_SIZE Optional fallback for legacy positive portfolios. PROMOTION_DEFAULT_POSITIVE_TILE_OVERLAP Optional fallback for legacy positive portfolios. + GE_INTEL_BASE_URL Default base URL when no argument is supplied. Runs the split background matrix, then builds a split-aware promotion report. The pure-empty background summary is the strict default gate. Sparse-context @@ -35,11 +36,24 @@ EOF ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" -BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" -if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then - usage - exit 0 -fi +BASE_URL="${GE_INTEL_BASE_URL:-http://localhost:1202}" +PREFLIGHT_ONLY=0 + +while [ "$#" -gt 0 ]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --preflight-only) + PREFLIGHT_ONLY=1 + ;; + *) + BASE_URL="$1" + ;; + esac + shift +done if [ -z "${PROMOTION_POSITIVE_PORTFOLIO_PATH:-}" ]; then echo "PROMOTION_POSITIVE_PORTFOLIO_PATH is required" >&2 @@ -52,6 +66,17 @@ if [ ! -s "${PROMOTION_POSITIVE_PORTFOLIO_PATH}" ]; then exit 1 fi +if [ -z "${OPERATOR_SAMPLE_MANIFEST_PATH:-}" ]; then + echo "OPERATOR_SAMPLE_MANIFEST_PATH is required" >&2 + usage + exit 1 +fi + +if [ ! -s "${OPERATOR_SAMPLE_MANIFEST_PATH}" ]; then + echo "Operator sample manifest is not readable: ${OPERATOR_SAMPLE_MANIFEST_PATH}" >&2 + exit 1 +fi + if [ -n "${PYTHON_BIN:-}" ]; then PYTHON_BIN="${PYTHON_BIN}" else @@ -69,6 +94,49 @@ if [ -z "${PYTHON_BIN}" ]; then exit 1 fi +if ! command -v curl >/dev/null 2>&1; then + echo "curl is required for split-background promotion preflight" >&2 + exit 1 +fi + +"${PYTHON_BIN}" - "${OPERATOR_SAMPLE_MANIFEST_PATH}" <<'PY' +import json +import sys +from pathlib import Path + +manifest_path = Path(sys.argv[1]) +payload = json.loads(manifest_path.read_text(encoding="utf-8-sig")) +samples = payload.get("samples") or payload.get("items") or [] +if not isinstance(samples, list) or not samples: + raise SystemExit(f"Operator sample manifest has no samples: {manifest_path}") + +background_categories = { + str(sample.get("background_category") or "") + for sample in samples + if str(sample.get("sample_role") or "") == "background_candidate" +} +missing = {"pure_empty_negative", "sparse_building_context"} - background_categories +if missing: + raise SystemExit( + "Operator sample manifest is missing background categories " + f"{sorted(missing)}; found {sorted(background_categories)}" + ) +PY + +api_response="$(curl -fsS "${BASE_URL%/}/api/v1/projects" 2>/dev/null || true)" +if ! printf '%s' "${api_response}" | grep -q '"data"'; then + echo "Runtime API proxy is not reachable or did not return the canonical envelope: ${BASE_URL%/}/api/v1/projects" >&2 + exit 1 +fi + +if [ "${PREFLIGHT_ONLY}" = "1" ]; then + echo "Split-background promotion preflight passed" + echo "Base URL: ${BASE_URL}" + echo "Positive portfolio: ${PROMOTION_POSITIVE_PORTFOLIO_PATH}" + echo "Operator sample manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}" + exit 0 +fi + stamp="$(date -u +%Y%m%dT%H%M%SZ)" BACKGROUND_SPLIT_OUTPUT_DIR="${BACKGROUND_SPLIT_OUTPUT_DIR:-artifacts/detection-hard-negatives/background-split/${stamp}}" PROMOTION_OUTPUT_DIR="${PROMOTION_OUTPUT_DIR:-artifacts/detection-model-promotion/split-aware/${stamp}}"