From 947ad6784db2576560e68a48b689429b33ddf29b Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Jul 2026 12:12:50 +0200 Subject: [PATCH] Add split background promotion workflow --- CHANGELOG.md | 7 ++ ...test_sprint159_split_promotion_workflow.py | 29 +++++ docs/AI_PIPELINES.md | 18 +++ docs/CODEX_EXECUTION_LOG.md | 38 ++++++ docs/TODO.md | 1 + scripts/README.md | 19 +++ scripts/run_readiness_check.sh | 1 + ...run_split_background_promotion_workflow.sh | 115 ++++++++++++++++++ 8 files changed, 228 insertions(+) create mode 100644 backend/tests/test_sprint159_split_promotion_workflow.py create mode 100644 scripts/run_split_background_promotion_workflow.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index fc8cdbe6..ca5af232 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 159 Split-background promotion workflow wrapper (2026-07-10) + +- Added `scripts/run_split_background_promotion_workflow.sh` to run the split background matrix and split-aware promotion report from one operator command. +- Added readiness syntax coverage and regression tests for the wrapper contract. +- Documented the one-command Tower/runtime flow for the inactive AOI1024 background-aware model candidate. +- No model default, backend API, database migration, provider fetching, fake detection output or model download behavior changed. + ## Sprint 158 Split-aware promotion report (2026-07-10) - Added `--background-split-summary` support to `scripts/build_detection_model_promotion_report.py`. diff --git a/backend/tests/test_sprint159_split_promotion_workflow.py b/backend/tests/test_sprint159_split_promotion_workflow.py new file mode 100644 index 00000000..8dc7b0bc --- /dev/null +++ b/backend/tests/test_sprint159_split_promotion_workflow.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_split_background_promotion_workflow_runs_split_then_split_aware_report() -> None: + script_path = ROOT / "scripts" / "run_split_background_promotion_workflow.sh" + assert script_path.exists() + source = script_path.read_text(encoding="utf-8") + + assert "run_background_corpus_split_matrix.sh" in source + assert "build_detection_model_promotion_report.py" in source + assert "--background-split-summary" in source + assert "background_corpus_split_summary.json" in source + assert "PROMOTION_POSITIVE_PORTFOLIO_PATH" in source + assert "BACKGROUND_SPLIT_OUTPUT_DIR" in source + assert "PROMOTION_OUTPUT_DIR" in source + assert "--hard-negative-summary" not in source + assert "download model" not in source.lower() + assert "promote model default" not in source.lower() + + +def test_readiness_checks_split_background_promotion_workflow_syntax() -> None: + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert "bash -n scripts/run_split_background_promotion_workflow.sh" in readiness diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 08c3deea..6315fd00 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -278,6 +278,24 @@ only strict background gate for default promotion, while `sparse_building_context` remains review-only evidence in the report. This keeps contextual buildings from being treated as empty-background false positives. +For the Tower/runtime pass, run the split matrix and promotion report together: + +```bash +PROMOTION_POSITIVE_PORTFOLIO_PATH=artifacts/detection-quality-matrix/multi-sample/aoi1024bg512r3e50-positive/multi_sample_quality_summary.json \ +OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8s-aoi1024bg512r3e50-pt" \ +QUALITY_TILE_SIZES="512" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.35 0.15" \ +BACKGROUND_SPLIT_OUTPUT_DIR=artifacts/detection-hard-negatives/aoi1024bg512r3e50-split \ +PROMOTION_OUTPUT_DIR=artifacts/detection-model-promotion/aoi1024bg512r3e50-split-aware \ +bash scripts/run_split_background_promotion_workflow.sh http://192.168.10.150:1202 +``` + +The wrapper keeps the same safety boundary: existing dataset upload, configured +YOLO detection and report tooling only. It does not change model configuration +or bypass the persisted QA/QC evidence requirement. + The underlying single-category matrix remains available: ```bash diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index c052c47f..55da0686 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -6312,3 +6312,41 @@ Open: - After redeploy, run `scripts/run_background_corpus_split_matrix.sh` on Tower for `geointel-building-yolov8s-aoi1024bg512r3e50-pt`. - Feed the generated `background_corpus_split_summary.json` into `scripts/build_detection_model_promotion_report.py --background-split-summary` together with the seven-AOI positive summary. - If pure-empty false-positive pressure still fails, retrain or recalibrate before any default activation. If pure-empty passes, inspect sparse-context review evidence before deciding whether to keep the model operator-only or prepare a guarded default-candidate decision. + +# Sprint 159 - Split-background promotion workflow wrapper + +## What changed + +- Added `scripts/run_split_background_promotion_workflow.sh`. +- The wrapper runs `scripts/run_background_corpus_split_matrix.sh`, verifies the emitted `background_corpus_split_summary.json`, then calls `scripts/build_detection_model_promotion_report.py --background-split-summary`. +- The wrapper requires `PROMOTION_POSITIVE_PORTFOLIO_PATH` and exposes promotion gate overrides through environment variables. +- Added readiness syntax coverage for the new wrapper. +- Updated operator pipeline docs, TODO and changelog. + +## What was tested + +- Red step: `python -m pytest tests/test_sprint159_split_promotion_workflow.py -q` failed because the wrapper script and readiness syntax check did not exist. +- Ran `python -m pytest tests/test_sprint159_split_promotion_workflow.py -q`: 2 passed. +- Ran `bash -n scripts/run_split_background_promotion_workflow.sh`. +- Ran `python -m pytest tests/test_sprint159_split_promotion_workflow.py tests/test_sprint158_promotion_report_split_background.py tests/test_sprint157_background_split_matrix_runner.py -q`: 7 passed. +- Ran `bash -n scripts/run_readiness_check.sh`. +- Ran `python -m compileall backend/app`. +- Ran `python -m pytest` in `backend`: 446 passed, 17 existing Pydantic namespace warnings. +- Ran `cd frontend && npm run typecheck`. +- Ran `cd frontend && npm run build`. +- Ran `cd backend && python -m alembic heads`: `202606120900 (head)`. +- Ran `cd backend && python -m alembic upgrade head --sql`. +- Ran `bash -n scripts/live_migration_smoke.sh`. +- Ran `bash scripts/run_readiness_check.sh`: passed. + +## Known limitations + +- This pass adds operator orchestration only. It does not run live inference locally, retrain YOLO, activate a model default, change backend API contracts, change migrations, fetch providers, fetch weights or create fake detections. +- `http://192.168.10.150:1202` is reachable from this workstation, but `ssh -p 222 root@192.168.10.150` returned `Permission denied (publickey)`. Port `222` appears to be the Gitea SSH endpoint rather than an Unraid shell endpoint, so Docker rebuild/restart still needs the existing Unraid deployment path or a separate shell-access route. + +## Next recommended pass + +- Redeploy/pull commit on Tower through the existing Gitea/Unraid flow. +- Run: + - `PROMOTION_POSITIVE_PORTFOLIO_PATH=... bash scripts/run_split_background_promotion_workflow.sh http://192.168.10.150:1202` +- Review the generated split-aware promotion report before any retraining, recalibration or default-model decision. diff --git a/docs/TODO.md b/docs/TODO.md index 7bca0d45..0e94f0f7 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -124,6 +124,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add pure-empty versus sparse-building contextual background corpus classification to operator manifests, hard-negative matrix filters and YOLO tile provenance. - [x] Add a split background-corpus matrix runner and report builder that runs pure-empty and sparse-context matrices separately. - [x] Teach the model promotion report to consume split background summaries so only `pure_empty_negative` blocks default promotion and `sparse_building_context` stays review-only. +- [x] Add one-command operator workflow to run split background matrices and immediately build the split-aware promotion report. - [ ] Rerun split background matrices on Tower after rebuild, then retrain or recalibrate against the cleaner pure-empty gate plus separate sparse-context inspection matrix. - [ ] Promote a V1 default building detector only after it passes seven positive AOIs, clean hard-negative/background gates and persisted QA/QC evidence without fake detections or model downloads. diff --git a/scripts/README.md b/scripts/README.md index 170aeba7..0735009e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -699,6 +699,25 @@ summary as the strict default-promotion false-positive gate. The `sparse_building_context` source remains visible in the JSON/Markdown report as review evidence only and is not counted as a default-promotion gate. +To run both steps after one redeploy, use the workflow wrapper: + +```bash +PROMOTION_POSITIVE_PORTFOLIO_PATH=/mnt/user/appdata/geointel/artifacts/detection-quality-matrix/multi-sample/aoi1024bg512r3e50-positive/multi_sample_quality_summary.json \ +OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8s-aoi1024bg512r3e50-pt" \ +QUALITY_TILE_SIZES="512" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.35 0.15" \ +BACKGROUND_SPLIT_OUTPUT_DIR=artifacts/detection-hard-negatives/aoi1024bg512r3e50-split \ +PROMOTION_OUTPUT_DIR=artifacts/detection-model-promotion/aoi1024bg512r3e50-split-aware \ +bash scripts/run_split_background_promotion_workflow.sh http://192.168.10.150:1202 +``` + +The wrapper first calls `run_background_corpus_split_matrix.sh`, then feeds the +generated `background_corpus_split_summary.json` into the split-aware promotion +report. It still uses only existing upload, detection and report paths; it does +not fetch providers, fetch model weights or activate a default. + If a legacy positive evidence portfolio records `model_asset_id` at portfolio level but does not include per-run tile size/overlap, pass explicit tile defaults instead of letting the report guess: diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 21126df7..6a3af4ad 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -69,6 +69,7 @@ bash -n scripts/run_detection_quality_matrix.sh bash -n scripts/run_multi_sample_detection_quality_matrix.sh bash -n scripts/run_operator_hard_negative_detection_matrix.sh bash -n scripts/run_background_corpus_split_matrix.sh +bash -n scripts/run_split_background_promotion_workflow.sh bash -n scripts/train_operator_yolo_detector.sh bash -n scripts/verify_workbench_default_state.sh bash -n scripts/verify_workbench_interactions.sh diff --git a/scripts/run_split_background_promotion_workflow.sh b/scripts/run_split_background_promotion_workflow.sh new file mode 100644 index 00000000..8365f950 --- /dev/null +++ b/scripts/run_split_background_promotion_workflow.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: + PROMOTION_POSITIVE_PORTFOLIO_PATH=artifacts/detection-quality-matrix/multi-sample/aoi1024-positive/multi_sample_quality_summary.json \ + OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ + QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8s-aoi1024bg512r3e50-pt" \ + QUALITY_TILE_SIZES="512" \ + QUALITY_TILE_OVERLAPS="64" \ + QUALITY_THRESHOLDS="0.35 0.15" \ + bash scripts/run_split_background_promotion_workflow.sh [base_url] + +Optional environment: + BACKGROUND_SPLIT_OUTPUT_DIR Split matrix output directory. + PROMOTION_OUTPUT_DIR Promotion report output directory. + PROMOTION_POSITIVE_PORTFOLIO_PATH Positive AOI portfolio or multi-sample summary. + PROMOTION_MIN_POSITIVE_SAMPLES Default: 7. + PROMOTION_MIN_BACKGROUND_SAMPLES Default: 2. + PROMOTION_MIN_MEAN_F1 Default: 0.25. + PROMOTION_MAX_BACKGROUND_DETECTIONS_PER_SAMPLE Default: 0. + PROMOTION_DEFAULT_POSITIVE_TILE_SIZE Optional fallback for legacy positive portfolios. + PROMOTION_DEFAULT_POSITIVE_TILE_OVERLAP Optional fallback for legacy positive portfolios. + +Runs the split background matrix, then builds a split-aware promotion report. +The pure-empty background summary is the strict default gate. Sparse-context +background evidence is included for operator review only. + +The script uses existing runtime/API flows and report tooling only. It does not +fetch providers, fetch weights, mutate model settings or activate a default. +EOF +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" +if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then + usage + exit 0 +fi + +if [ -z "${PROMOTION_POSITIVE_PORTFOLIO_PATH:-}" ]; then + echo "PROMOTION_POSITIVE_PORTFOLIO_PATH is required" >&2 + usage + exit 1 +fi + +if [ ! -s "${PROMOTION_POSITIVE_PORTFOLIO_PATH}" ]; then + echo "Positive portfolio is not readable: ${PROMOTION_POSITIVE_PORTFOLIO_PATH}" >&2 + exit 1 +fi + +if [ -n "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="${PYTHON_BIN}" +else + PYTHON_BIN="" + for candidate in python3 python.exe python; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then + PYTHON_BIN="${candidate}" + break + fi + done +fi + +if [ -z "${PYTHON_BIN}" ]; then + echo "A Python interpreter is required for split-background promotion reporting" >&2 + exit 1 +fi + +stamp="$(date -u +%Y%m%dT%H%M%SZ)" +BACKGROUND_SPLIT_OUTPUT_DIR="${BACKGROUND_SPLIT_OUTPUT_DIR:-artifacts/detection-hard-negatives/background-split/${stamp}}" +PROMOTION_OUTPUT_DIR="${PROMOTION_OUTPUT_DIR:-artifacts/detection-model-promotion/split-aware/${stamp}}" + +echo "== GeoIntel split-background promotion workflow ==" +echo "Base URL: ${BASE_URL}" +echo "Positive portfolio: ${PROMOTION_POSITIVE_PORTFOLIO_PATH}" +echo "Background split output: ${BACKGROUND_SPLIT_OUTPUT_DIR}" +echo "Promotion output: ${PROMOTION_OUTPUT_DIR}" + +BACKGROUND_SPLIT_OUTPUT_DIR="${BACKGROUND_SPLIT_OUTPUT_DIR}" \ +bash scripts/run_background_corpus_split_matrix.sh "${BASE_URL}" + +split_summary="${BACKGROUND_SPLIT_OUTPUT_DIR}/background_corpus_split_summary.json" +if [ ! -s "${split_summary}" ]; then + echo "Split summary was not produced: ${split_summary}" >&2 + exit 1 +fi + +mkdir -p "${PROMOTION_OUTPUT_DIR}" + +promotion_args=( + scripts/build_detection_model_promotion_report.py + --positive-portfolio "${PROMOTION_POSITIVE_PORTFOLIO_PATH}" + --background-split-summary "${split_summary}" + --output-dir "${PROMOTION_OUTPUT_DIR}" + --min-positive-samples "${PROMOTION_MIN_POSITIVE_SAMPLES:-7}" + --min-background-samples "${PROMOTION_MIN_BACKGROUND_SAMPLES:-2}" + --min-mean-f1 "${PROMOTION_MIN_MEAN_F1:-0.25}" + --max-background-detections-per-sample "${PROMOTION_MAX_BACKGROUND_DETECTIONS_PER_SAMPLE:-0}" +) + +if [ -n "${PROMOTION_DEFAULT_POSITIVE_TILE_SIZE:-}" ]; then + promotion_args+=(--default-positive-tile-size "${PROMOTION_DEFAULT_POSITIVE_TILE_SIZE}") +fi + +if [ -n "${PROMOTION_DEFAULT_POSITIVE_TILE_OVERLAP:-}" ]; then + promotion_args+=(--default-positive-tile-overlap "${PROMOTION_DEFAULT_POSITIVE_TILE_OVERLAP}") +fi + +"${PYTHON_BIN}" "${promotion_args[@]}" + +echo "Split summary: ${split_summary}" +echo "Promotion report: ${PROMOTION_OUTPUT_DIR}/detection_model_promotion_report.json"