From 558c17129bddd082fbf69cf7188ca398c01e6bb4 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 7 Jul 2026 22:07:56 +0200 Subject: [PATCH] Add operator hard-negative detection matrix --- CHANGELOG.md | 9 + backend/README.md | 18 + ...sprint132_operator_hard_negative_matrix.py | 27 + docs/AI_PIPELINES.md | 19 +- docs/CODEX_EXECUTION_LOG.md | 37 ++ docs/TODO.md | 3 +- scripts/README.md | 23 + ...operator_hard_negative_detection_matrix.sh | 513 ++++++++++++++++++ scripts/run_readiness_check.sh | 1 + 9 files changed, 648 insertions(+), 2 deletions(-) create mode 100644 backend/tests/test_sprint132_operator_hard_negative_matrix.py create mode 100644 scripts/run_operator_hard_negative_detection_matrix.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 892fba9a..ea0204fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ # Changelog +## Sprint 132 Operator hard-negative detection matrix (2026-07-07) + +- Added `scripts/run_operator_hard_negative_detection_matrix.sh` to score configured-YOLO false-positive pressure on documented background-candidate operator AOIs without uploading reference vectors or running QA/QC. +- Added readiness shell-syntax coverage and regression coverage in `backend/tests/test_sprint132_operator_hard_negative_matrix.py`. +- Live Tower 27-run hard-negative matrix compared `geointel-building-yolov8n-expanded160e50-pt`, `geointel-building-yolov8n-tile30-pt` and `yolov8s-building-segmentation-pt` on Postel-bos, Lommel-heide and Kasterlee-bos at thresholds `0.25`/`0.15`/`0.05`. +- Result: `geointel-building-yolov8n-expanded160e50-pt` produced 0 detections on Postel-bos and Lommel-heide at thresholds `0.25` and `0.15`, but produced 38/46/76 detections on Kasterlee-bos at thresholds `0.25`/`0.15`/`0.05`. +- Decision: the expanded local model remains the best dense-AOI candidate, but Kasterlee-bos false-positive pressure blocks it from becoming a V1 default. The next model pass must train against stronger hard-negative coverage or tune per-model threshold/max-detection policy. +- No QA metrics were faked; background scoring is detection-count based only. No provider fetching, fixture detections, model downloads, API contract changes or app-side training behavior were introduced. + ## Sprint 131 Operator sample expansion and negative-tile YOLO candidate (2026-07-07) - Expanded `scripts/prepare_operator_real_data_samples.py` from the original Geel/Mol/Turnhout corpus to 7 reference AOIs plus 3 background-candidate AOIs. diff --git a/backend/README.md b/backend/README.md index a641e2a6..b384974c 100644 --- a/backend/README.md +++ b/backend/README.md @@ -492,6 +492,24 @@ The combined `multi_sample_quality_summary.json` reports per-sample and overall best configurations. It is an operator benchmarking command, not a backend API or provider import path. +Before promoting any local model as a default, also run the hard-negative +matrix against the documented background candidates: + +```bash +OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos" \ +QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-building-yolov8n-tile30-pt yolov8s-building-segmentation-pt" \ +QUALITY_TILE_SIZES="640" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.25 0.15 0.05" \ +bash scripts/run_operator_hard_negative_detection_matrix.sh http://192.168.10.150:1202 +``` + +This path uploads only background rasters, runs configured-YOLO detection and +counts detections as false-positive pressure. It does not upload reference +vectors or run QA/QC, so it cannot produce fake precision/recall metrics for +empty background AOIs. + To inspect the evidence behind a calibration run, export the persisted QA evidence bundle: diff --git a/backend/tests/test_sprint132_operator_hard_negative_matrix.py b/backend/tests/test_sprint132_operator_hard_negative_matrix.py new file mode 100644 index 00000000..ceb285c1 --- /dev/null +++ b/backend/tests/test_sprint132_operator_hard_negative_matrix.py @@ -0,0 +1,27 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_operator_hard_negative_matrix_scores_background_samples_without_qa() -> None: + script_path = ROOT / "scripts" / "run_operator_hard_negative_detection_matrix.sh" + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert script_path.exists() + script = script_path.read_text(encoding="utf-8") + + assert "bash -n scripts/run_operator_hard_negative_detection_matrix.sh" in readiness + assert "OPERATOR_SAMPLE_MANIFEST_PATH" in script + assert "OPERATOR_BACKGROUND_SAMPLE_SLUGS" in script + assert "background_candidate" in script + assert "allow_empty_reference" in script + assert "/api/v1/detection/run" in script + assert "/api/v1/detection/runs/${analysis_run_id}/detections" in script + assert "hard_negative_matrix_summary.json" in script + assert "false_positive_pressure" in script + assert "best_by_lowest_pressure" in script + assert "REAL_REFERENCE_VECTOR_PATH" not in script + assert "/qa/reference" not in script + assert "fixture_mode" not in script + assert "demo/workflow" not in script diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 5cb4303f..421ddb8e 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -215,7 +215,24 @@ and writes `quality_matrix_summary.json` with detection count, QA score, precision, recall, F1, mean IoU and false-positive/false-negative counts. The rankings `best_by_score`, `best_by_recall` and `best_by_precision` are operator decision aids only; GeoIntel still does not download models, seed fixture -detections or treat AI detections as ground truth without QA/QC. +detections or treat AI detections as ground truth without QA/QC. The same +candidate should also pass the background false-positive matrix before it is +considered as a default: + +```bash +OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos" \ +QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-building-yolov8n-tile30-pt yolov8s-building-segmentation-pt" \ +QUALITY_TILE_SIZES="640" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.25 0.15 0.05" \ +bash scripts/run_operator_hard_negative_detection_matrix.sh http://192.168.10.150:1202 +``` + +The hard-negative matrix uploads only background rasters and counts detections +as false-positive pressure. It does not run QA/QC or invent reference metrics +for empty/sparse background AOIs. The first expanded local model improved dense +AOI F1, but Kasterlee-bos false positives block default promotion. To compare the same model/tile/threshold grid across all prepared operator samples, use: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index dc5d1d95..ea047240 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,40 @@ +## Sprint 132 Operator hard-negative detection matrix (2026-07-07) + +Changed: +- Added `scripts/run_operator_hard_negative_detection_matrix.sh`. +- The script reads `operator_samples_manifest.json`, selects samples marked `background_candidate` or `allow_empty_reference`, uploads only the raster, generates a tile manifest, checks configured-YOLO preflight, runs `POST /api/v1/detection/run` and counts persisted detections. +- It intentionally does not upload reference vectors and does not call detection QA/QC endpoints, because background AOIs have no meaningful reference target. +- Added readiness shell-syntax coverage for the new script. +- Added regression coverage in `backend/tests/test_sprint132_operator_hard_negative_matrix.py`. +- Updated `scripts/README.md`, `docs/TODO.md` and `CHANGELOG.md`. + +Tested: +- RED: `python -m pytest backend\tests\test_sprint132_operator_hard_negative_matrix.py -q` failed while `scripts/run_operator_hard_negative_detection_matrix.sh` did not exist. +- `python -m pytest backend\tests\test_sprint132_operator_hard_negative_matrix.py -q` passed. +- `bash -n scripts/run_operator_hard_negative_detection_matrix.sh` passed. +- Live Tower 27-run hard-negative matrix completed: + - output: `/mnt/user/appdata/geointel/artifacts/detection-hard-negatives/expanded160e50-live/hard_negative_matrix_summary.json` + - samples: Postel-bos, Lommel-heide and Kasterlee-bos + - models: `geointel-building-yolov8n-expanded160e50-pt`, `geointel-building-yolov8n-tile30-pt`, `yolov8s-building-segmentation-pt` + - tile size: `640` + - overlap: `64` + - thresholds: `0.25`, `0.15`, `0.05` +- Live false-positive pressure results: + - Postel-bos: expanded160e50 produced 0 detections at `0.25`/`0.15`, 1 at `0.05`; tile30 produced 0/0/1; yolov8s produced 0/3/6. + - Lommel-heide: expanded160e50 produced 0 detections at `0.25`/`0.15`, 10 at `0.05`; tile30 produced 0/0/3; yolov8s produced 0/0/0. + - Kasterlee-bos: expanded160e50 produced 38/46/76 detections at `0.25`/`0.15`/`0.05`; tile30 produced 15/22/42; yolov8s produced 5/6/7. + +Open: +- None for the hard-negative matrix tooling itself. + +Limitations: +- Background matrix scores false-positive pressure from detection counts only. It does not calculate precision/recall/F1 because background candidates intentionally do not provide a full reference target. +- Kasterlee-bos still has 7 GRB features and is best interpreted as a sparse/hard-negative AOI, not a purely empty background tile. +- `geointel-building-yolov8n-expanded160e50-pt` should not be promoted to default model while Kasterlee-bos false-positive pressure remains high. + +Next recommended pass: +- Train a hard-negative-balanced candidate: oversample sparse/background tiles, lower the dense-AOI max-detection bias, and rerun both dense QA matrix and hard-negative matrix before changing any default model selection. + ## Sprint 131 Operator sample expansion and negative-tile YOLO candidate (2026-07-07) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index f0cdc46a..592a92ad 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -105,7 +105,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Train and benchmark the first tile-level local YOLO candidate on Tower through the persisted QA/QC matrix. - [x] Calibrate confidence, IoU and model selection against additional local orthophoto/reference samples beyond Geel/Mol/Turnhout. - [x] Add negative/background AOIs to the operator sample corpus and train an expanded local tile-level YOLO candidate. -- [ ] Add a hard-negative model-quality pass with more sparse/background AOIs, balanced tile export and explicit false-positive scoring. +- [x] Add a hard-negative model-quality pass with sparse/background AOIs and explicit false-positive scoring. +- [ ] Train a hard-negative-balanced YOLO candidate and rerun dense QA plus background false-positive matrices. - [ ] Find or train a materially stronger aerial/Kempen building model candidate; `geointel-building-yolov8n-expanded160e50-pt` is the best current dense-AOI candidate but still too weak and too noisy for a V1 default. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index bfbb0675..575e8997 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -365,6 +365,29 @@ The expanded 50-epoch candidate improved dense Geel/Mol/Turnhout/Retie scores, but the sparse Kasterlee-bos run still showed too many false positives. Treat it as the best current experimental dense-AOI candidate, not as a V1 default. +Run a dedicated hard-negative matrix against documented background candidates +before changing model defaults: + +```bash +OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ +OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos" \ +QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-building-yolov8n-tile30-pt yolov8s-building-segmentation-pt" \ +QUALITY_TILE_SIZES="640" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.25 0.15 0.05" \ +HARD_NEGATIVE_OUTPUT_DIR=artifacts/detection-hard-negatives/expanded160e50-live \ +bash scripts/run_operator_hard_negative_detection_matrix.sh http://192.168.10.150:1202 +``` + +The hard-negative matrix uploads only the background raster, generates tiles, +runs configured-YOLO detection and counts persisted detections as +`false_positive_pressure`. It does not upload a reference vector and does not +run QA/QC, because empty or sparse background AOIs do not have a meaningful +precision/recall target. In the first live run, `geointel-building-yolov8n-expanded160e50-pt` +was clean on Postel-bos and Lommel-heide at thresholds `0.25` and `0.15`, but +produced 38 detections on Kasterlee-bos even at `0.25`. That blocks it from +becoming a V1 default until a hard-negative-balanced candidate improves. + Export calibration QA evidence for visual review: ```bash diff --git a/scripts/run_operator_hard_negative_detection_matrix.sh b/scripts/run_operator_hard_negative_detection_matrix.sh new file mode 100644 index 00000000..f130c1fe --- /dev/null +++ b/scripts/run_operator_hard_negative_detection_matrix.sh @@ -0,0 +1,513 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: + OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ + OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos" \ + QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-building-yolov8n-tile30-pt" \ + QUALITY_THRESHOLDS="0.50 0.25 0.15" \ + bash scripts/run_operator_hard_negative_detection_matrix.sh [base_url] + +Optional environment: + OPERATOR_SAMPLE_MANIFEST_PATH Manifest from prepare_operator_real_data_samples.py. + OPERATOR_BACKGROUND_SAMPLE_SLUGS Optional comma/space separated filter. Defaults to samples marked background_candidate or allow_empty_reference. + HARD_NEGATIVE_OUTPUT_DIR Output directory, default: artifacts/detection-hard-negatives/. + QUALITY_MODEL_ASSET_IDS Space/comma separated local model asset IDs. Default: active configured model asset. + QUALITY_TILE_SIZES Space/comma separated raster tile sizes, default: 640. + QUALITY_TILE_OVERLAPS Space/comma separated raster tile overlaps, default: 64. + QUALITY_THRESHOLDS Space/comma separated confidence thresholds, default: 0.50 0.25 0.15. + +This script scores false-positive pressure on documented operator background +samples only. It runs the existing raster upload, tile, configured-YOLO +preflight and detection API path, then counts detections. It does not upload +reference vectors, run QA/QC, use fixture detections, fetch providers or +download model weights. +EOF +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" +OPERATOR_SAMPLE_MANIFEST_PATH="${OPERATOR_SAMPLE_MANIFEST_PATH:-storage/operator-data/operator_samples_manifest.json}" +OPERATOR_BACKGROUND_SAMPLE_SLUGS="${OPERATOR_BACKGROUND_SAMPLE_SLUGS:-}" +HARD_NEGATIVE_OUTPUT_DIR="${HARD_NEGATIVE_OUTPUT_DIR:-artifacts/detection-hard-negatives/$(date -u +%Y%m%dT%H%M%SZ)}" +QUALITY_MODEL_ASSET_IDS="${QUALITY_MODEL_ASSET_IDS:-${REAL_MODEL_ASSET_ID:-__active__}}" +QUALITY_TILE_SIZES="${QUALITY_TILE_SIZES:-640}" +QUALITY_TILE_OVERLAPS="${QUALITY_TILE_OVERLAPS:-64}" +QUALITY_THRESHOLDS="${QUALITY_THRESHOLDS:-0.50 0.25 0.15}" + +if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then + usage + exit 0 +fi + +if [ ! -f "${OPERATOR_SAMPLE_MANIFEST_PATH}" ]; then + echo "OPERATOR_SAMPLE_MANIFEST_PATH does not point to a readable manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}" >&2 + exit 2 +fi + +if ! command -v curl >/dev/null 2>&1; then + echo "curl is required for hard-negative detection matrix verification" >&2 + exit 1 +fi + +if [ -n "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="${PYTHON_BIN}" +else + PYTHON_BIN="" + for candidate in python3 python.exe python; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then + PYTHON_BIN="${candidate}" + break + fi + done +fi + +if [ -z "${PYTHON_BIN}" ]; then + echo "A Python interpreter is required for JSON parsing" >&2 + exit 1 +fi + +mkdir -p "${HARD_NEGATIVE_OUTPUT_DIR}" +sample_manifest_tsv="${HARD_NEGATIVE_OUTPUT_DIR}/hard_negative_samples.tsv" +matrix_manifest="${HARD_NEGATIVE_OUTPUT_DIR}/hard_negative_requests.tsv" + +"${PYTHON_BIN}" - \ + "${ROOT}" \ + "${OPERATOR_SAMPLE_MANIFEST_PATH}" \ + "${OPERATOR_BACKGROUND_SAMPLE_SLUGS}" \ + "${sample_manifest_tsv}" <<'PY' +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]).resolve() +manifest_path = Path(sys.argv[2]) +slug_filter_raw = sys.argv[3] +output_path = Path(sys.argv[4]) + +payload = json.loads(manifest_path.read_text(encoding="utf-8-sig")) +samples = payload.get("samples") or [] +if not samples: + raise SystemExit("Operator sample manifest contains no samples") + +requested_slugs = { + value.strip().lower() + for value in slug_filter_raw.replace(",", " ").split() + if value.strip() +} + + +def resolve_path(raw: str) -> str: + path = Path(raw) + if path.exists(): + return str(path) + if raw.startswith("/app/"): + candidate = root / raw.removeprefix("/app/") + if candidate.exists(): + return str(candidate) + candidate = root / raw + if candidate.exists(): + return str(candidate) + raise SystemExit(f"Sample file is not readable from this host: {raw}") + + +with output_path.open("w", encoding="utf-8") as handle: + selected = 0 + for sample in samples: + sample_slug = str(sample.get("sample_slug") or "").lower() + sample_role = str(sample.get("sample_role") or "") + allow_empty_reference = bool(sample.get("allow_empty_reference")) + if not sample_slug: + raise SystemExit("Operator sample is missing sample_slug") + if requested_slugs and sample_slug not in requested_slugs: + continue + if not requested_slugs and sample_role != "background_candidate" and not allow_empty_reference: + continue + raster_path = resolve_path(str(sample.get("raster_path") or "")) + reference_count = int(sample.get("reference_feature_count") or 0) + handle.write(f"{sample_slug}\t{raster_path}\t{sample_role}\t{allow_empty_reference}\t{reference_count}\n") + selected += 1 + +if selected == 0: + raise SystemExit("No background_candidate samples matched OPERATOR_BACKGROUND_SAMPLE_SLUGS") +PY + +model_requests_normalized="$(printf '%s' "${QUALITY_MODEL_ASSET_IDS}" | tr ',' ' ')" +tile_sizes_normalized="$(printf '%s' "${QUALITY_TILE_SIZES}" | tr ',' ' ')" +tile_overlaps_normalized="$(printf '%s' "${QUALITY_TILE_OVERLAPS}" | tr ',' ' ')" +thresholds_normalized="$(printf '%s' "${QUALITY_THRESHOLDS}" | tr ',' ' ')" + +"${PYTHON_BIN}" - \ + "${model_requests_normalized}" \ + "${tile_sizes_normalized}" \ + "${tile_overlaps_normalized}" \ + "${thresholds_normalized}" \ + "${matrix_manifest}" <<'PY' +import re +import sys + +models, tile_sizes, tile_overlaps, thresholds, output_path = sys.argv[1:6] + + +def split_values(raw: str, name: str) -> list[str]: + values = [value.strip() for value in raw.split() if value.strip()] + if not values: + raise SystemExit(f"{name} must contain at least one value") + return values + + +def safe_label(value: str) -> str: + label = re.sub(r"[^A-Za-z0-9_.-]+", "_", value.strip()) + return label.replace(".", "p").replace("-", "_") or "active" + + +model_values = split_values(models, "QUALITY_MODEL_ASSET_IDS") +tile_size_values = split_values(tile_sizes, "QUALITY_TILE_SIZES") +tile_overlap_values = split_values(tile_overlaps, "QUALITY_TILE_OVERLAPS") +threshold_values = split_values(thresholds, "QUALITY_THRESHOLDS") + +for value in tile_size_values: + tile_size = int(value) + if tile_size < 64: + raise SystemExit(f"Tile size must be at least 64: {value}") + +for value in tile_overlap_values: + tile_overlap = int(value) + if tile_overlap < 0: + raise SystemExit(f"Tile overlap cannot be negative: {value}") + +for value in threshold_values: + threshold = float(value) + if threshold < 0.0 or threshold > 1.0: + raise SystemExit(f"Confidence threshold must be between 0 and 1: {value}") + +with open(output_path, "w", encoding="utf-8") as handle: + for model in model_values: + for tile_size in tile_size_values: + for overlap in tile_overlap_values: + for threshold in threshold_values: + label = "_".join( + [ + f"model_{safe_label(model)}", + f"tile_{safe_label(tile_size)}", + f"overlap_{safe_label(overlap)}", + f"threshold_{safe_label(threshold)}", + ] + ) + handle.write(f"{model}\t{tile_size}\t{overlap}\t{threshold}\t{label}\n") +PY + +json_field() { + local file_path="$1" + local expression="$2" + "${PYTHON_BIN}" - "$file_path" "$expression" <<'PY' +import json +import sys + +path, expression = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as handle: + payload = json.load(handle) +value = payload +for part in expression.split("."): + if part: + value = value[part] +print(value) +PY +} + +require_json_data() { + local file_path="$1" + "${PYTHON_BIN}" - "$file_path" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as handle: + payload = json.load(handle) +if "data" not in payload: + raise SystemExit("Response is not a canonical GeoIntel data envelope") +PY +} + +echo "== GeoIntel operator hard-negative detection matrix ==" +echo "Base URL: ${BASE_URL}" +echo "Manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}" +echo "Background filter: ${OPERATOR_BACKGROUND_SAMPLE_SLUGS:-background_candidate samples}" +echo "Models: ${model_requests_normalized}" +echo "Tile sizes: ${tile_sizes_normalized}" +echo "Tile overlaps: ${tile_overlaps_normalized}" +echo "Thresholds: ${thresholds_normalized}" +echo "Output: ${HARD_NEGATIVE_OUTPUT_DIR}" + +while IFS=$'\t' read -r sample_slug raster_path sample_role allow_empty_reference reference_feature_count; do + echo "-- Background sample ${sample_slug}: role=${sample_role} allow_empty_reference=${allow_empty_reference} reference_features=${reference_feature_count} --" + while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label; do + sample_output_dir="${HARD_NEGATIVE_OUTPUT_DIR}/${sample_slug}" + mkdir -p "${sample_output_dir}" + tmp_dir="$(mktemp -d)" + run_log="${sample_output_dir}/${run_label}.log" + run_summary="${sample_output_dir}/${run_label}_summary.json" + model_env="${model_request}" + if [ "${model_request}" = "__active__" ]; then + model_env="" + fi + + { + echo "sample=${sample_slug} model=${model_request} tile=${tile_size} overlap=${tile_overlap} threshold=${threshold}" + "${PYTHON_BIN}" - "${tmp_dir}/project_request.json" "${sample_slug}" "${model_request}" "${threshold}" <<'PY' +import json +import sys +from datetime import datetime, timezone + +path, sample_slug, model_request, threshold = sys.argv[1:5] +stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") +payload = { + "name": f"GeoIntel hard-negative {sample_slug} {model_request} {threshold} {stamp}", + "description": "Operator hard-negative validation: raster upload, tiling, configured YOLO detection count, no QA reference.", + "region": "Kempen", +} +with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) +PY + + curl -fsS -X POST "${BASE_URL%/}/api/v1/projects" \ + -H "Content-Type: application/json" \ + --data-binary "@${tmp_dir}/project_request.json" > "${tmp_dir}/project.json" + require_json_data "${tmp_dir}/project.json" + project_id="$(json_field "${tmp_dir}/project.json" "data.id")" + + curl -fsS -X POST "${BASE_URL%/}/api/v1/projects/${project_id}/datasets/upload" \ + -F "file=@${raster_path}" \ + -F "dataset_type=raster" \ + -F "source=user_upload" \ + -F "dataset_role=source" \ + -F "source_name=manual" \ + -F 'source_metadata_json={"validation_workflow":"operator_hard_negative_detection_matrix","input_kind":"orthophoto_background_candidate"}' \ + -F 'provenance_metadata_json={"operator_supplied":true,"no_external_fetch":true}' \ + > "${tmp_dir}/raster_upload.json" + require_json_data "${tmp_dir}/raster_upload.json" + raster_dataset_id="$(json_field "${tmp_dir}/raster_upload.json" "data.id")" + + curl -fsS -X POST "${BASE_URL%/}/api/v1/projects/${project_id}/datasets/${raster_dataset_id}/raster/tile" \ + -H "Content-Type: application/json" \ + -d "{\"tile_size\":${tile_size},\"overlap\":${tile_overlap},\"output_name\":\"hard_negative_detection_tiles\"}" > "${tmp_dir}/tile.json" + require_json_data "${tmp_dir}/tile.json" + manifest_path="$(json_field "${tmp_dir}/tile.json" "data.result_json.manifest_path")" + + curl -fsS "${BASE_URL%/}/api/v1/detection/model-assets" > "${tmp_dir}/model_assets.json" + require_json_data "${tmp_dir}/model_assets.json" + model_asset_id="$("${PYTHON_BIN}" - "${tmp_dir}/model_assets.json" "${model_env}" <<'PY' +import json +import sys + +path, requested = sys.argv[1:3] +with open(path, "r", encoding="utf-8") as handle: + items = json.load(handle)["data"].get("items") or [] +if requested: + match = next((item for item in items if item.get("model_asset_id") == requested), None) + if not match: + raise SystemExit(f"Requested model asset was not found: {requested}") + print(match["model_asset_id"]) +else: + active = next((item for item in items if item.get("active")), None) + if not active: + raise SystemExit("No active local model asset found; set QUALITY_MODEL_ASSET_IDS") + print(active["model_asset_id"]) +PY +)" + + curl -fsS -G "${BASE_URL%/}/api/v1/detection/yolo/preflight" \ + --data-urlencode "tile_manifest_path=${manifest_path}" \ + --data-urlencode "model_asset_id=${model_asset_id}" \ + --data-urlencode "check_model_load=false" > "${tmp_dir}/preflight.json" + require_json_data "${tmp_dir}/preflight.json" + "${PYTHON_BIN}" - "${tmp_dir}/preflight.json" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +if data.get("status") != "ready": + raise SystemExit(f"YOLO preflight is not ready: {data.get('status')} {data.get('message')}") +if data.get("will_download_models") is not False: + raise SystemExit("YOLO preflight reported possible model download") +PY + + "${PYTHON_BIN}" - "${tmp_dir}/run_request.json" "${project_id}" "${raster_dataset_id}" "${model_asset_id}" "${manifest_path}" "${threshold}" <<'PY' +import json +import sys + +path, project_id, dataset_id, model_asset_id, tile_manifest_path, confidence = sys.argv[1:7] +payload = { + "project_id": project_id, + "dataset_id": dataset_id, + "model_id": "yolo-configured", + "model_asset_id": model_asset_id, + "confidence_threshold": float(confidence), + "tile_manifest_path": tile_manifest_path, +} +with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) +PY + + curl -fsS -X POST "${BASE_URL%/}/api/v1/detection/run" \ + -H "Content-Type: application/json" \ + --data-binary "@${tmp_dir}/run_request.json" > "${tmp_dir}/detection_run.json" + require_json_data "${tmp_dir}/detection_run.json" + analysis_run_id="$(json_field "${tmp_dir}/detection_run.json" "data.analysis_run_id")" + detection_count="$(json_field "${tmp_dir}/detection_run.json" "data.detection_count")" + + curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}/detections" > "${tmp_dir}/detections.json" + require_json_data "${tmp_dir}/detections.json" + detections_list_count="$("${PYTHON_BIN}" - "${tmp_dir}/detections.json" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as handle: + data = json.load(handle)["data"] +print(len(data.get("items") or [])) +PY +)" + + tile_count="$(json_field "${tmp_dir}/preflight.json" "data.tile_count")" + "${PYTHON_BIN}" - \ + "${run_summary}" \ + "${sample_slug}" \ + "${sample_role}" \ + "${allow_empty_reference}" \ + "${reference_feature_count}" \ + "${model_request}" \ + "${model_asset_id}" \ + "${tile_size}" \ + "${tile_overlap}" \ + "${threshold}" \ + "${project_id}" \ + "${raster_dataset_id}" \ + "${manifest_path}" \ + "${analysis_run_id}" \ + "${detection_count}" \ + "${detections_list_count}" \ + "${tile_count}" \ + "${run_log}" <<'PY' +import json +import sys + +( + output_path, + sample_slug, + sample_role, + allow_empty_reference, + reference_feature_count, + model_request, + model_asset_id, + tile_size, + tile_overlap, + threshold, + project_id, + raster_dataset_id, + manifest_path, + analysis_run_id, + detection_count, + detections_list_count, + tile_count, + run_log, +) = sys.argv[1:19] + +detections = int(detection_count) +listed = int(detections_list_count) +tiles = max(int(tile_count), 1) +if detections != listed: + raise SystemExit(f"Detection run count {detections} did not match list count {listed}") +summary = { + "sample_slug": sample_slug, + "sample_role": sample_role, + "allow_empty_reference": allow_empty_reference == "True", + "reference_feature_count": int(reference_feature_count), + "model_request": model_request, + "model_asset_id": model_asset_id, + "tile_size": int(tile_size), + "tile_overlap": int(tile_overlap), + "threshold": float(threshold), + "project_id": project_id, + "raster_dataset_id": raster_dataset_id, + "manifest_path": manifest_path, + "analysis_run_id": analysis_run_id, + "tile_count": tiles, + "detection_count": detections, + "false_positive_pressure": detections / tiles, + "run_log": run_log, +} +with open(output_path, "w", encoding="utf-8") as handle: + json.dump(summary, handle, indent=2, sort_keys=True) +print( + "sample={sample_slug} model={model_asset_id} tile={tile_size} overlap={tile_overlap} " + "threshold={threshold} detections={detection_count} false_positive_pressure={false_positive_pressure}".format( + **summary + ) +) +PY + } >"${run_log}" 2>&1 + + cat "${run_log}" | tail -n 1 + rm -rf "${tmp_dir}" + done < "${matrix_manifest}" +done < "${sample_manifest_tsv}" + +"${PYTHON_BIN}" - "${HARD_NEGATIVE_OUTPUT_DIR}" "${BASE_URL}" "${OPERATOR_SAMPLE_MANIFEST_PATH}" <<'PY' +import glob +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +output_dir = Path(sys.argv[1]) +base_url = sys.argv[2] +manifest_path = sys.argv[3] + +items = [] +for path in sorted(glob.glob(str(output_dir / "*" / "*_summary.json"))): + with open(path, "r", encoding="utf-8") as handle: + items.append(json.load(handle)) + +if not items: + raise SystemExit("No hard-negative detection summaries were produced") + + +def key_lowest_pressure(item: dict) -> tuple[float, int]: + return (float(item.get("false_positive_pressure") or 0), int(item.get("detection_count") or 0)) + + +best_by_lowest_pressure = min(items, key=key_lowest_pressure) +summary = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "base_url": base_url, + "operator_sample_manifest_path": manifest_path, + "sample_count": len({item["sample_slug"] for item in items}), + "run_count": len(items), + "best_by_lowest_pressure": best_by_lowest_pressure, + "items": items, +} +summary_path = output_dir / "hard_negative_matrix_summary.json" +summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8") + +print("") +print("Operator hard-negative detection summary") +print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\tfalse_positive_pressure") +for item in items: + print( + "{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{false_positive_pressure}".format( + **item + ) + ) +print("") +print(f"Summary: {summary_path}") +item = best_by_lowest_pressure +print( + f"best_by_lowest_pressure sample={item['sample_slug']} model={item['model_asset_id']} " + f"tile={item['tile_size']} overlap={item['tile_overlap']} threshold={item['threshold']:.2f} " + f"detections={item['detection_count']} false_positive_pressure={item['false_positive_pressure']}" +) +PY diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 29c601dc..537f4d8d 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -62,6 +62,7 @@ bash -n scripts/run_detection_calibration_sweep.sh bash -n scripts/export_detection_calibration_evidence.sh bash -n scripts/run_detection_quality_matrix.sh bash -n scripts/run_multi_sample_detection_quality_matrix.sh +bash -n scripts/run_operator_hard_negative_detection_matrix.sh bash -n scripts/train_operator_yolo_detector.sh bash -n scripts/verify_workbench_default_state.sh bash -n scripts/verify_workbench_interactions.sh