From 673f6c616006fbea206747aea75b225a219eedf7 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 8 Jul 2026 13:41:39 +0200 Subject: [PATCH] Support browser calibration summaries in evidence export --- CHANGELOG.md | 7 ++ ...ser_calibration_summary_evidence_script.py | 25 ++++++ docs/CODEX_EXECUTION_LOG.md | 24 ++++++ docs/TODO.md | 1 + scripts/README.md | 6 ++ .../export_detection_calibration_evidence.sh | 82 ++++++++++++++++++- 6 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 backend/tests/test_sprint137_browser_calibration_summary_evidence_script.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7341aa1c..1714fdaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 137 Browser calibration summary evidence bundle handoff (2026-07-08) + +- Extended `scripts/export_detection_calibration_evidence.sh` so it accepts Detection Lab `detection-calibration-summary.json` browser exports in addition to the older operator calibration summary format. +- Added summary normalization for browser-exported `rows`, root `project_id`, persisted `quality_check_id` values and best-mode fallback selection. +- Updated operator docs with the direct browser summary command. +- No backend API, migration, inference, provider fetching, model download or frontend runtime behavior changed. + ## Sprint 136 Guided calibration summary export (2026-07-08) - Added a Detection Lab `Download calibration summary` action for guided calibration rows. diff --git a/backend/tests/test_sprint137_browser_calibration_summary_evidence_script.py b/backend/tests/test_sprint137_browser_calibration_summary_evidence_script.py new file mode 100644 index 00000000..2854f66a --- /dev/null +++ b/backend/tests/test_sprint137_browser_calibration_summary_evidence_script.py @@ -0,0 +1,25 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_evidence_bundle_script_accepts_browser_calibration_summary_export() -> None: + script_path = ROOT / "scripts" / "export_detection_calibration_evidence.sh" + readme_path = ROOT / "scripts" / "README.md" + todo_path = ROOT / "docs" / "TODO.md" + + script = script_path.read_text(encoding="utf-8") + readme = readme_path.read_text(encoding="utf-8") + todo = todo_path.read_text(encoding="utf-8") + + assert "detection-calibration-summary.json" in script + assert "export_type" in script + assert "detection_calibration_summary" in script + assert "normalize_calibration_items" in script + assert "summary.get(\"rows\")" in script + assert "root_project_id = summary.get(\"project_id\")" in script + assert "quality_check_ids" in script + assert "Browser Detection Lab calibration summary" in readme + assert "bash scripts/export_detection_calibration_evidence.sh http://192.168.10.150:1202 ./detection-calibration-summary.json" in readme + assert "[x] Allow the evidence bundle script to consume Detection Lab calibration summary exports" in todo diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 583b7f45..86cd449c 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,27 @@ +## Sprint 137 Browser calibration summary evidence bundle handoff (2026-07-08) + +Changed: +- Extended `scripts/export_detection_calibration_evidence.sh` so it can consume Detection Lab `detection-calibration-summary.json` browser exports as well as the existing operator `calibration_summary.json` format. +- Added summary normalization for browser-exported `rows`, root `project_id`, persisted `quality_check_id` values and `CALIBRATION_EVIDENCE_MODE=best` fallback selection. +- Updated `scripts/README.md`, `CHANGELOG.md` and `docs/TODO.md`. +- Added regression coverage in `backend/tests/test_sprint137_browser_calibration_summary_evidence_script.py`. + +Tested: +- Red step: `python -m pytest backend\tests\test_sprint137_browser_calibration_summary_evidence_script.py -q` failed while browser summary support was absent. +- `python -m pytest backend\tests\test_sprint137_browser_calibration_summary_evidence_script.py backend\tests\test_sprint136_calibration_summary_export_ui.py backend\tests\test_sprint125_detection_calibration_evidence_bundle.py -q` (`3 passed`) +- `bash -n scripts/export_detection_calibration_evidence.sh` +- `python -m compileall backend/app` +- `bash scripts/run_readiness_check.sh` (`416 passed`; frontend typecheck/build passed; Alembic head `202606120900`; live smoke syntax passed) + +Open: +- Live Tower deploy validation still needs to run for this pass. + +Limitations: +- This is operator evidence tooling only. It does not add backend endpoints, change migrations, rerun inference, create new QA metrics, promote thresholds, mutate model configuration, download models, add provider fetching or change frontend runtime behavior. + +Next recommended pass: +- Add a tiny local fixture smoke for the evidence bundle script that uses a saved browser-style summary plus mocked canonical evidence responses, so the bundle renderer itself is tested beyond static contract checks. + ## Sprint 136 Guided calibration summary export (2026-07-08) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index dc001def..76e9e7db 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -423,5 +423,6 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add guided in-app detection calibration runner for explicit threshold sweeps. - [x] Link guided calibration rows to the QA evidence map. - [x] Add guided calibration summary export from the Detection Lab. +- [x] Allow the evidence bundle script to consume Detection Lab calibration summary exports. - [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout. - [ ] Add negative/background AOIs so the next tile dataset is not all positive tiles. diff --git a/scripts/README.md b/scripts/README.md index c56140c3..8e981e2f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -442,6 +442,12 @@ CALIBRATION_SUMMARY_PATH=/mnt/user/appdata/geointel/artifacts/detection-calibrat bash scripts/export_detection_calibration_evidence.sh http://192.168.10.150:1202 ``` +Browser Detection Lab calibration summary exports are supported too: + +```bash +bash scripts/export_detection_calibration_evidence.sh http://192.168.10.150:1202 ./detection-calibration-summary.json +``` + The evidence export reads each persisted `quality_check_id`, calls the existing QA evidence GeoJSON endpoint, writes `calibration_evidence.geojson`, `calibration_evidence_summary.json` and a standalone diff --git a/scripts/export_detection_calibration_evidence.sh b/scripts/export_detection_calibration_evidence.sh index 3ccf5dd5..4922b7c5 100644 --- a/scripts/export_detection_calibration_evidence.sh +++ b/scripts/export_detection_calibration_evidence.sh @@ -10,8 +10,12 @@ Usage: or: bash scripts/export_detection_calibration_evidence.sh [base_url] /path/to/calibration_summary.json +or with the browser Detection Lab export: + bash scripts/export_detection_calibration_evidence.sh [base_url] /path/to/detection-calibration-summary.json + Required input: - CALIBRATION_SUMMARY_PATH calibration_summary.json produced by run_detection_calibration_sweep.sh. + CALIBRATION_SUMMARY_PATH calibration_summary.json produced by run_detection_calibration_sweep.sh + or detection-calibration-summary.json from the Detection Lab. Optional environment: CALIBRATION_EVIDENCE_MODE all or best, default: all. @@ -86,9 +90,42 @@ summary_path, manifest_path, mode = sys.argv[1:4] with open(summary_path, "r", encoding="utf-8") as handle: summary = json.load(handle) -items = summary.get("items") or [] +def normalize_calibration_items(summary): + root_project_id = summary.get("project_id") + if summary.get("export_type") == "detection_calibration_summary": + rows = summary.get("rows") or [] + quality_check_ids = summary.get("quality_check_ids") or [] + items = [] + for row in rows: + if not isinstance(row, dict): + continue + quality_check_id = row.get("quality_check_id") + if not quality_check_id: + continue + items.append( + { + "project_id": row.get("project_id") or root_project_id, + "analysis_run_id": row.get("analysis_run_id"), + "job_id": row.get("job_id"), + "quality_check_id": quality_check_id, + "threshold": row.get("threshold"), + "detection_count": row.get("detection_count"), + "quality_score": row.get("quality_score") or row.get("f1_score"), + "precision": row.get("precision"), + "recall": row.get("recall"), + "f1_score": row.get("f1_score"), + } + ) + if quality_check_ids and not items: + raise SystemExit("detection-calibration-summary.json has quality_check_ids but no export rows") + return items + return summary.get("items") or [] + +items = normalize_calibration_items(summary) if mode == "best": best = summary.get("best_by_score") + if best is None and items: + best = max(items, key=lambda item: item.get("quality_score") if isinstance(item.get("quality_score"), (int, float)) else float("-inf")) if not isinstance(best, dict): raise SystemExit("calibration_summary.json has no best_by_score object") items = [best] @@ -133,9 +170,46 @@ summary_path, output_dir, mode = sys.argv[1:4] with open(summary_path, "r", encoding="utf-8") as handle: calibration_summary = json.load(handle) -selected_items = calibration_summary.get("items") or [] +def normalize_calibration_items(summary): + root_project_id = summary.get("project_id") + if summary.get("export_type") == "detection_calibration_summary": + rows = summary.get("rows") or [] + quality_check_ids = summary.get("quality_check_ids") or [] + items = [] + for row in rows: + if not isinstance(row, dict): + continue + quality_check_id = row.get("quality_check_id") + if not quality_check_id: + continue + items.append( + { + "project_id": row.get("project_id") or root_project_id, + "analysis_run_id": row.get("analysis_run_id"), + "job_id": row.get("job_id"), + "quality_check_id": quality_check_id, + "threshold": row.get("threshold"), + "detection_count": row.get("detection_count"), + "quality_score": row.get("quality_score") or row.get("f1_score"), + "precision": row.get("precision"), + "recall": row.get("recall"), + "f1_score": row.get("f1_score"), + } + ) + if quality_check_ids and not items: + raise SystemExit("detection-calibration-summary.json has quality_check_ids but no export rows") + return items + return summary.get("items") or [] + +selected_items = normalize_calibration_items(calibration_summary) if mode == "best": - selected_items = [calibration_summary["best_by_score"]] + best = calibration_summary.get("best_by_score") + if best is None and selected_items: + best = max( + selected_items, + key=lambda item: item.get("quality_score") if isinstance(item.get("quality_score"), (int, float)) else float("-inf"), + ) + selected_items = [best] items_by_quality_check = {str(item["quality_check_id"]): item for item in selected_items} combined_features = []