From 51b7d1eec7df4a36d463eb9693957deea2fab5ee Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 8 Jul 2026 14:25:01 +0200 Subject: [PATCH] Preserve calibration portfolio model provenance --- CHANGELOG.md | 8 +++ ...ulti_aoi_calibration_evidence_portfolio.py | 11 ++++ docs/CODEX_EXECUTION_LOG.md | 54 +++++++++++++++++++ docs/TODO.md | 2 + .../export_detection_calibration_evidence.sh | 16 ++++++ 5 files changed, 91 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c5dca27..aa88e07e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ # Changelog +## Sprint 141 Expanded positive-AOI matrix and portfolio metadata hardening (2026-07-08) + +- Ran a fresh Tower quality matrix for Balen, Herentals and Westerlo using `geointel-building-yolov8n-expanded160e50-pt` and `geointel-building-yolov8n-hardneg160r8e40-pt`. +- Assembled an expanded 7-AOI positive evidence portfolio across Geel, Mol, Turnhout, Retie, Balen, Herentals and Westerlo. +- Hardened calibration evidence exports so model asset id, model request, tile size and tile overlap survive into evidence bundle summaries and GeoJSON properties. +- Result: expanded160e50 is stronger on positive AOIs, with Westerlo reaching F1 `0.3659305993690852`, but hard-negative matrices still show false-positive pressure tradeoffs that prevent blind default promotion. +- No backend API, migration, frontend runtime, model weight, provider fetching or Docker runtime change was introduced. + ## Sprint 140 Live multi-AOI calibration portfolio run (2026-07-08) - Ran the new multi-AOI calibration evidence portfolio assembler on Tower against existing persisted quality-matrix summaries for Geel, Mol and Turnhout. diff --git a/backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py b/backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py index fc3503ef..af078d7d 100644 --- a/backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py +++ b/backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py @@ -50,6 +50,9 @@ def test_multi_aoi_calibration_evidence_portfolio_assembles_existing_evidence(tm "analysis_run_id": "analysis-geel", "job_id": "job-geel", "detection_count": 5, + "model_asset_id": "geointel-building-yolov8s-smoke-pt", + "tile_size": 640, + "tile_overlap": 64, "quality_score": 0.42, "precision": 0.7, "recall": 0.3, @@ -72,6 +75,9 @@ def test_multi_aoi_calibration_evidence_portfolio_assembles_existing_evidence(tm "analysis_run_id": "analysis-mol", "job_id": "job-mol", "detection_count": 3, + "model_asset_id": "geointel-building-yolov8s-smoke-pt", + "tile_size": 640, + "tile_overlap": 64, "quality_score": 0.6, "precision": 1.0, "recall": 0.43, @@ -173,6 +179,11 @@ JSON assert portfolio["total_evidence_feature_count"] == 2 assert {sample["sample_slug"] for sample in portfolio["samples"]} == {"geel", "mol"} assert portfolio["best_sample_by_score"]["sample_slug"] == "mol" + assert portfolio["best_sample_by_score"]["best_run_by_score"]["model_asset_id"] == ( + "geointel-building-yolov8s-smoke-pt" + ) + assert portfolio["best_sample_by_score"]["best_run_by_score"]["tile_size"] == 640 + assert portfolio["best_sample_by_score"]["best_run_by_score"]["tile_overlap"] == 64 assert "Geel center" in markdown assert "Mol edge" in markdown assert "calibration_evidence_review.html" in markdown diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 45a54959..732eacfa 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,57 @@ +## Sprint 141 Expanded positive-AOI matrix and portfolio metadata hardening (2026-07-08) + +Changed: +- Ran a fresh Tower multi-sample quality matrix for additional positive AOIs `balen`, `herentals` and `westerlo` using: + - `geointel-building-yolov8n-expanded160e50-pt` + - `geointel-building-yolov8n-hardneg160r8e40-pt` + - tile size `640`, overlap `64`, thresholds `0.15` and `0.05`. +- Assembled a broader positive-AOI evidence portfolio across 7 AOIs: + - Geel, Mol, Turnhout and Retie from `expanded160e50-live`. + - Balen, Herentals and Westerlo from the fresh `balen-herentals-westerlo-live-20260708` run. +- Hardened `scripts/export_detection_calibration_evidence.sh` so evidence bundles preserve calibration provenance fields in run summaries and GeoJSON properties: + - `model_asset_id` + - `model_request` + - `tile_size` + - `tile_overlap` +- Extended `backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py` to assert portfolio best-run model/tile provenance is retained. + +Tested: +- Red step: `python -m pytest backend\tests\test_sprint139_multi_aoi_calibration_evidence_portfolio.py -q` failed with `KeyError: 'model_asset_id'` while bundle summaries dropped model provenance. +- `python -m pytest backend\tests\test_sprint139_multi_aoi_calibration_evidence_portfolio.py -q` (`1 passed`) +- `python -m pytest backend\tests\test_sprint139_multi_aoi_calibration_evidence_portfolio.py backend\tests\test_sprint138_calibration_evidence_bundle_smoke.py backend\tests\test_sprint137_browser_calibration_summary_evidence_script.py backend\tests\test_sprint125_detection_calibration_evidence_bundle.py -q` (`4 passed`) +- `bash -n scripts/export_detection_calibration_evidence.sh` +- `bash -n scripts/assemble_detection_calibration_evidence_portfolio.sh` +- `python -m compileall backend/app` +- `bash scripts/run_readiness_check.sh` (`418 passed`; frontend typecheck/build passed; Alembic head `202606120900`; shell syntax gates passed) +- Tower fresh matrix completed for Balen, Herentals and Westerlo. +- Tower positive-AOI portfolio assembly completed for 7 AOIs. +- Tower runtime remained healthy on `0.0.0.0:1202->80/tcp`. + +Evidence: +- Fresh Balen/Herentals/Westerlo output: `/mnt/user/appdata/geointel/artifacts/detection-quality-matrix/multi-sample/balen-herentals-westerlo-live-20260708/multi_sample_quality_summary.json`. +- Expanded 7-AOI portfolio output: `/mnt/user/appdata/geointel/artifacts/detection-calibration-portfolio/positive-aoi-expanded-20260708/output/calibration_evidence_portfolio.json`. +- Expanded positive-AOI portfolio sample count: `7`. +- Expanded positive-AOI evidence features: `12438`. +- Expanded positive-AOI role counts: `false_negative=9200`, `false_positive=2270`, `match_candidate=484`, `match_reference=484`. +- Best fresh positive-AOI result: Westerlo with `geointel-building-yolov8n-expanded160e50-pt`, threshold `0.05`, F1/score `0.3659305993690852`, precision `0.38666666666666666`, recall `0.3473053892215569`. +- Fresh Balen best: expanded160e50 threshold `0.05`, F1/score `0.16091954022988506`, precision `0.16333333333333333`, recall `0.15857605177993528`. +- Fresh Herentals best: expanded160e50 threshold `0.05`, F1/score `0.14093264248704665`, precision `0.22666666666666666`, recall `0.10225563909774436`. +- Hard-negative context from existing matrices: + - expanded160e50 at `0.05`: Kasterlee-bos `76`, Lommel-heide `10`, Postel-bos `1` detections. + - hardneg160r8e40 at `0.05`: Kasterlee-bos `25`, Lommel-heide `0`, Postel-bos `0` detections. + +Open: +- expanded160e50 is stronger on positive AOIs, especially Westerlo, but still produces many false positives and misses many references. +- hardneg160r8e40 is cleaner on hard-negative AOIs but materially weaker on the fresh positive AOIs. +- No current candidate should be promoted blindly as V1 default without a combined positive/hard-negative decision rule. + +Limitations: +- This pass ran additional live inference/QA workflows and produced operator artifacts, but did not change backend APIs, migrations, frontend runtime behavior, model weights, provider fetching or Docker runtime configuration. +- The 7-AOI portfolio initially generated before the metadata fix lacked model/tile provenance in `best_run_by_score`; it should be regenerated after Tower pulls this commit. + +Next recommended pass: +- Add a model promotion decision report that combines positive-AOI F1/recall and hard-negative false-positive pressure into one explicit accept/reject table per model and threshold. + ## Sprint 140 Live multi-AOI calibration portfolio run (2026-07-08) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index af17cfe7..0532dc7f 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -427,5 +427,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add a local browser-summary QA evidence bundle smoke using mocked canonical evidence responses. - [x] Add a multi-AOI calibration evidence portfolio convention for model-review handoff. - [x] Run the first live multi-AOI calibration evidence portfolio on Tower for Geel, Mol and Turnhout. +- [x] Run fresh positive-AOI matrix coverage for Balen, Herentals and Westerlo. +- [x] Preserve model/tile provenance in calibration evidence bundle summaries. - [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout. - [ ] Add negative/background AOIs so the next tile dataset is not all positive tiles. diff --git a/scripts/export_detection_calibration_evidence.sh b/scripts/export_detection_calibration_evidence.sh index 4da07327..a1111045 100644 --- a/scripts/export_detection_calibration_evidence.sh +++ b/scripts/export_detection_calibration_evidence.sh @@ -110,6 +110,10 @@ def normalize_calibration_items(summary): "analysis_run_id": row.get("analysis_run_id"), "job_id": row.get("job_id"), "quality_check_id": quality_check_id, + "model_asset_id": row.get("model_asset_id"), + "model_request": row.get("model_request"), + "tile_size": row.get("tile_size"), + "tile_overlap": row.get("tile_overlap"), "threshold": row.get("threshold"), "detection_count": row.get("detection_count"), "quality_score": row.get("quality_score") or row.get("f1_score"), @@ -190,6 +194,10 @@ def normalize_calibration_items(summary): "analysis_run_id": row.get("analysis_run_id"), "job_id": row.get("job_id"), "quality_check_id": quality_check_id, + "model_asset_id": row.get("model_asset_id"), + "model_request": row.get("model_request"), + "tile_size": row.get("tile_size"), + "tile_overlap": row.get("tile_overlap"), "threshold": row.get("threshold"), "detection_count": row.get("detection_count"), "quality_score": row.get("quality_score") or row.get("f1_score"), @@ -240,6 +248,10 @@ for response_path in sorted(glob.glob(os.path.join(output_dir, "threshold_*_evid properties.update( { "calibration_threshold": threshold, + "calibration_model_asset_id": item.get("model_asset_id"), + "calibration_model_request": item.get("model_request"), + "calibration_tile_size": item.get("tile_size"), + "calibration_tile_overlap": item.get("tile_overlap"), "calibration_quality_score": item.get("quality_score"), "calibration_precision": item.get("precision"), "calibration_recall": item.get("recall"), @@ -257,6 +269,10 @@ for response_path in sorted(glob.glob(os.path.join(output_dir, "threshold_*_evid "project_id": item.get("project_id"), "analysis_run_id": item.get("analysis_run_id"), "quality_check_id": quality_check_id, + "model_asset_id": item.get("model_asset_id"), + "model_request": item.get("model_request"), + "tile_size": item.get("tile_size"), + "tile_overlap": item.get("tile_overlap"), "detection_count": item.get("detection_count"), "quality_score": item.get("quality_score"), "precision": item.get("precision"),