Preserve calibration portfolio model provenance

2026-07-08 14:25:01 +02:00
parent 8e1e80e22e
commit 51b7d1eec7
5 changed files with 91 additions and 0 deletions
@@ -7,6 +7,14 @@

 # Changelog

+## Sprint 141 Expanded positive-AOI matrix and portfolio metadata hardening (2026-07-08)
+
+- Ran a fresh Tower quality matrix for Balen, Herentals and Westerlo using `geointel-building-yolov8n-expanded160e50-pt` and `geointel-building-yolov8n-hardneg160r8e40-pt`.
+- Assembled an expanded 7-AOI positive evidence portfolio across Geel, Mol, Turnhout, Retie, Balen, Herentals and Westerlo.
+- Hardened calibration evidence exports so model asset id, model request, tile size and tile overlap survive into evidence bundle summaries and GeoJSON properties.
+- Result: expanded160e50 is stronger on positive AOIs, with Westerlo reaching F1 `0.3659305993690852`, but hard-negative matrices still show false-positive pressure tradeoffs that prevent blind default promotion.
+- No backend API, migration, frontend runtime, model weight, provider fetching or Docker runtime change was introduced.
+
 ## Sprint 140 Live multi-AOI calibration portfolio run (2026-07-08)

 - Ran the new multi-AOI calibration evidence portfolio assembler on Tower against existing persisted quality-matrix summaries for Geel, Mol and Turnhout.
@@ -50,6 +50,9 @@ def test_multi_aoi_calibration_evidence_portfolio_assembles_existing_evidence(tm
                        "analysis_run_id": "analysis-geel",
                        "job_id": "job-geel",
                        "detection_count": 5,
+                        "model_asset_id": "geointel-building-yolov8s-smoke-pt",
+                        "tile_size": 640,
+                        "tile_overlap": 64,
                        "quality_score": 0.42,
                        "precision": 0.7,
                        "recall": 0.3,
@@ -72,6 +75,9 @@ def test_multi_aoi_calibration_evidence_portfolio_assembles_existing_evidence(tm
                        "analysis_run_id": "analysis-mol",
                        "job_id": "job-mol",
                        "detection_count": 3,
+                        "model_asset_id": "geointel-building-yolov8s-smoke-pt",
+                        "tile_size": 640,
+                        "tile_overlap": 64,
                        "quality_score": 0.6,
                        "precision": 1.0,
                        "recall": 0.43,
@@ -173,6 +179,11 @@ JSON
    assert portfolio["total_evidence_feature_count"] == 2
    assert {sample["sample_slug"] for sample in portfolio["samples"]} == {"geel", "mol"}
    assert portfolio["best_sample_by_score"]["sample_slug"] == "mol"
+    assert portfolio["best_sample_by_score"]["best_run_by_score"]["model_asset_id"] == (
+        "geointel-building-yolov8s-smoke-pt"
+    )
+    assert portfolio["best_sample_by_score"]["best_run_by_score"]["tile_size"] == 640
+    assert portfolio["best_sample_by_score"]["best_run_by_score"]["tile_overlap"] == 64
    assert "Geel center" in markdown
    assert "Mol edge" in markdown
    assert "calibration_evidence_review.html" in markdown
@@ -1,3 +1,57 @@
+## Sprint 141 Expanded positive-AOI matrix and portfolio metadata hardening (2026-07-08)
+
+Changed:
+- Ran a fresh Tower multi-sample quality matrix for additional positive AOIs `balen`, `herentals` and `westerlo` using:
+  - `geointel-building-yolov8n-expanded160e50-pt`
+  - `geointel-building-yolov8n-hardneg160r8e40-pt`
+  - tile size `640`, overlap `64`, thresholds `0.15` and `0.05`.
+- Assembled a broader positive-AOI evidence portfolio across 7 AOIs:
+  - Geel, Mol, Turnhout and Retie from `expanded160e50-live`.
+  - Balen, Herentals and Westerlo from the fresh `balen-herentals-westerlo-live-20260708` run.
+- Hardened `scripts/export_detection_calibration_evidence.sh` so evidence bundles preserve calibration provenance fields in run summaries and GeoJSON properties:
+  - `model_asset_id`
+  - `model_request`
+  - `tile_size`
+  - `tile_overlap`
+- Extended `backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py` to assert portfolio best-run model/tile provenance is retained.
+
+Tested:
+- Red step: `python -m pytest backend\tests\test_sprint139_multi_aoi_calibration_evidence_portfolio.py -q` failed with `KeyError: 'model_asset_id'` while bundle summaries dropped model provenance.
+- `python -m pytest backend\tests\test_sprint139_multi_aoi_calibration_evidence_portfolio.py -q` (`1 passed`)
+- `python -m pytest backend\tests\test_sprint139_multi_aoi_calibration_evidence_portfolio.py backend\tests\test_sprint138_calibration_evidence_bundle_smoke.py backend\tests\test_sprint137_browser_calibration_summary_evidence_script.py backend\tests\test_sprint125_detection_calibration_evidence_bundle.py -q` (`4 passed`)
+- `bash -n scripts/export_detection_calibration_evidence.sh`
+- `bash -n scripts/assemble_detection_calibration_evidence_portfolio.sh`
+- `python -m compileall backend/app`
+- `bash scripts/run_readiness_check.sh` (`418 passed`; frontend typecheck/build passed; Alembic head `202606120900`; shell syntax gates passed)
+- Tower fresh matrix completed for Balen, Herentals and Westerlo.
+- Tower positive-AOI portfolio assembly completed for 7 AOIs.
+- Tower runtime remained healthy on `0.0.0.0:1202->80/tcp`.
+
+Evidence:
+- Fresh Balen/Herentals/Westerlo output: `/mnt/user/appdata/geointel/artifacts/detection-quality-matrix/multi-sample/balen-herentals-westerlo-live-20260708/multi_sample_quality_summary.json`.
+- Expanded 7-AOI portfolio output: `/mnt/user/appdata/geointel/artifacts/detection-calibration-portfolio/positive-aoi-expanded-20260708/output/calibration_evidence_portfolio.json`.
+- Expanded positive-AOI portfolio sample count: `7`.
+- Expanded positive-AOI evidence features: `12438`.
+- Expanded positive-AOI role counts: `false_negative=9200`, `false_positive=2270`, `match_candidate=484`, `match_reference=484`.
+- Best fresh positive-AOI result: Westerlo with `geointel-building-yolov8n-expanded160e50-pt`, threshold `0.05`, F1/score `0.3659305993690852`, precision `0.38666666666666666`, recall `0.3473053892215569`.
+- Fresh Balen best: expanded160e50 threshold `0.05`, F1/score `0.16091954022988506`, precision `0.16333333333333333`, recall `0.15857605177993528`.
+- Fresh Herentals best: expanded160e50 threshold `0.05`, F1/score `0.14093264248704665`, precision `0.22666666666666666`, recall `0.10225563909774436`.
+- Hard-negative context from existing matrices:
+  - expanded160e50 at `0.05`: Kasterlee-bos `76`, Lommel-heide `10`, Postel-bos `1` detections.
+  - hardneg160r8e40 at `0.05`: Kasterlee-bos `25`, Lommel-heide `0`, Postel-bos `0` detections.
+
+Open:
+- expanded160e50 is stronger on positive AOIs, especially Westerlo, but still produces many false positives and misses many references.
+- hardneg160r8e40 is cleaner on hard-negative AOIs but materially weaker on the fresh positive AOIs.
+- No current candidate should be promoted blindly as V1 default without a combined positive/hard-negative decision rule.
+
+Limitations:
+- This pass ran additional live inference/QA workflows and produced operator artifacts, but did not change backend APIs, migrations, frontend runtime behavior, model weights, provider fetching or Docker runtime configuration.
+- The 7-AOI portfolio initially generated before the metadata fix lacked model/tile provenance in `best_run_by_score`; it should be regenerated after Tower pulls this commit.
+
+Next recommended pass:
+- Add a model promotion decision report that combines positive-AOI F1/recall and hard-negative false-positive pressure into one explicit accept/reject table per model and threshold.
+
 ## Sprint 140 Live multi-AOI calibration portfolio run (2026-07-08)

 Changed:
@@ -427,5 +427,7 @@ This file now starts with the current implementation status. Older preparation/b
 - [x] Add a local browser-summary QA evidence bundle smoke using mocked canonical evidence responses.
 - [x] Add a multi-AOI calibration evidence portfolio convention for model-review handoff.
 - [x] Run the first live multi-AOI calibration evidence portfolio on Tower for Geel, Mol and Turnhout.
+- [x] Run fresh positive-AOI matrix coverage for Balen, Herentals and Westerlo.
+- [x] Preserve model/tile provenance in calibration evidence bundle summaries.
 - [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout.
 - [ ] Add negative/background AOIs so the next tile dataset is not all positive tiles.
@@ -110,6 +110,10 @@ def normalize_calibration_items(summary):
                    "analysis_run_id": row.get("analysis_run_id"),
                    "job_id": row.get("job_id"),
                    "quality_check_id": quality_check_id,
+                    "model_asset_id": row.get("model_asset_id"),
+                    "model_request": row.get("model_request"),
+                    "tile_size": row.get("tile_size"),
+                    "tile_overlap": row.get("tile_overlap"),
                    "threshold": row.get("threshold"),
                    "detection_count": row.get("detection_count"),
                    "quality_score": row.get("quality_score") or row.get("f1_score"),
@@ -190,6 +194,10 @@ def normalize_calibration_items(summary):
                    "analysis_run_id": row.get("analysis_run_id"),
                    "job_id": row.get("job_id"),
                    "quality_check_id": quality_check_id,
+                    "model_asset_id": row.get("model_asset_id"),
+                    "model_request": row.get("model_request"),
+                    "tile_size": row.get("tile_size"),
+                    "tile_overlap": row.get("tile_overlap"),
                    "threshold": row.get("threshold"),
                    "detection_count": row.get("detection_count"),
                    "quality_score": row.get("quality_score") or row.get("f1_score"),
@@ -240,6 +248,10 @@ for response_path in sorted(glob.glob(os.path.join(output_dir, "threshold_*_evid
        properties.update(
            {
                "calibration_threshold": threshold,
+                "calibration_model_asset_id": item.get("model_asset_id"),
+                "calibration_model_request": item.get("model_request"),
+                "calibration_tile_size": item.get("tile_size"),
+                "calibration_tile_overlap": item.get("tile_overlap"),
                "calibration_quality_score": item.get("quality_score"),
                "calibration_precision": item.get("precision"),
                "calibration_recall": item.get("recall"),
@@ -257,6 +269,10 @@ for response_path in sorted(glob.glob(os.path.join(output_dir, "threshold_*_evid
            "project_id": item.get("project_id"),
            "analysis_run_id": item.get("analysis_run_id"),
            "quality_check_id": quality_check_id,
+            "model_asset_id": item.get("model_asset_id"),
+            "model_request": item.get("model_request"),
+            "tile_size": item.get("tile_size"),
+            "tile_overlap": item.get("tile_overlap"),
            "detection_count": item.get("detection_count"),
            "quality_score": item.get("quality_score"),
            "precision": item.get("precision"),