Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add detection model promotion report
GeoIntel CI / docs-smoke (push) Has been cancelled
Details
GeoIntel CI / contract-smoke (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Codex
2026-07-08 16:41:11 +02:00
parent d03a86a8ce
commit c2fba67569
7 changed files with 498 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_sprint143_detection_model_promotion_report.py
+169
View File
@@ -0,0 +1,169 @@
from __future__ import annotations
import json
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1].parent
def test_detection_model_promotion_report_combines_positive_and_background_gates(
tmp_path: Path,
) -> None:
script_path = ROOT / "scripts" / "build_detection_model_promotion_report.py"
assert script_path.exists()
positive_path = tmp_path / "positive_portfolio.json"
positive_path.write_text(
json.dumps(
{
"portfolio_name": "Positive AOI portfolio",
"samples": [
{
"sample_slug": "geel",
"runs": [
{
"model_asset_id": "candidate-clean",
"tile_size": 640,
"tile_overlap": 64,
"threshold": 0.25,
"quality_score": 0.42,
"precision": 0.7,
"recall": 0.3,
"f1_score": 0.42,
},
{
"model_asset_id": "candidate-leaky",
"tile_size": 640,
"tile_overlap": 64,
"threshold": 0.05,
"quality_score": 0.55,
"precision": 0.6,
"recall": 0.52,
"f1_score": 0.55,
},
],
},
{
"sample_slug": "mol",
"runs": [
{
"model_asset_id": "candidate-clean",
"tile_size": 640,
"tile_overlap": 64,
"threshold": 0.25,
"quality_score": 0.38,
"precision": 0.64,
"recall": 0.27,
"f1_score": 0.38,
},
{
"model_asset_id": "candidate-leaky",
"tile_size": 640,
"tile_overlap": 64,
"threshold": 0.05,
"quality_score": 0.5,
"precision": 0.55,
"recall": 0.46,
"f1_score": 0.5,
},
],
},
],
}
),
encoding="utf-8",
)
background_path = tmp_path / "hard_negative_matrix_summary.json"
background_path.write_text(
json.dumps(
{
"items": [
{
"sample_slug": "postel_bos",
"model_asset_id": "candidate-clean",
"tile_size": 640,
"tile_overlap": 64,
"threshold": 0.25,
"detection_count": 0,
},
{
"sample_slug": "lommel_heide",
"model_asset_id": "candidate-clean",
"tile_size": 640,
"tile_overlap": 64,
"threshold": 0.25,
"detection_count": 0,
},
{
"sample_slug": "postel_bos",
"model_asset_id": "candidate-leaky",
"tile_size": 640,
"tile_overlap": 64,
"threshold": 0.05,
"detection_count": 3,
},
{
"sample_slug": "lommel_heide",
"model_asset_id": "candidate-leaky",
"tile_size": 640,
"tile_overlap": 64,
"threshold": 0.05,
"detection_count": 1,
},
]
}
),
encoding="utf-8",
)
output_dir = tmp_path / "promotion-report"
result = subprocess.run(
[
"python",
str(script_path),
"--positive-portfolio",
str(positive_path),
"--hard-negative-summary",
str(background_path),
"--output-dir",
str(output_dir),
"--min-positive-samples",
"2",
"--min-background-samples",
"2",
"--min-mean-f1",
"0.35",
"--max-background-detections-per-sample",
"0",
],
cwd=ROOT,
check=True,
text=True,
capture_output=True,
)
assert "Detection model promotion report passed" in result.stdout
report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8"))
decisions = {
item["candidate_key"]: item["promotion_status"]
for item in report["candidate_decisions"]
}
assert decisions["candidate-clean|640|64|0.25"] == "promote_candidate"
assert decisions["candidate-leaky|640|64|0.05"] == "reject"
leaky = next(
item
for item in report["candidate_decisions"]
if item["candidate_key"] == "candidate-leaky|640|64|0.05"
)
assert "background_false_positive_pressure" in leaky["rejection_reasons"]
assert leaky["max_background_detections"] == 3
assert report["recommended_candidate"]["candidate_key"] == "candidate-clean|640|64|0.25"
markdown = (output_dir / "detection_model_promotion_report.md").read_text(encoding="utf-8")
assert "candidate-clean" in markdown
assert "candidate-leaky" in markdown
assert "background_false_positive_pressure" in markdown