255 lines
8.7 KiB
Python
255 lines
8.7 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1].parent
|
|
|
|
|
|
def test_detection_model_promotion_report_combines_positive_and_background_gates(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script_path = ROOT / "scripts" / "build_detection_model_promotion_report.py"
|
|
assert script_path.exists()
|
|
|
|
positive_path = tmp_path / "positive_portfolio.json"
|
|
positive_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"portfolio_name": "Positive AOI portfolio",
|
|
"samples": [
|
|
{
|
|
"sample_slug": "geel",
|
|
"runs": [
|
|
{
|
|
"model_asset_id": "candidate-clean",
|
|
"tile_size": 640,
|
|
"tile_overlap": 64,
|
|
"threshold": 0.25,
|
|
"quality_score": 0.42,
|
|
"precision": 0.7,
|
|
"recall": 0.3,
|
|
"f1_score": 0.42,
|
|
},
|
|
{
|
|
"model_asset_id": "candidate-leaky",
|
|
"tile_size": 640,
|
|
"tile_overlap": 64,
|
|
"threshold": 0.05,
|
|
"quality_score": 0.55,
|
|
"precision": 0.6,
|
|
"recall": 0.52,
|
|
"f1_score": 0.55,
|
|
},
|
|
],
|
|
},
|
|
{
|
|
"sample_slug": "mol",
|
|
"runs": [
|
|
{
|
|
"model_asset_id": "candidate-clean",
|
|
"tile_size": 640,
|
|
"tile_overlap": 64,
|
|
"threshold": 0.25,
|
|
"quality_score": 0.38,
|
|
"precision": 0.64,
|
|
"recall": 0.27,
|
|
"f1_score": 0.38,
|
|
},
|
|
{
|
|
"model_asset_id": "candidate-leaky",
|
|
"tile_size": 640,
|
|
"tile_overlap": 64,
|
|
"threshold": 0.05,
|
|
"quality_score": 0.5,
|
|
"precision": 0.55,
|
|
"recall": 0.46,
|
|
"f1_score": 0.5,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
background_path = tmp_path / "hard_negative_matrix_summary.json"
|
|
background_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"items": [
|
|
{
|
|
"sample_slug": "postel_bos",
|
|
"model_asset_id": "candidate-clean",
|
|
"tile_size": 640,
|
|
"tile_overlap": 64,
|
|
"threshold": 0.25,
|
|
"detection_count": 0,
|
|
},
|
|
{
|
|
"sample_slug": "lommel_heide",
|
|
"model_asset_id": "candidate-clean",
|
|
"tile_size": 640,
|
|
"tile_overlap": 64,
|
|
"threshold": 0.25,
|
|
"detection_count": 0,
|
|
},
|
|
{
|
|
"sample_slug": "postel_bos",
|
|
"model_asset_id": "candidate-leaky",
|
|
"tile_size": 640,
|
|
"tile_overlap": 64,
|
|
"threshold": 0.05,
|
|
"detection_count": 3,
|
|
},
|
|
{
|
|
"sample_slug": "lommel_heide",
|
|
"model_asset_id": "candidate-leaky",
|
|
"tile_size": 640,
|
|
"tile_overlap": 64,
|
|
"threshold": 0.05,
|
|
"detection_count": 1,
|
|
},
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
output_dir = tmp_path / "promotion-report"
|
|
result = subprocess.run(
|
|
[
|
|
"python",
|
|
str(script_path),
|
|
"--positive-portfolio",
|
|
str(positive_path),
|
|
"--hard-negative-summary",
|
|
str(background_path),
|
|
"--output-dir",
|
|
str(output_dir),
|
|
"--min-positive-samples",
|
|
"2",
|
|
"--min-background-samples",
|
|
"2",
|
|
"--min-mean-f1",
|
|
"0.35",
|
|
"--max-background-detections-per-sample",
|
|
"0",
|
|
],
|
|
cwd=ROOT,
|
|
check=True,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
|
|
assert "Detection model promotion report passed" in result.stdout
|
|
report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8"))
|
|
decisions = {
|
|
item["candidate_key"]: item["promotion_status"]
|
|
for item in report["candidate_decisions"]
|
|
}
|
|
assert decisions["candidate-clean|640|64|0.25"] == "promote_candidate"
|
|
assert decisions["candidate-leaky|640|64|0.05"] == "reject"
|
|
|
|
leaky = next(
|
|
item
|
|
for item in report["candidate_decisions"]
|
|
if item["candidate_key"] == "candidate-leaky|640|64|0.05"
|
|
)
|
|
assert "background_false_positive_pressure" in leaky["rejection_reasons"]
|
|
assert leaky["max_background_detections"] == 3
|
|
assert report["recommended_candidate"]["candidate_key"] == "candidate-clean|640|64|0.25"
|
|
|
|
markdown = (output_dir / "detection_model_promotion_report.md").read_text(encoding="utf-8")
|
|
assert "candidate-clean" in markdown
|
|
assert "candidate-leaky" in markdown
|
|
assert "background_false_positive_pressure" in markdown
|
|
|
|
|
|
def test_detection_model_promotion_report_uses_portfolio_and_tile_defaults(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script_path = ROOT / "scripts" / "build_detection_model_promotion_report.py"
|
|
|
|
positive_path = tmp_path / "positive_portfolio.json"
|
|
positive_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"portfolio_name": "Positive AOI portfolio",
|
|
"model_asset_id": "candidate-from-portfolio",
|
|
"samples": [
|
|
{
|
|
"sample_slug": "geel",
|
|
"runs": [
|
|
{
|
|
"model_asset_id": None,
|
|
"tile_size": None,
|
|
"tile_overlap": None,
|
|
"threshold": 0.25,
|
|
"precision": 0.7,
|
|
"recall": 0.42,
|
|
"f1_score": 0.525,
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
background_path = tmp_path / "hard_negative_matrix_summary.json"
|
|
background_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"items": [
|
|
{
|
|
"sample_slug": "postel_bos",
|
|
"model_asset_id": "candidate-from-portfolio",
|
|
"tile_size": 640,
|
|
"tile_overlap": 64,
|
|
"threshold": 0.25,
|
|
"detection_count": 0,
|
|
}
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
output_dir = tmp_path / "promotion-report"
|
|
subprocess.run(
|
|
[
|
|
"python",
|
|
str(script_path),
|
|
"--positive-portfolio",
|
|
str(positive_path),
|
|
"--hard-negative-summary",
|
|
str(background_path),
|
|
"--output-dir",
|
|
str(output_dir),
|
|
"--min-positive-samples",
|
|
"1",
|
|
"--min-background-samples",
|
|
"1",
|
|
"--min-mean-f1",
|
|
"0.35",
|
|
"--max-background-detections-per-sample",
|
|
"0",
|
|
"--default-positive-tile-size",
|
|
"640",
|
|
"--default-positive-tile-overlap",
|
|
"64",
|
|
],
|
|
cwd=ROOT,
|
|
check=True,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
|
|
report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8"))
|
|
assert report["recommended_candidate"]["candidate_key"] == "candidate-from-portfolio|640|64|0.25"
|