from __future__ import annotations import json import subprocess from pathlib import Path ROOT = Path(__file__).resolve().parents[1].parent def test_detection_model_promotion_report_combines_positive_and_background_gates( tmp_path: Path, ) -> None: script_path = ROOT / "scripts" / "build_detection_model_promotion_report.py" assert script_path.exists() positive_path = tmp_path / "positive_portfolio.json" positive_path.write_text( json.dumps( { "portfolio_name": "Positive AOI portfolio", "samples": [ { "sample_slug": "geel", "runs": [ { "model_asset_id": "candidate-clean", "tile_size": 640, "tile_overlap": 64, "threshold": 0.25, "quality_score": 0.42, "precision": 0.7, "recall": 0.3, "f1_score": 0.42, }, { "model_asset_id": "candidate-leaky", "tile_size": 640, "tile_overlap": 64, "threshold": 0.05, "quality_score": 0.55, "precision": 0.6, "recall": 0.52, "f1_score": 0.55, }, ], }, { "sample_slug": "mol", "runs": [ { "model_asset_id": "candidate-clean", "tile_size": 640, "tile_overlap": 64, "threshold": 0.25, "quality_score": 0.38, "precision": 0.64, "recall": 0.27, "f1_score": 0.38, }, { "model_asset_id": "candidate-leaky", "tile_size": 640, "tile_overlap": 64, "threshold": 0.05, "quality_score": 0.5, "precision": 0.55, "recall": 0.46, "f1_score": 0.5, }, ], }, ], } ), encoding="utf-8", ) background_path = tmp_path / "hard_negative_matrix_summary.json" background_path.write_text( json.dumps( { "items": [ { "sample_slug": "postel_bos", "model_asset_id": "candidate-clean", "tile_size": 640, "tile_overlap": 64, "threshold": 0.25, "detection_count": 0, }, { "sample_slug": "lommel_heide", "model_asset_id": "candidate-clean", "tile_size": 640, "tile_overlap": 64, "threshold": 0.25, "detection_count": 0, }, { "sample_slug": "postel_bos", "model_asset_id": "candidate-leaky", "tile_size": 640, "tile_overlap": 64, "threshold": 0.05, "detection_count": 3, }, { "sample_slug": "lommel_heide", "model_asset_id": "candidate-leaky", "tile_size": 640, "tile_overlap": 64, "threshold": 0.05, "detection_count": 1, }, ] } ), encoding="utf-8", ) output_dir = tmp_path / "promotion-report" result = subprocess.run( [ "python", str(script_path), "--positive-portfolio", str(positive_path), "--hard-negative-summary", str(background_path), "--output-dir", str(output_dir), "--min-positive-samples", "2", "--min-background-samples", "2", "--min-mean-f1", "0.35", "--max-background-detections-per-sample", "0", ], cwd=ROOT, check=True, text=True, capture_output=True, ) assert "Detection model promotion report passed" in result.stdout report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8")) decisions = { item["candidate_key"]: item["promotion_status"] for item in report["candidate_decisions"] } assert decisions["candidate-clean|640|64|0.25"] == "promote_candidate" assert decisions["candidate-leaky|640|64|0.05"] == "reject" leaky = next( item for item in report["candidate_decisions"] if item["candidate_key"] == "candidate-leaky|640|64|0.05" ) assert "background_false_positive_pressure" in leaky["rejection_reasons"] assert leaky["max_background_detections"] == 3 assert report["recommended_candidate"]["candidate_key"] == "candidate-clean|640|64|0.25" markdown = (output_dir / "detection_model_promotion_report.md").read_text(encoding="utf-8") assert "candidate-clean" in markdown assert "candidate-leaky" in markdown assert "background_false_positive_pressure" in markdown def test_detection_model_promotion_report_uses_portfolio_and_tile_defaults( tmp_path: Path, ) -> None: script_path = ROOT / "scripts" / "build_detection_model_promotion_report.py" positive_path = tmp_path / "positive_portfolio.json" positive_path.write_text( json.dumps( { "portfolio_name": "Positive AOI portfolio", "model_asset_id": "candidate-from-portfolio", "samples": [ { "sample_slug": "geel", "runs": [ { "model_asset_id": None, "tile_size": None, "tile_overlap": None, "threshold": 0.25, "precision": 0.7, "recall": 0.42, "f1_score": 0.525, } ], } ], } ), encoding="utf-8", ) background_path = tmp_path / "hard_negative_matrix_summary.json" background_path.write_text( json.dumps( { "items": [ { "sample_slug": "postel_bos", "model_asset_id": "candidate-from-portfolio", "tile_size": 640, "tile_overlap": 64, "threshold": 0.25, "detection_count": 0, } ] } ), encoding="utf-8", ) output_dir = tmp_path / "promotion-report" subprocess.run( [ "python", str(script_path), "--positive-portfolio", str(positive_path), "--hard-negative-summary", str(background_path), "--output-dir", str(output_dir), "--min-positive-samples", "1", "--min-background-samples", "1", "--min-mean-f1", "0.35", "--max-background-detections-per-sample", "0", "--default-positive-tile-size", "640", "--default-positive-tile-overlap", "64", ], cwd=ROOT, check=True, text=True, capture_output=True, ) report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8")) assert report["recommended_candidate"]["candidate_key"] == "candidate-from-portfolio|640|64|0.25" def test_detection_model_promotion_report_accepts_multi_sample_quality_summary( tmp_path: Path, ) -> None: script_path = ROOT / "scripts" / "build_detection_model_promotion_report.py" positive_path = tmp_path / "multi_sample_quality_summary.json" positive_path.write_text( json.dumps( { "items": [ { "sample_slug": "geel", "model_asset_id": "candidate-multi", "tile_size": 512, "tile_overlap": 64, "threshold": 0.15, "precision": 0.2, "recall": 0.1, "f1": 0.1333333333, }, { "sample_slug": "retie", "model_asset_id": "candidate-multi", "tile_size": 512, "tile_overlap": 64, "threshold": 0.15, "precision": 0.3, "recall": 0.2, "f1_score": 0.24, }, ] } ), encoding="utf-8", ) background_path = tmp_path / "hard_negative_matrix_summary.json" background_path.write_text( json.dumps( { "items": [ { "sample_slug": "postel_bos", "model_asset_id": "candidate-multi", "tile_size": 512, "tile_overlap": 64, "threshold": 0.15, "detection_count": 0, }, { "sample_slug": "lommel_heide", "model_asset_id": "candidate-multi", "tile_size": 512, "tile_overlap": 64, "threshold": 0.15, "detection_count": 0, }, ] } ), encoding="utf-8", ) output_dir = tmp_path / "promotion-report" subprocess.run( [ "python", str(script_path), "--positive-portfolio", str(positive_path), "--hard-negative-summary", str(background_path), "--output-dir", str(output_dir), "--min-positive-samples", "2", "--min-background-samples", "2", "--min-mean-f1", "0.1", "--max-background-detections-per-sample", "0", ], cwd=ROOT, check=True, text=True, capture_output=True, ) report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8")) decision = report["candidate_decisions"][0] assert decision["candidate_key"] == "candidate-multi|512|64|0.15" assert decision["positive_sample_count"] == 2 assert decision["mean_f1"] > 0.18 assert decision["promotion_status"] == "promote_candidate"