426 lines
17 KiB
Python
426 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Build an operator-only detection model promotion report.
|
|
|
|
The report combines persisted positive-AOI QA portfolio evidence with
|
|
hard-negative/background detection-count summaries. It does not run inference,
|
|
mutate application state, download models or change active model configuration.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from statistics import mean
|
|
from typing import Any
|
|
|
|
|
|
PURE_EMPTY_CATEGORY = "pure_empty_negative"
|
|
SPARSE_CONTEXT_CATEGORY = "sparse_building_context"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CandidateKey:
|
|
model_asset_id: str
|
|
tile_size: int
|
|
tile_overlap: int
|
|
threshold: float
|
|
|
|
@property
|
|
def label(self) -> str:
|
|
threshold = ("%f" % self.threshold).rstrip("0").rstrip(".")
|
|
return f"{self.model_asset_id}|{self.tile_size}|{self.tile_overlap}|{threshold}"
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Build a detection model promotion report from positive and hard-negative evidence."
|
|
)
|
|
parser.add_argument("--positive-portfolio", required=True, help="Path to calibration_evidence_portfolio.json")
|
|
parser.add_argument(
|
|
"--hard-negative-summary",
|
|
action="append",
|
|
default=[],
|
|
help="Path to hard_negative_matrix_summary.json. May be supplied multiple times.",
|
|
)
|
|
parser.add_argument(
|
|
"--background-split-summary",
|
|
action="append",
|
|
default=[],
|
|
help=(
|
|
"Path to background_corpus_split_summary.json. The pure-empty source summary is used as the "
|
|
"strict default gate; sparse-context evidence is kept as review-only context."
|
|
),
|
|
)
|
|
parser.add_argument("--output-dir", required=True, help="Directory for JSON and Markdown report output.")
|
|
parser.add_argument("--min-positive-samples", type=int, default=3)
|
|
parser.add_argument("--min-background-samples", type=int, default=3)
|
|
parser.add_argument("--min-mean-f1", type=float, default=0.25)
|
|
parser.add_argument("--max-background-detections-per-sample", type=int, default=0)
|
|
parser.add_argument(
|
|
"--default-positive-tile-size",
|
|
type=int,
|
|
default=None,
|
|
help="Tile size to use for positive portfolio runs that do not record tile_size.",
|
|
)
|
|
parser.add_argument(
|
|
"--default-positive-tile-overlap",
|
|
type=int,
|
|
default=None,
|
|
help="Tile overlap to use for positive portfolio runs that do not record tile_overlap.",
|
|
)
|
|
args = parser.parse_args()
|
|
if not args.hard_negative_summary and not args.background_split_summary:
|
|
parser.error("at least one --hard-negative-summary or --background-split-summary is required")
|
|
return args
|
|
|
|
|
|
def read_json(path: Path) -> dict[str, Any]:
|
|
if not path.is_file():
|
|
raise SystemExit(f"JSON input is not readable: {path}")
|
|
return json.loads(path.read_text(encoding="utf-8-sig"))
|
|
|
|
|
|
def as_candidate_key(
|
|
item: dict[str, Any],
|
|
*,
|
|
fallback_model_asset_id: str | None = None,
|
|
fallback_tile_size: int | None = None,
|
|
fallback_tile_overlap: int | None = None,
|
|
) -> CandidateKey | None:
|
|
model_asset_id = item.get("model_asset_id") or item.get("model_request") or fallback_model_asset_id
|
|
if not model_asset_id:
|
|
return None
|
|
try:
|
|
return CandidateKey(
|
|
model_asset_id=str(model_asset_id),
|
|
tile_size=int(item.get("tile_size") if item.get("tile_size") is not None else fallback_tile_size),
|
|
tile_overlap=int(
|
|
item.get("tile_overlap") if item.get("tile_overlap") is not None else fallback_tile_overlap
|
|
),
|
|
threshold=float(item.get("threshold")),
|
|
)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def numeric(item: dict[str, Any], key: str) -> float | None:
|
|
value = item.get(key)
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def collect_positive_runs(
|
|
portfolio: dict[str, Any],
|
|
*,
|
|
default_tile_size: int | None = None,
|
|
default_tile_overlap: int | None = None,
|
|
) -> dict[CandidateKey, list[dict[str, Any]]]:
|
|
best_by_candidate_sample: dict[tuple[CandidateKey, str], dict[str, Any]] = {}
|
|
portfolio_model_asset_id = portfolio.get("model_asset_id")
|
|
|
|
positive_runs: list[tuple[str, dict[str, Any]]] = []
|
|
for sample in portfolio.get("samples") or []:
|
|
sample_slug = str(sample.get("sample_slug") or "unknown")
|
|
for run in sample.get("runs") or []:
|
|
positive_runs.append((sample_slug, run))
|
|
for item in portfolio.get("items") or []:
|
|
sample_slug = str(item.get("sample_slug") or "unknown")
|
|
positive_runs.append((sample_slug, item))
|
|
|
|
for sample_slug, run in positive_runs:
|
|
key = as_candidate_key(
|
|
run,
|
|
fallback_model_asset_id=str(portfolio_model_asset_id) if portfolio_model_asset_id else None,
|
|
fallback_tile_size=default_tile_size,
|
|
fallback_tile_overlap=default_tile_overlap,
|
|
)
|
|
f1 = numeric(run, "f1_score")
|
|
if f1 is None:
|
|
f1 = numeric(run, "f1")
|
|
if key is None or f1 is None:
|
|
continue
|
|
enriched = dict(run)
|
|
enriched["f1_score"] = f1
|
|
enriched["sample_slug"] = sample_slug
|
|
existing = best_by_candidate_sample.get((key, sample_slug))
|
|
existing_f1 = numeric(existing or {}, "f1_score")
|
|
if existing is None or existing_f1 is None or f1 > existing_f1:
|
|
best_by_candidate_sample[(key, sample_slug)] = enriched
|
|
|
|
grouped: dict[CandidateKey, list[dict[str, Any]]] = defaultdict(list)
|
|
for (key, _sample_slug), run in best_by_candidate_sample.items():
|
|
grouped[key].append(run)
|
|
return grouped
|
|
|
|
|
|
def collect_background_runs(summary_paths: list[Path]) -> dict[CandidateKey, list[dict[str, Any]]]:
|
|
best_by_candidate_sample: dict[tuple[CandidateKey, str], dict[str, Any]] = {}
|
|
for path in summary_paths:
|
|
payload = read_json(path)
|
|
for item in payload.get("items") or []:
|
|
key = as_candidate_key(item)
|
|
sample_slug = str(item.get("sample_slug") or "unknown")
|
|
if key is None:
|
|
continue
|
|
enriched = dict(item)
|
|
enriched["source_summary_path"] = str(path)
|
|
detection_count = int(enriched.get("detection_count") or 0)
|
|
existing = best_by_candidate_sample.get((key, sample_slug))
|
|
existing_count = int((existing or {}).get("detection_count") or 0)
|
|
if existing is None or detection_count > existing_count:
|
|
best_by_candidate_sample[(key, sample_slug)] = enriched
|
|
|
|
grouped: dict[CandidateKey, list[dict[str, Any]]] = defaultdict(list)
|
|
for (key, _sample_slug), item in best_by_candidate_sample.items():
|
|
grouped[key].append(item)
|
|
return grouped
|
|
|
|
|
|
def resolve_split_source_path(source_path: Any, split_summary_path: Path) -> Path:
|
|
path = Path(str(source_path))
|
|
if path.is_file():
|
|
return path
|
|
if not path.is_absolute():
|
|
candidate = split_summary_path.parent / path
|
|
if candidate.is_file():
|
|
return candidate
|
|
return path
|
|
|
|
|
|
def collect_split_background_inputs(
|
|
split_summary_paths: list[Path],
|
|
) -> tuple[list[Path], list[dict[str, Any]]]:
|
|
strict_background_paths: list[Path] = []
|
|
context_reviews: list[dict[str, Any]] = []
|
|
|
|
for split_summary_path in split_summary_paths:
|
|
payload = read_json(split_summary_path)
|
|
source_summaries = payload.get("source_summaries") or {}
|
|
pure_empty_source = source_summaries.get(PURE_EMPTY_CATEGORY)
|
|
if not pure_empty_source:
|
|
raise SystemExit(f"Split summary has no {PURE_EMPTY_CATEGORY} source: {split_summary_path}")
|
|
|
|
strict_block = payload.get("strict_default_gate") or {}
|
|
strict_category = strict_block.get("category")
|
|
if strict_category and strict_category != PURE_EMPTY_CATEGORY:
|
|
raise SystemExit(
|
|
f"Split summary strict gate must be {PURE_EMPTY_CATEGORY}; found {strict_category}: {split_summary_path}"
|
|
)
|
|
|
|
strict_background_paths.append(resolve_split_source_path(pure_empty_source, split_summary_path))
|
|
|
|
context_block = payload.get("context_review") or {}
|
|
sparse_context_source = source_summaries.get(SPARSE_CONTEXT_CATEGORY)
|
|
if context_block or sparse_context_source:
|
|
context_entry = dict(context_block)
|
|
context_entry["source_split_summary_path"] = str(split_summary_path)
|
|
if sparse_context_source:
|
|
context_entry["source_summary_path"] = str(
|
|
resolve_split_source_path(sparse_context_source, split_summary_path)
|
|
)
|
|
context_reviews.append(context_entry)
|
|
|
|
return strict_background_paths, context_reviews
|
|
|
|
|
|
def unique_paths(paths: list[Path]) -> list[Path]:
|
|
seen: set[str] = set()
|
|
unique: list[Path] = []
|
|
for path in paths:
|
|
marker = str(path)
|
|
if marker in seen:
|
|
continue
|
|
seen.add(marker)
|
|
unique.append(path)
|
|
return unique
|
|
|
|
|
|
def average_metric(runs: list[dict[str, Any]], metric: str) -> float | None:
|
|
values = [numeric(run, metric) for run in runs]
|
|
values = [value for value in values if value is not None]
|
|
return mean(values) if values else None
|
|
|
|
|
|
def build_decisions(args: argparse.Namespace) -> dict[str, Any]:
|
|
portfolio_path = Path(args.positive_portfolio)
|
|
positive_portfolio = read_json(portfolio_path)
|
|
direct_background_paths = [Path(path) for path in args.hard_negative_summary]
|
|
split_summary_paths = [Path(path) for path in args.background_split_summary]
|
|
split_background_paths, context_reviews = collect_split_background_inputs(split_summary_paths)
|
|
background_paths = unique_paths([*direct_background_paths, *split_background_paths])
|
|
positive = collect_positive_runs(
|
|
positive_portfolio,
|
|
default_tile_size=args.default_positive_tile_size,
|
|
default_tile_overlap=args.default_positive_tile_overlap,
|
|
)
|
|
background = collect_background_runs(background_paths)
|
|
|
|
decisions = []
|
|
for key in sorted(set(positive) | set(background), key=lambda item: item.label):
|
|
positive_runs = positive.get(key, [])
|
|
background_runs = background.get(key, [])
|
|
f1_values = [numeric(run, "f1_score") for run in positive_runs]
|
|
f1_values = [value for value in f1_values if value is not None]
|
|
background_counts = [int(run.get("detection_count") or 0) for run in background_runs]
|
|
|
|
mean_f1 = mean(f1_values) if f1_values else None
|
|
min_f1 = min(f1_values) if f1_values else None
|
|
total_background = sum(background_counts)
|
|
max_background = max(background_counts, default=None)
|
|
|
|
rejection_reasons: list[str] = []
|
|
if len(positive_runs) < args.min_positive_samples:
|
|
rejection_reasons.append("insufficient_positive_samples")
|
|
if len(background_runs) < args.min_background_samples:
|
|
rejection_reasons.append("insufficient_background_samples")
|
|
if mean_f1 is None or mean_f1 < args.min_mean_f1:
|
|
rejection_reasons.append("positive_mean_f1_below_gate")
|
|
if max_background is None or max_background > args.max_background_detections_per_sample:
|
|
rejection_reasons.append("background_false_positive_pressure")
|
|
|
|
decisions.append(
|
|
{
|
|
"candidate_key": key.label,
|
|
"model_asset_id": key.model_asset_id,
|
|
"tile_size": key.tile_size,
|
|
"tile_overlap": key.tile_overlap,
|
|
"threshold": key.threshold,
|
|
"promotion_status": "reject" if rejection_reasons else "promote_candidate",
|
|
"rejection_reasons": rejection_reasons,
|
|
"positive_sample_count": len(positive_runs),
|
|
"background_sample_count": len(background_runs),
|
|
"mean_f1": mean_f1,
|
|
"min_f1": min_f1,
|
|
"mean_precision": average_metric(positive_runs, "precision"),
|
|
"mean_recall": average_metric(positive_runs, "recall"),
|
|
"total_background_detections": total_background,
|
|
"max_background_detections": max_background,
|
|
"positive_samples": sorted(str(run.get("sample_slug")) for run in positive_runs),
|
|
"background_samples": sorted(
|
|
{
|
|
str(run.get("sample_slug")): int(run.get("detection_count") or 0)
|
|
for run in background_runs
|
|
}.items()
|
|
),
|
|
}
|
|
)
|
|
|
|
promoted = [item for item in decisions if item["promotion_status"] == "promote_candidate"]
|
|
recommended = max(
|
|
promoted,
|
|
key=lambda item: (
|
|
item["mean_f1"] if item["mean_f1"] is not None else float("-inf"),
|
|
-(item["total_background_detections"] or 0),
|
|
),
|
|
default=None,
|
|
)
|
|
|
|
return {
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"positive_portfolio_path": str(portfolio_path),
|
|
"hard_negative_summary_paths": [str(path) for path in background_paths],
|
|
"background_split_summary_paths": [str(path) for path in split_summary_paths],
|
|
"background_context_reviews": context_reviews,
|
|
"gates": {
|
|
"min_positive_samples": args.min_positive_samples,
|
|
"min_background_samples": args.min_background_samples,
|
|
"min_mean_f1": args.min_mean_f1,
|
|
"max_background_detections_per_sample": args.max_background_detections_per_sample,
|
|
},
|
|
"candidate_count": len(decisions),
|
|
"recommended_candidate": recommended,
|
|
"candidate_decisions": decisions,
|
|
}
|
|
|
|
|
|
def write_markdown(report: dict[str, Any], path: Path) -> None:
|
|
lines = [
|
|
"# Detection Model Promotion Report",
|
|
"",
|
|
f"- Generated: {report['generated_at']}",
|
|
f"- Positive portfolio: `{report['positive_portfolio_path']}`",
|
|
f"- Hard-negative summaries: {len(report['hard_negative_summary_paths'])}",
|
|
f"- Background split summaries: {len(report['background_split_summary_paths'])}",
|
|
f"- Candidates: {report['candidate_count']}",
|
|
"",
|
|
"## Gates",
|
|
"",
|
|
]
|
|
for key, value in report["gates"].items():
|
|
lines.append(f"- {key}: `{value}`")
|
|
lines.extend(["", "## Recommendation", ""])
|
|
recommended = report.get("recommended_candidate")
|
|
if recommended:
|
|
lines.append(f"- Promote candidate for operator review: `{recommended['candidate_key']}`")
|
|
else:
|
|
lines.append("- No candidate passed all positive and hard-negative gates.")
|
|
if report.get("background_context_reviews"):
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Sparse-context review evidence",
|
|
"",
|
|
(
|
|
f"- Sparse-context review evidence: {len(report['background_context_reviews'])} "
|
|
"(not used as a default-promotion gate)"
|
|
),
|
|
]
|
|
)
|
|
for item in report["background_context_reviews"]:
|
|
lines.append(
|
|
"- "
|
|
f"`{item.get('category', SPARSE_CONTEXT_CATEGORY)}` from "
|
|
f"`{item.get('source_summary_path', 'unknown')}`; "
|
|
f"max detections `{item.get('max_detection_count')}`"
|
|
)
|
|
lines.extend(["", "## Candidate Decisions", ""])
|
|
for item in report["candidate_decisions"]:
|
|
reasons = ", ".join(item["rejection_reasons"]) or "none"
|
|
lines.extend(
|
|
[
|
|
f"### `{item['candidate_key']}`",
|
|
"",
|
|
f"- Status: `{item['promotion_status']}`",
|
|
f"- Rejection reasons: `{reasons}`",
|
|
f"- Positive samples: `{item['positive_sample_count']}`",
|
|
f"- Background samples: `{item['background_sample_count']}`",
|
|
f"- Mean F1: `{item['mean_f1']}`",
|
|
f"- Mean precision: `{item['mean_precision']}`",
|
|
f"- Mean recall: `{item['mean_recall']}`",
|
|
f"- Max background detections: `{item['max_background_detections']}`",
|
|
f"- Total background detections: `{item['total_background_detections']}`",
|
|
"",
|
|
]
|
|
)
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
output_dir = Path(args.output_dir)
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
report = build_decisions(args)
|
|
report_path = output_dir / "detection_model_promotion_report.json"
|
|
markdown_path = output_dir / "detection_model_promotion_report.md"
|
|
report_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
|
|
write_markdown(report, markdown_path)
|
|
|
|
print("Detection model promotion report passed")
|
|
print(f"Report JSON: {report_path}")
|
|
print(f"Report Markdown: {markdown_path}")
|
|
recommended = report.get("recommended_candidate")
|
|
print(f"Recommended candidate: {recommended['candidate_key'] if recommended else 'none'}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|