Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add coverage-aware Mol benchmark
GeoIntel CI / docs-smoke (push) Canceled after 0s
Details
GeoIntel CI / contract-smoke (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
Codex
2026-07-14 12:27:48 +02:00
parent 4f0d2891dd
commit 9f61037794
12 changed files with 682 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run_detection_quality_matrix.sh
+34 -3
View File
@@ -286,6 +286,13 @@ metrics = {
if metric.get("metric_key")
}
findings = quality_check.get("findings_json") or {}
coverage = findings.get("coverage") or {}
diagnostics = findings.get("box_to_footprint_diagnostics") or {}
reference_raw_count = coverage.get("reference_raw_count")
reference_evaluated_count = coverage.get("reference_evaluated_count")
reference_coverage_ratio = None
if isinstance(reference_raw_count, int) and reference_raw_count > 0 and isinstance(reference_evaluated_count, int):
reference_coverage_ratio = reference_evaluated_count / reference_raw_count
summary = {
"model_request": model_request,
"model_asset_id": selected_model_asset_id,
@@ -312,6 +319,27 @@ summary = {
"matches": findings.get("matches"),
"false_positives": findings.get("false_positives"),
"false_negatives": findings.get("false_negatives"),
"coverage_applied": coverage.get("applied", False),
"coverage_mode": coverage.get("mode"),
"coverage_tile_count": coverage.get("tile_count", 0),
"coverage_source_crs_values": coverage.get("source_crs_values") or [],
"candidate_raw_count": coverage.get("candidate_raw_count"),
"candidate_evaluated_count": coverage.get("candidate_evaluated_count"),
"candidate_excluded_outside_count": coverage.get("candidate_excluded_outside_count"),
"candidate_clipped_boundary_count": coverage.get("candidate_clipped_boundary_count"),
"reference_raw_count": reference_raw_count,
"reference_evaluated_count": reference_evaluated_count,
"reference_excluded_outside_count": coverage.get("reference_excluded_outside_count"),
"reference_clipped_boundary_count": coverage.get("reference_clipped_boundary_count"),
"reference_coverage_ratio": reference_coverage_ratio,
"diagnostic_only": diagnostics.get("diagnostic_only"),
"diagnostic_method": diagnostics.get("diagnostic_method"),
"strict_matches": diagnostics.get("strict_matches"),
"envelope_matches": diagnostics.get("envelope_matches"),
"possible_box_to_footprint_mismatch_count": diagnostics.get("possible_box_to_footprint_mismatch_count"),
"envelope_precision": diagnostics.get("envelope_precision"),
"envelope_recall": diagnostics.get("envelope_recall"),
"envelope_f1_score": diagnostics.get("envelope_f1_score"),
"export_id": export_id,
"run_log": run_log,
}
@@ -319,7 +347,8 @@ with open(output_path, "w", encoding="utf-8") as handle:
json.dump(summary, handle, indent=2, sort_keys=True)
print(
"model={model} tile={tile} overlap={overlap} threshold={threshold} detections={detections} raw={raw} suppressed={suppressed} "
"score={score} precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format(
"score={score} precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn} "
"coverage={coverage} diagnostic_gap={diagnostic_gap}".format(
model=summary["model_asset_id"],
tile=summary["tile_size"],
overlap=summary["tile_overlap"],
@@ -334,6 +363,8 @@ print(
matches=summary["matches"],
fp=summary["false_positives"],
fn=summary["false_negatives"],
coverage=summary["reference_coverage_ratio"],
diagnostic_gap=summary["possible_box_to_footprint_mismatch_count"],
)
)
PY
@@ -381,10 +412,10 @@ with open(summary_path, "w", encoding="utf-8") as handle:
print("")
print("Detection quality matrix summary")
print("model\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
print("model\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn\tcoverage\tdiagnostic_gap")
for item in items:
print(
"{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
"{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}\t{reference_coverage_ratio}\t{possible_box_to_footprint_mismatch_count}".format(
**item
)
)