Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

let an operator's adjudication reach the score

Browse Source 
The review vocabulary already separates a model error from a reference gap,
because the product's position is that official footprints are not
automatically perfect ground truth. Those verdicts were only counted. An
operator who inspected forty false positives and established that twelve are
buildings the reference simply lacks still saw a precision counting all forty
against the model — a number they had personally disproved, on the panel where
they disproved it.

Applying the verdicts gives an adjudicated score reported next to the raw one,
so nothing is quietly improved. Not being able to judge is not evidence in the
model's favour, so uncertain and obscured verdicts keep counting, as does a
decision from a later release that this runtime does not recognise.

Because part of the evidence is usually still unreviewed, the honest form is an
interval rather than a single corrected number: pessimistic assumes every
unreviewed finding is a model error, optimistic assumes none is, and the
headline equals the pessimistic reading so a partly reviewed check never
presents as a settled one.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jens
2026-08-22 20:43:22 +02:00
co-authored by Claude Opus 5
parent f6eced1b94
commit ff4a15aa74
7 changed files with 501 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/schemas/detection_review.py
+4
View File
@@ -53,6 +53,10 @@ class DetectionReviewSummary(BaseModel):
false_positive_total: int
false_negative_total: int
decision_counts: dict[str, int]
# The score with the operator's verdicts applied, next to the raw one. A
# finding adjudicated as a reference gap is not the model's error, and an
# interval covers what the unreviewed remainder could still turn out to be.
reviewed_metrics: dict | None = None
class DetectionReviewList(BaseModel):
backend/app/services/detection_review_service.py
+44 -2
View File
@@ -7,6 +7,7 @@ from uuid import UUID
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.services.reviewed_metrics_service import ReviewedMetricsService
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
from app.schemas.detection_review import (
DetectionReviewList,
@@ -80,7 +81,11 @@ class DetectionReviewService:
return {(row.evidence_role, row.evidence_feature_id): row for row in rows}
@staticmethod
def _summary(evidence: list[dict[str, str]], reviews: dict[tuple[str, str], DetectionReview]) -> DetectionReviewSummary:
def _summary(
evidence: list[dict[str, str]],
reviews: dict[tuple[str, str], DetectionReview],
quality_check: QualityCheck | None = None,
) -> DetectionReviewSummary:
evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}
decisions = Counter(
reviews[key].decision if key in reviews else "unreviewed"
@@ -96,6 +101,43 @@ class DetectionReviewService:
false_positive_total=false_positive_total,
false_negative_total=false_negative_total,
decision_counts=dict(sorted(decisions.items())),
reviewed_metrics=DetectionReviewService._reviewed_metrics(evidence_keys, reviews, quality_check),
)
@staticmethod
def _reviewed_metrics(
evidence_keys: set[tuple[str, str]],
reviews: dict[tuple[str, str], DetectionReview],
quality_check: QualityCheck | None,
) -> dict | None:
"""The score with the operator's verdicts applied.
Without this the panel shows a precision the operator has already
disproved: a false positive adjudicated as a reference gap is not the
model's error, and the raw number keeps counting it as one.
"""
if quality_check is None:
return None
findings = quality_check.findings_json if isinstance(quality_check.findings_json, dict) else {}
matches = findings.get("matches")
false_positives = findings.get("false_positives")
false_negatives = findings.get("false_negatives")
if not all(isinstance(value, int) for value in (matches, false_positives, false_negatives)):
return None
per_role: dict[str, Counter] = {"false_positive": Counter(), "false_negative": Counter()}
for role, feature_id in evidence_keys:
review = reviews.get((role, feature_id))
if review is not None and role in per_role:
per_role[role][review.decision] += 1
return ReviewedMetricsService.adjudicate(
matches=int(matches),
false_positives=int(false_positives),
false_negatives=int(false_negatives),
false_positive_decisions=dict(per_role["false_positive"]),
false_negative_decisions=dict(per_role["false_negative"]),
)
@staticmethod
@@ -180,7 +222,7 @@ class DetectionReviewService:
total=len(filtered),
limit=limit,
offset=offset,
summary=DetectionReviewService._summary(evidence, reviews),
summary=DetectionReviewService._summary(evidence, reviews, quality_check),
)
@staticmethod
backend/app/services/reviewed_metrics_service.py
+112
View File
@@ -0,0 +1,112 @@
"""Detection metrics after an operator has adjudicated the evidence.
The review vocabulary already separates a model error from a reference gap,
because the product's position is that official footprints are not
automatically perfect ground truth. Until now those verdicts were only counted:
an operator who established that twelve of forty false positives are buildings
the reference simply lacks still saw a precision counting all forty against the
model — a number they had personally disproved.
Applying the verdicts gives an adjudicated score. Because part of the evidence
is usually still unreviewed, the honest form is an interval: pessimistic
assumes every unreviewed item is a model error, optimistic assumes none is. The
raw score stays reported alongside, so nothing is quietly improved.
"""
from __future__ import annotations
from typing import Any
# A verdict that the finding was not the model's fault. The detection (or the
# missing detection) was right; the reference or the matching rule was not.
EXONERATING_DECISIONS = frozenset({"reference_gap_or_change", "qa_alignment_mismatch"})
# Verdicts that confirm the finding, and verdicts that reach no conclusion.
# Both keep counting: being unable to judge is not evidence in the model's
# favour, and treating it as such is how a score drifts upward unearned.
CONFIRMING_DECISIONS = frozenset(
{"confirmed_model_false_positive", "confirmed_model_false_negative"}
)
INCONCLUSIVE_DECISIONS = frozenset({"uncertain", "imagery_obscured_or_uncertain"})
def _score(matches: int, false_positives: int, false_negatives: int) -> dict[str, Any]:
precision = matches / (matches + false_positives) if matches + false_positives > 0 else None
recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None
f1_score = None
if precision is not None and recall is not None:
f1_score = (2 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0
return {
"matches": matches,
"false_positives": false_positives,
"false_negatives": false_negatives,
"precision": precision,
"recall": recall,
"f1_score": f1_score,
}
class ReviewedMetricsService:
@staticmethod
def _adjudicate_role(total: int, decisions: dict[str, int]) -> tuple[int, int, int]:
"""Split a finding count into exonerated, confirmed and unreviewed.
A decision the runtime does not recognise — one from a later release —
counts as no judgement rather than as an exoneration.
"""
exonerated = sum(count for name, count in decisions.items() if name in EXONERATING_DECISIONS)
judged = sum(
count
for name, count in decisions.items()
if name in EXONERATING_DECISIONS | CONFIRMING_DECISIONS | INCONCLUSIVE_DECISIONS
)
exonerated = min(exonerated, total)
judged = min(judged, total)
return exonerated, judged - exonerated, max(0, total - judged)
@staticmethod
def adjudicate(
*,
matches: int,
false_positives: int,
false_negatives: int,
false_positive_decisions: dict[str, int],
false_negative_decisions: dict[str, int],
) -> dict[str, Any]:
"""Apply operator verdicts to a quality check's counts."""
fp_exonerated, _fp_confirmed, fp_unreviewed = ReviewedMetricsService._adjudicate_role(
false_positives, false_positive_decisions
)
fn_exonerated, _fn_confirmed, fn_unreviewed = ReviewedMetricsService._adjudicate_role(
false_negatives, false_negative_decisions
)
adjudicated_fp = false_positives - fp_exonerated
adjudicated_fn = false_negatives - fn_exonerated
# The interval covers what the unreviewed remainder could still turn
# out to be, so a partly reviewed check never reads as a settled one.
pessimistic = _score(matches, adjudicated_fp, adjudicated_fn)
optimistic = _score(matches, adjudicated_fp - fp_unreviewed, adjudicated_fn - fn_unreviewed)
return {
"raw": _score(matches, false_positives, false_negatives),
"adjudicated": pessimistic,
"pessimistic": pessimistic,
"optimistic": optimistic,
"review_complete": fp_unreviewed == 0 and fn_unreviewed == 0,
"false_positive_breakdown": {
"total": false_positives,
"exonerated": fp_exonerated,
"confirmed_or_inconclusive": false_positives - fp_exonerated - fp_unreviewed,
"unreviewed": fp_unreviewed,
},
"false_negative_breakdown": {
"total": false_negatives,
"exonerated": fn_exonerated,
"confirmed_or_inconclusive": false_negatives - fn_exonerated - fn_unreviewed,
"unreviewed": fn_unreviewed,
},
}