Suppress duplicate YOLO tile detections
GeoIntel CI / docs-smoke (push) Has been cancelled
GeoIntel CI / contract-smoke (push) Has been cancelled

This commit is contained in:
Codex
2026-07-09 10:55:45 +02:00
parent 638534f011
commit 3f5ca4c85e
24 changed files with 266 additions and 39 deletions
+1
View File
@@ -16,6 +16,7 @@ YOLO_DEVICE=cpu
YOLO_IMAGE_SIZE=640
YOLO_MAX_TILES=100
YOLO_MAX_DETECTIONS=1000
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
YOLO_BATCH_SIZE=1
ENABLE_GRB_WFS=false
GRB_WFS_URL=
+9
View File
@@ -7,6 +7,15 @@
# Changelog
## Sprint 149 YOLO duplicate suppression evidence (2026-07-09)
- Added configured-YOLO cross-tile duplicate suppression before `Detection` rows are persisted.
- Added `YOLO_DUPLICATE_IOU_THRESHOLD` with default `0.5`; `0` disables the GeoIntel-side pass for debugging.
- Detection run result metadata now records raw candidate count, suppressed duplicate count and duplicate IoU threshold.
- Calibration and quality matrix scripts now fetch detection run details and include raw/suppressed counts in summaries.
- Updated Docker/Unraid env examples, API/AI/backend docs and detection pipeline notes.
- No model was activated, no detections were faked, and no migration changed.
## Sprint 148 YOLO max-detection cap hardening (2026-07-09)
- Added `YOLO_MAX_DETECTIONS` with default `1000` and forward it to Ultralytics as `max_det`.
+10 -1
View File
@@ -360,6 +360,7 @@ YOLO_DEVICE=cpu
YOLO_IMAGE_SIZE=640
YOLO_MAX_TILES=100
YOLO_MAX_DETECTIONS=1000
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
YOLO_BATCH_SIZE=1
```
@@ -389,6 +390,13 @@ The preflight checks configuration, dependency availability, local model file ex
`1000` because dense building AOIs can exceed the upstream default cap of 300
detections before QA/QC can measure recall honestly.
`YOLO_DUPLICATE_IOU_THRESHOLD` controls GeoIntel-side cross-tile duplicate
suppression after YOLO pixel boxes are converted to EPSG:4326 polygons and
before `Detection` rows are persisted. Candidates are sorted by confidence per
class; lower-confidence same-class candidates with geometry IoU greater than or
equal to the threshold are suppressed. The default is `0.5`; set `0` to disable
this post-processing for debugging.
The same read-only status is available through the API and Detection Lab UI:
```bash
@@ -458,7 +466,8 @@ bash scripts/run_detection_calibration_sweep.sh http://192.168.10.150:1202
The sweep creates one real persisted workflow run per threshold, fetches the
persisted `QualityCheck`/`Metric` rows and writes a `calibration_summary.json`
with detection count, score, precision, recall, F1, mean IoU and false
with persisted detection count, raw candidate count, suppressed duplicate count,
duplicate IoU threshold, score, precision, recall, F1, mean IoU and false
positive/negative counts. It is intended to tune confidence/IoU/model choices,
not to add new inference behavior.
+1
View File
@@ -32,6 +32,7 @@ class Settings(BaseSettings):
yolo_image_size: int = Field(default=640, validation_alias="YOLO_IMAGE_SIZE")
yolo_max_tiles: int = Field(default=100, validation_alias="YOLO_MAX_TILES")
yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS")
yolo_duplicate_iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0, validation_alias="YOLO_DUPLICATE_IOU_THRESHOLD")
yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE")
cors_origins: list[str] | str = Field(
default=["http://localhost:5173", "http://127.0.0.1:5173"],
+63 -11
View File
@@ -138,7 +138,7 @@ class DetectionService:
if model.model_id == resolved_settings.yolo_model_id:
try:
detections = DetectionService._run_configured_yolo(
detections, postprocess_summary = DetectionService._run_configured_yolo(
db=db,
project_id=project_id,
dataset_id=dataset_id,
@@ -165,7 +165,7 @@ class DetectionService:
error_code=exc.code,
message=exc.message,
)
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections))
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections), extra_result=postprocess_summary)
return DetectionRunResponse(
analysis_run_id=analysis_run.id,
job_id=job.id,
@@ -454,8 +454,10 @@ class DetectionService:
db.refresh(job)
@staticmethod
def _mark_success(db, analysis_run: AnalysisRun, job: Job, detection_count: int) -> None:
def _mark_success(db, analysis_run: AnalysisRun, job: Job, detection_count: int, extra_result: dict[str, Any] | None = None) -> None:
result = {"detection_count": detection_count}
if extra_result:
result.update(extra_result)
analysis_run.status = "success"
analysis_run.finished_at = DetectionService._now()
analysis_run.result_json = result
@@ -536,13 +538,13 @@ class DetectionService:
class_filter: list[str],
settings: Settings,
yolo_adapter_class: Type[YoloDetectionAdapter],
) -> list[Detection]:
) -> tuple[list[Detection], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
model_path = Path(settings.yolo_model_path or "").expanduser()
adapter = yolo_adapter_class(settings)
model = adapter.load_model(model_path)
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
persisted: list[Detection] = []
candidates: list[dict[str, Any]] = []
manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") or "EPSG:4326"
for tile in manifest["tiles"]:
tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser())
@@ -561,6 +563,23 @@ class DetectionService:
properties = dict(raw.get("properties") or {})
if model_class_name and model_class_name != class_name:
properties.setdefault("model_class_name", model_class_name)
candidates.append(
{
"class_name": class_name,
"confidence": confidence,
"geometry": geometry,
"bbox": bbox,
"source_tile_path": str(tile_path),
"properties": {**properties, "tile_index": tile.get("index")},
}
)
filtered_candidates = DetectionService._suppress_duplicate_candidates(
candidates,
iou_threshold=float(settings.yolo_duplicate_iou_threshold),
)
persisted: list[Detection] = []
for candidate in filtered_candidates:
bbox = candidate["bbox"]
detection = Detection(
id=uuid.uuid4(),
project_id=project_id,
@@ -569,29 +588,62 @@ class DetectionService:
job_id=job.id,
model_name=model_name,
model_version=model_version,
class_name=class_name,
confidence=confidence,
geometry=from_shape(geometry, srid=4326),
class_name=candidate["class_name"],
confidence=candidate["confidence"],
geometry=from_shape(candidate["geometry"], srid=4326),
bbox_json={
"x_min": float(bbox[0]),
"y_min": float(bbox[1]),
"x_max": float(bbox[2]),
"y_max": float(bbox[3]),
},
source_tile_path=str(tile_path),
properties_json={**properties, "tile_index": tile.get("index")},
source_tile_path=candidate["source_tile_path"],
properties_json=candidate["properties"],
)
db.add(detection)
persisted.append(detection)
db.commit()
for detection in persisted:
db.refresh(detection)
return persisted
return persisted, {
"raw_detection_count": len(candidates),
"suppressed_detection_count": len(candidates) - len(filtered_candidates),
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
}
@staticmethod
def _canonical_class_name(value: Any) -> str:
return str(value or "").strip().casefold()
@staticmethod
def _suppress_duplicate_candidates(candidates: list[dict[str, Any]], iou_threshold: float) -> list[dict[str, Any]]:
if iou_threshold <= 0 or len(candidates) < 2:
return candidates
kept: list[dict[str, Any]] = []
for candidate in sorted(candidates, key=lambda item: float(item["confidence"]), reverse=True):
duplicate = False
for kept_candidate in kept:
if candidate["class_name"] != kept_candidate["class_name"]:
continue
if DetectionService._geometry_iou(candidate["geometry"], kept_candidate["geometry"]) >= iou_threshold:
duplicate = True
break
if not duplicate:
kept.append(candidate)
return kept
@staticmethod
def _geometry_iou(left, right) -> float:
if left.is_empty or right.is_empty:
return 0.0
intersection_area = left.intersection(right).area
if intersection_area <= 0:
return 0.0
union_area = left.union(right).area
if union_area <= 0:
return 0.0
return intersection_area / union_area
@staticmethod
def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int) -> dict[str, Any]:
if not tile_manifest_path:
@@ -94,6 +94,7 @@ def test_env_example_uses_runtime_env_names_read_by_backend_and_frontend() -> No
assert "YOLO_CONFIG_DIR=./storage/ultralytics" in env_example
assert "YOLO_MAX_TILES=100" in env_example
assert "YOLO_MAX_DETECTIONS=1000" in env_example
assert "YOLO_DUPLICATE_IOU_THRESHOLD=0.5" in env_example
assert "ENABLE_YOLO" not in env_example
assert "ENABLE_SAM" not in env_example
assert "VITE_API_BASE_URL=" in env_example
@@ -247,4 +248,5 @@ def test_unraid_deploy_passes_ai_build_arg_and_yolo_runtime_env() -> None:
assert '-e YOLO_MODEL_PATH="$YOLO_MODEL_PATH"' in run_script
assert '-e YOLO_MAX_TILES="$YOLO_MAX_TILES"' in run_script
assert '-e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS"' in run_script
assert '-e YOLO_DUPLICATE_IOU_THRESHOLD="$YOLO_DUPLICATE_IOU_THRESHOLD"' in run_script
assert "-v \"${GEOINTEL_MODELS_PATH}:/app/models\"" in run_script
@@ -18,7 +18,11 @@ def test_detection_calibration_sweep_reuses_real_data_workflow_and_reports_qa_me
assert "REAL_RASTER_PATH" in script
assert "REAL_REFERENCE_VECTOR_PATH" in script
assert "/api/v1/projects/${project_id}/quality-checks" in script
assert "/api/v1/detection/runs/${analysis_run_id}" in script
assert "quality_check_id" in script
assert "raw_detection_count" in script
assert "suppressed_detection_count" in script
assert "duplicate_iou_threshold" in script
assert "false_positives" in script
assert "false_negatives" in script
assert "quality_score" in script
@@ -24,7 +24,11 @@ def test_detection_quality_matrix_compares_models_tiles_and_thresholds() -> None
assert "REAL_RASTER_PATH" in script
assert "REAL_REFERENCE_VECTOR_PATH" in script
assert "/api/v1/projects/${project_id}/quality-checks" in script
assert "/api/v1/detection/runs/${analysis_run_id}" in script
assert "quality_matrix_summary.json" in script
assert "raw_detection_count" in script
assert "suppressed_detection_count" in script
assert "duplicate_iou_threshold" in script
assert "best_by_score" in script
assert "best_by_recall" in script
assert "best_by_precision" in script
@@ -66,6 +66,8 @@ def test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sam
assert "best_overall_by_score" in script
assert "best_by_sample" in script
assert "sample_slug" in script
assert "raw_detection_count" in script
assert "suppressed_detection_count" in script
assert "demo/workflow" not in script
assert "fixture_mode" not in script
assert "will_download_models" not in script
@@ -89,6 +89,25 @@ class MixedCaseYoloAdapter(MockYoloAdapter):
]
class OverlappingTileYoloAdapter(MockYoloAdapter):
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
tile_index = int(tile_path.stem.split("_")[-1])
if tile_index == 0:
bbox = [10.0, 20.0, 30.0, 40.0]
confidence = 0.82
else:
bbox = [11.0, 21.0, 31.0, 41.0]
confidence = 0.91
return [
{
"class_name": "building",
"confidence": confidence,
"bbox": bbox,
"properties": {"adapter": "overlap"},
}
]
class RecordingPredictModel:
def __init__(self) -> None:
self.seen_sources: list[dict] = []
@@ -383,6 +402,37 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_
assert detections[0].properties_json["model_class_name"] == "Building"
def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) -> None:
db, project_id, dataset_id = _project_and_dataset()
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5)
manifest_path = _manifest(tmp_path, tile_count=2)
result = DetectionService.run_detection(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-configured",
confidence_threshold=0.5,
class_filter=["building"],
tile_manifest_path=str(manifest_path),
settings=settings,
yolo_adapter_class=OverlappingTileYoloAdapter,
)
detections = [item for item in db.added if isinstance(item, Detection)]
runs = [item for item in db.added if isinstance(item, AnalysisRun)]
assert result.status == "success"
assert result.detection_count == 1
assert detections[0].confidence == 0.91
assert detections[0].source_tile_path.endswith("tile_0001.tif")
assert runs[0].result_json["raw_detection_count"] == 2
assert runs[0].result_json["suppressed_detection_count"] == 1
assert runs[0].result_json["duplicate_iou_threshold"] == 0.5
def test_yolo_adapter_converts_single_band_tiles_to_rgb_before_prediction(tmp_path: Path) -> None:
Image = pytest.importorskip("PIL.Image")
tile_path = tmp_path / "single_band_tile.tif"
+1
View File
@@ -37,4 +37,5 @@ YOLO_DEVICE=cpu
YOLO_IMAGE_SIZE=640
YOLO_MAX_TILES=100
YOLO_MAX_DETECTIONS=1000
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
YOLO_BATCH_SIZE=1
+2
View File
@@ -31,6 +31,7 @@ YOLO_DEVICE="${YOLO_DEVICE:-cpu}"
YOLO_IMAGE_SIZE="${YOLO_IMAGE_SIZE:-640}"
YOLO_MAX_TILES="${YOLO_MAX_TILES:-100}"
YOLO_MAX_DETECTIONS="${YOLO_MAX_DETECTIONS:-1000}"
YOLO_DUPLICATE_IOU_THRESHOLD="${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5}"
YOLO_BATCH_SIZE="${YOLO_BATCH_SIZE:-1}"
install_dockerman_metadata() {
@@ -92,6 +93,7 @@ docker run -d \
-e YOLO_IMAGE_SIZE="$YOLO_IMAGE_SIZE" \
-e YOLO_MAX_TILES="$YOLO_MAX_TILES" \
-e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS" \
-e YOLO_DUPLICATE_IOU_THRESHOLD="$YOLO_DUPLICATE_IOU_THRESHOLD" \
-e YOLO_BATCH_SIZE="$YOLO_BATCH_SIZE" \
-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data" \
-v "${GEOINTEL_STORAGE_PATH}:/app/storage" \
+1
View File
@@ -29,6 +29,7 @@ services:
YOLO_IMAGE_SIZE: ${YOLO_IMAGE_SIZE:-640}
YOLO_MAX_TILES: ${YOLO_MAX_TILES:-100}
YOLO_MAX_DETECTIONS: ${YOLO_MAX_DETECTIONS:-1000}
YOLO_DUPLICATE_IOU_THRESHOLD: ${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5}
YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1}
ports:
- "${GEOINTEL_FRONTEND_PORT:-1202}:80"
+1
View File
@@ -34,6 +34,7 @@ services:
YOLO_IMAGE_SIZE: ${YOLO_IMAGE_SIZE:-640}
YOLO_MAX_TILES: ${YOLO_MAX_TILES:-100}
YOLO_MAX_DETECTIONS: ${YOLO_MAX_DETECTIONS:-1000}
YOLO_DUPLICATE_IOU_THRESHOLD: ${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5}
YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1}
ports:
- "${GEOINTEL_BACKEND_PORT:-8000}:8000"
+13
View File
@@ -110,6 +110,7 @@ Environment variables:
- `YOLO_IMAGE_SIZE`
- `YOLO_MAX_TILES`
- `YOLO_MAX_DETECTIONS`
- `YOLO_DUPLICATE_IOU_THRESHOLD`
- `YOLO_BATCH_SIZE`
`YOLO_MAX_DETECTIONS` is forwarded to Ultralytics as `max_det` for each
@@ -119,6 +120,15 @@ the upstream default would cap recall before QA/QC begins. Operators may lower
the value for small rasters or raise it for dense urban tiles after reviewing
runtime and false-positive behavior.
After YOLO boxes are georeferenced, configured-YOLO runs apply a GeoIntel
cross-tile duplicate suppression pass before persistence. Candidates are grouped
by canonical class and sorted by confidence; lower-confidence same-class
candidates with EPSG:4326 geometry IoU greater than or equal to
`YOLO_DUPLICATE_IOU_THRESHOLD` are suppressed. The default is `0.5`; set it to
`0` to disable this post-processing for operator debugging. Run summaries record
raw, persisted and suppressed detection counts so calibration evidence remains
auditable.
### Local model asset catalog
GeoIntel can list local runtime model files mounted into the backend model
@@ -205,6 +215,9 @@ persisted project quality-check list to build `calibration_summary.json`.
Results are honest QA/QC evidence from persisted detections and persisted
reference `vector_features`; no demo detections, live provider fetches or model
downloads are introduced by the calibration tool.
Summaries include raw detection candidate count, persisted detection count and
suppressed duplicate count so operators can distinguish model output volume from
GeoIntel post-processing.
For model/tile/threshold selection, use the quality matrix wrapper:
+6
View File
@@ -785,6 +785,12 @@ Validation errors:
- Configured YOLO inference forwards `YOLO_MAX_DETECTIONS` to Ultralytics
`max_det` and defaults to `1000` so dense building AOIs are not silently
limited by the upstream default of 300 detections before persisted QA/QC.
- Configured YOLO applies cross-tile duplicate suppression after pixel boxes are
converted to EPSG:4326 geometries and before `Detection` rows are persisted.
Same-class candidates are confidence-sorted and lower-confidence candidates
with geometry IoU greater than or equal to
`YOLO_DUPLICATE_IOU_THRESHOLD` are suppressed. The default is `0.5`; `0`
disables this GeoIntel-side post-processing for debugging.
- `DETECTION_DEPENDENCY_UNAVAILABLE` when YOLO dependencies are not installed.
- `DETECTION_MODEL_LOAD_FAILED` when the local model file exists but cannot be loaded.
+32
View File
@@ -1,3 +1,35 @@
## Sprint 149 YOLO duplicate suppression evidence (2026-07-09)
Changed:
- Added configured-YOLO cross-tile duplicate suppression after pixel boxes are converted to EPSG:4326 geometries and before `Detection` rows are persisted.
- Added backend setting `YOLO_DUPLICATE_IOU_THRESHOLD` with default `0.5`; `0` disables the GeoIntel-side pass for debugging.
- Detection run `result_json` now records:
- `raw_detection_count`
- `suppressed_detection_count`
- `duplicate_iou_threshold`
- Updated `.env.example`, Docker Compose, Unraid env examples and the Dockerman run script.
- Updated calibration and quality matrix scripts to fetch detection run details and include raw/suppressed counts in per-run and aggregate summaries.
- Updated backend/API/AI/pipeline documentation.
Why:
- Dense overlapping tile inference can produce duplicate candidate buildings, which inflates persisted false positives before QA/QC.
- The previous Sprint 148 cap fix allowed dense AOIs to persist more candidates, but made duplicate pressure more visible.
- This pass keeps all outputs honest: no model activation, no fake detections and no migration. It only removes lower-confidence same-class geometric duplicates before persistence.
Tested:
- Red step: `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py::test_yolo_run_suppresses_cross_tile_duplicate_detections -q` failed with `detection_count == 2`.
- `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py::test_yolo_run_suppresses_cross_tile_duplicate_detections -q` (`1 passed`)
- Red step: Docker runtime config tests failed before `.env.example` and Unraid runner exposed `YOLO_DUPLICATE_IOU_THRESHOLD`.
- `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py backend\tests\test_docker_runtime_config.py::test_env_example_uses_runtime_env_names_read_by_backend_and_frontend backend\tests\test_docker_runtime_config.py::test_unraid_deploy_passes_ai_build_arg_and_yolo_runtime_env -q` (`17 passed`)
- Red step: calibration/matrix script tests failed before detection-run raw/suppressed metadata was included.
- `python -m pytest backend\tests\test_sprint124_detection_calibration_sweep.py backend\tests\test_sprint126_detection_quality_matrix.py backend\tests\test_sprint127_operator_sample_quality_matrix.py::test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sample -q` (`3 passed`)
- `bash -n scripts/run_detection_calibration_sweep.sh`
- `bash -n scripts/run_detection_quality_matrix.sh`
- `bash -n scripts/run_multi_sample_detection_quality_matrix.sh`
Next:
- Run full readiness, then deploy Tower and rerun the dense Westerlo/Turnhout sweeps with duplicate suppression enabled to quantify quality impact versus the Sprint 148 uncapped baseline.
## Sprint 148 YOLO max-detection cap hardening (2026-07-09)
Changed:
+3
View File
@@ -110,6 +110,9 @@ Default method:
- calculate IoU between overlapping geospatial bboxes
- apply non-maximum suppression by confidence
- default IoU merge threshold: 0.5
- implemented for configured-YOLO as EPSG:4326 cross-tile duplicate suppression
before `Detection` persistence; `YOLO_DUPLICATE_IOU_THRESHOLD=0` disables the
GeoIntel-side pass for debugging.
## Step 6 — Storage
+1
View File
@@ -22,6 +22,7 @@ YOLO_MODEL_PATH=
YOLO_MODEL_VERSION=
YOLO_MAX_TILES=100
YOLO_MAX_DETECTIONS=1000
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
ENABLE_GRB_WFS=false
GRB_WFS_URL=
OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter
+2 -1
View File
@@ -112,10 +112,11 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Train and gate an AOI-scale `aoi512e80` YOLOv8s candidate to test the 160px training-scale hypothesis.
- [x] Raise configured-YOLO `max_det` through `YOLO_MAX_DETECTIONS` so dense AOIs are not capped at 300 detections before QA/QC.
- [x] Rerun live dense-AOI calibration after redeploy with `YOLO_MAX_DETECTIONS=1000`; Westerlo reached 523/1000 detections at lower thresholds and Turnhout reached 822/1000, confirming the old 300 cap is removed.
- [x] Add configured-YOLO cross-tile duplicate suppression and raw/suppressed calibration evidence fields.
- [ ] Find or train a materially stronger aerial/Kempen building model candidate; `geointel-building-yolov8n-expanded160e50-pt` is the best current dense-AOI candidate but still too weak and too noisy for a V1 default.
- [ ] Train a higher-capacity local aerial-building detector with stronger positive recall while preserving the hard-negative false-positive gate.
- [ ] Add more diverse positive AOIs and revisit geometry-to-box label strategy before the next default-model training attempt.
- [ ] Add operator-side duplicate suppression/post-processing analysis for dense overlapping tile detections before the next promotion gate.
- [ ] Rerun live dense-AOI calibration after redeploy with `YOLO_DUPLICATE_IOU_THRESHOLD=0.5` to measure post-processing impact versus the Sprint 148 uncapped baseline.
## Sprint 8 status
+3
View File
@@ -226,6 +226,9 @@ logs plus `calibration_summary.json` under
`CALIBRATION_OUTPUT_DIR` is set. This is a calibration/benchmarking tool only:
it does not seed demo data, enable fixture detections, fetch external data or
download model weights.
Per-threshold summaries include persisted detection count, raw candidate count
before GeoIntel duplicate suppression, suppressed duplicate count and the
configured duplicate IoU threshold.
Run a broader model/tile/threshold quality matrix when multiple local model
assets or tile settings need to be compared:
+18 -4
View File
@@ -98,6 +98,7 @@ for threshold in ${thresholds_normalized}; do
threshold_label="$(printf '%s' "${threshold}" | tr '.-' 'pm')"
run_log="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}.log"
quality_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_quality_checks.json"
detection_run_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_detection_run.json"
run_summary="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_summary.json"
echo "-- Threshold ${threshold} (${run_index}) --"
@@ -121,9 +122,11 @@ for threshold in ${thresholds_normalized}; do
fi
curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}"
curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}" > "${detection_run_response}"
"${PYTHON_BIN}" - \
"${quality_response}" \
"${detection_run_response}" \
"${run_summary}" \
"${threshold}" \
"${project_id}" \
@@ -134,11 +137,17 @@ for threshold in ${thresholds_normalized}; do
import json
import sys
quality_path, output_path, threshold, project_id, analysis_run_id, quality_check_id, detection_count, run_log = sys.argv[1:9]
quality_path, detection_run_path, output_path, threshold, project_id, analysis_run_id, quality_check_id, detection_count, run_log = sys.argv[1:10]
with open(quality_path, "r", encoding="utf-8") as handle:
payload = json.load(handle)
if "data" not in payload:
raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope")
with open(detection_run_path, "r", encoding="utf-8") as handle:
detection_run_payload = json.load(handle)
if "data" not in detection_run_payload:
raise SystemExit("Detection run detail is not a canonical GeoIntel data envelope")
detection_run = detection_run_payload["data"]
run_result = detection_run.get("result_json") or {}
items = payload["data"].get("items") or []
quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None)
if quality_check is None:
@@ -157,6 +166,9 @@ summary = {
"detection_count": int(detection_count),
"quality_status": quality_check.get("status"),
"quality_score": quality_check.get("score"),
"raw_detection_count": run_result.get("raw_detection_count", int(detection_count)),
"suppressed_detection_count": run_result.get("suppressed_detection_count", 0),
"duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"),
"precision": metrics.get("precision"),
"recall": metrics.get("recall"),
"f1_score": metrics.get("f1_score", metrics.get("f1")),
@@ -169,10 +181,12 @@ summary = {
with open(output_path, "w", encoding="utf-8") as handle:
json.dump(summary, handle, indent=2, sort_keys=True)
print(
"threshold={threshold} detections={detections} score={score} "
"threshold={threshold} detections={detections} raw={raw} suppressed={suppressed} score={score} "
"precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format(
threshold=summary["threshold"],
detections=summary["detection_count"],
raw=summary["raw_detection_count"],
suppressed=summary["suppressed_detection_count"],
score=summary["quality_score"],
precision=summary["precision"],
recall=summary["recall"],
@@ -217,10 +231,10 @@ with open(summary_path, "w", encoding="utf-8") as handle:
print("")
print("Detection calibration summary")
print("threshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
print("threshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
for item in items:
print(
"{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
"{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
**item
)
)
+19 -4
View File
@@ -168,6 +168,7 @@ while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label
run_index=$((run_index + 1))
run_log="${QUALITY_OUTPUT_DIR}/${run_label}.log"
quality_response="${QUALITY_OUTPUT_DIR}/${run_label}_quality_checks.json"
detection_run_response="${QUALITY_OUTPUT_DIR}/${run_label}_detection_run.json"
run_summary="${QUALITY_OUTPUT_DIR}/${run_label}_summary.json"
model_env="${model_request}"
if [ "${model_request}" = "__active__" ]; then
@@ -203,9 +204,11 @@ while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label
fi
curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}"
curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}" > "${detection_run_response}"
"${PYTHON_BIN}" - \
"${quality_response}" \
"${detection_run_response}" \
"${run_summary}" \
"${model_request}" \
"${selected_model_asset_id}" \
@@ -226,6 +229,7 @@ import sys
(
quality_path,
detection_run_path,
output_path,
model_request,
selected_model_asset_id,
@@ -241,12 +245,18 @@ import sys
detection_count,
export_id,
run_log,
) = sys.argv[1:17]
) = sys.argv[1:18]
with open(quality_path, "r", encoding="utf-8") as handle:
payload = json.load(handle)
if "data" not in payload:
raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope")
with open(detection_run_path, "r", encoding="utf-8") as handle:
detection_run_payload = json.load(handle)
if "data" not in detection_run_payload:
raise SystemExit("Detection run detail is not a canonical GeoIntel data envelope")
detection_run = detection_run_payload["data"]
run_result = detection_run.get("result_json") or {}
items = payload["data"].get("items") or []
quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None)
if quality_check is None:
@@ -270,6 +280,9 @@ summary = {
"analysis_run_id": analysis_run_id,
"quality_check_id": quality_check_id,
"detection_count": int(detection_count),
"raw_detection_count": run_result.get("raw_detection_count", int(detection_count)),
"suppressed_detection_count": run_result.get("suppressed_detection_count", 0),
"duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"),
"quality_status": quality_check.get("status"),
"quality_score": quality_check.get("score"),
"precision": metrics.get("precision"),
@@ -285,13 +298,15 @@ summary = {
with open(output_path, "w", encoding="utf-8") as handle:
json.dump(summary, handle, indent=2, sort_keys=True)
print(
"model={model} tile={tile} overlap={overlap} threshold={threshold} detections={detections} "
"model={model} tile={tile} overlap={overlap} threshold={threshold} detections={detections} raw={raw} suppressed={suppressed} "
"score={score} precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format(
model=summary["model_asset_id"],
tile=summary["tile_size"],
overlap=summary["tile_overlap"],
threshold=summary["threshold"],
detections=summary["detection_count"],
raw=summary["raw_detection_count"],
suppressed=summary["suppressed_detection_count"],
score=summary["quality_score"],
precision=summary["precision"],
recall=summary["recall"],
@@ -346,10 +361,10 @@ with open(summary_path, "w", encoding="utf-8") as handle:
print("")
print("Detection quality matrix summary")
print("model\ttile\toverlap\tthreshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
print("model\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
for item in items:
print(
"{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
"{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
**item
)
)
@@ -205,10 +205,10 @@ summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding=
print("")
print("Multi-sample detection quality summary")
print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
for item in flat_items:
print(
"{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
"{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
**item
)
)