From 3f5ca4c85e0b93cadb14778ba539a710773469d4 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 9 Jul 2026 10:55:45 +0200 Subject: [PATCH] Suppress duplicate YOLO tile detections --- .env.example | 1 + CHANGELOG.md | 9 ++ backend/README.md | 11 +- backend/app/core/config.py | 1 + backend/app/services/detection_service.py | 106 +++++++++++++----- backend/tests/test_docker_runtime_config.py | 2 + ...t_sprint124_detection_calibration_sweep.py | 4 + ...test_sprint126_detection_quality_matrix.py | 4 + ...print127_operator_sample_quality_matrix.py | 2 + .../tests/test_sprint8b_yolo_foundation.py | 50 +++++++++ deploy/unraid/geointel.env.example | 1 + deploy/unraid/run-dockerman-container.sh | 2 + docker-compose.unraid.yml | 1 + docker-compose.yml | 1 + docs/AI_PIPELINES.md | 13 +++ docs/API_CONTRACTS.md | 6 + docs/CODEX_EXECUTION_LOG.md | 32 ++++++ docs/DETECTION_PIPELINE_SPEC.md | 3 + docs/ENVIRONMENT_SPEC.md | 1 + docs/TODO.md | 3 +- scripts/README.md | 3 + scripts/run_detection_calibration_sweep.sh | 22 +++- scripts/run_detection_quality_matrix.sh | 23 +++- ...n_multi_sample_detection_quality_matrix.sh | 4 +- 24 files changed, 266 insertions(+), 39 deletions(-) diff --git a/.env.example b/.env.example index c9d262ff..9eb94e86 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,7 @@ YOLO_DEVICE=cpu YOLO_IMAGE_SIZE=640 YOLO_MAX_TILES=100 YOLO_MAX_DETECTIONS=1000 +YOLO_DUPLICATE_IOU_THRESHOLD=0.5 YOLO_BATCH_SIZE=1 ENABLE_GRB_WFS=false GRB_WFS_URL= diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fd13cfb..52900fbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ # Changelog +## Sprint 149 YOLO duplicate suppression evidence (2026-07-09) + +- Added configured-YOLO cross-tile duplicate suppression before `Detection` rows are persisted. +- Added `YOLO_DUPLICATE_IOU_THRESHOLD` with default `0.5`; `0` disables the GeoIntel-side pass for debugging. +- Detection run result metadata now records raw candidate count, suppressed duplicate count and duplicate IoU threshold. +- Calibration and quality matrix scripts now fetch detection run details and include raw/suppressed counts in summaries. +- Updated Docker/Unraid env examples, API/AI/backend docs and detection pipeline notes. +- No model was activated, no detections were faked, and no migration changed. + ## Sprint 148 YOLO max-detection cap hardening (2026-07-09) - Added `YOLO_MAX_DETECTIONS` with default `1000` and forward it to Ultralytics as `max_det`. diff --git a/backend/README.md b/backend/README.md index 92388409..b7a51e77 100644 --- a/backend/README.md +++ b/backend/README.md @@ -360,6 +360,7 @@ YOLO_DEVICE=cpu YOLO_IMAGE_SIZE=640 YOLO_MAX_TILES=100 YOLO_MAX_DETECTIONS=1000 +YOLO_DUPLICATE_IOU_THRESHOLD=0.5 YOLO_BATCH_SIZE=1 ``` @@ -389,6 +390,13 @@ The preflight checks configuration, dependency availability, local model file ex `1000` because dense building AOIs can exceed the upstream default cap of 300 detections before QA/QC can measure recall honestly. +`YOLO_DUPLICATE_IOU_THRESHOLD` controls GeoIntel-side cross-tile duplicate +suppression after YOLO pixel boxes are converted to EPSG:4326 polygons and +before `Detection` rows are persisted. Candidates are sorted by confidence per +class; lower-confidence same-class candidates with geometry IoU greater than or +equal to the threshold are suppressed. The default is `0.5`; set `0` to disable +this post-processing for debugging. + The same read-only status is available through the API and Detection Lab UI: ```bash @@ -458,7 +466,8 @@ bash scripts/run_detection_calibration_sweep.sh http://192.168.10.150:1202 The sweep creates one real persisted workflow run per threshold, fetches the persisted `QualityCheck`/`Metric` rows and writes a `calibration_summary.json` -with detection count, score, precision, recall, F1, mean IoU and false +with persisted detection count, raw candidate count, suppressed duplicate count, +duplicate IoU threshold, score, precision, recall, F1, mean IoU and false positive/negative counts. It is intended to tune confidence/IoU/model choices, not to add new inference behavior. diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 425b3454..1698e69d 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -32,6 +32,7 @@ class Settings(BaseSettings): yolo_image_size: int = Field(default=640, validation_alias="YOLO_IMAGE_SIZE") yolo_max_tiles: int = Field(default=100, validation_alias="YOLO_MAX_TILES") yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS") + yolo_duplicate_iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0, validation_alias="YOLO_DUPLICATE_IOU_THRESHOLD") yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE") cors_origins: list[str] | str = Field( default=["http://localhost:5173", "http://127.0.0.1:5173"], diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index dca7c2d9..5beb0554 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -138,7 +138,7 @@ class DetectionService: if model.model_id == resolved_settings.yolo_model_id: try: - detections = DetectionService._run_configured_yolo( + detections, postprocess_summary = DetectionService._run_configured_yolo( db=db, project_id=project_id, dataset_id=dataset_id, @@ -165,7 +165,7 @@ class DetectionService: error_code=exc.code, message=exc.message, ) - DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections)) + DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections), extra_result=postprocess_summary) return DetectionRunResponse( analysis_run_id=analysis_run.id, job_id=job.id, @@ -454,8 +454,10 @@ class DetectionService: db.refresh(job) @staticmethod - def _mark_success(db, analysis_run: AnalysisRun, job: Job, detection_count: int) -> None: + def _mark_success(db, analysis_run: AnalysisRun, job: Job, detection_count: int, extra_result: dict[str, Any] | None = None) -> None: result = {"detection_count": detection_count} + if extra_result: + result.update(extra_result) analysis_run.status = "success" analysis_run.finished_at = DetectionService._now() analysis_run.result_json = result @@ -536,13 +538,13 @@ class DetectionService: class_filter: list[str], settings: Settings, yolo_adapter_class: Type[YoloDetectionAdapter], - ) -> list[Detection]: + ) -> tuple[list[Detection], dict[str, Any]]: manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles) model_path = Path(settings.yolo_model_path or "").expanduser() adapter = yolo_adapter_class(settings) model = adapter.load_model(model_path) allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)} - persisted: list[Detection] = [] + candidates: list[dict[str, Any]] = [] manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") or "EPSG:4326" for tile in manifest["tiles"]: tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser()) @@ -561,37 +563,87 @@ class DetectionService: properties = dict(raw.get("properties") or {}) if model_class_name and model_class_name != class_name: properties.setdefault("model_class_name", model_class_name) - detection = Detection( - id=uuid.uuid4(), - project_id=project_id, - dataset_id=dataset_id, - analysis_run_id=analysis_run.id, - job_id=job.id, - model_name=model_name, - model_version=model_version, - class_name=class_name, - confidence=confidence, - geometry=from_shape(geometry, srid=4326), - bbox_json={ - "x_min": float(bbox[0]), - "y_min": float(bbox[1]), - "x_max": float(bbox[2]), - "y_max": float(bbox[3]), - }, - source_tile_path=str(tile_path), - properties_json={**properties, "tile_index": tile.get("index")}, + candidates.append( + { + "class_name": class_name, + "confidence": confidence, + "geometry": geometry, + "bbox": bbox, + "source_tile_path": str(tile_path), + "properties": {**properties, "tile_index": tile.get("index")}, + } ) - db.add(detection) - persisted.append(detection) + filtered_candidates = DetectionService._suppress_duplicate_candidates( + candidates, + iou_threshold=float(settings.yolo_duplicate_iou_threshold), + ) + persisted: list[Detection] = [] + for candidate in filtered_candidates: + bbox = candidate["bbox"] + detection = Detection( + id=uuid.uuid4(), + project_id=project_id, + dataset_id=dataset_id, + analysis_run_id=analysis_run.id, + job_id=job.id, + model_name=model_name, + model_version=model_version, + class_name=candidate["class_name"], + confidence=candidate["confidence"], + geometry=from_shape(candidate["geometry"], srid=4326), + bbox_json={ + "x_min": float(bbox[0]), + "y_min": float(bbox[1]), + "x_max": float(bbox[2]), + "y_max": float(bbox[3]), + }, + source_tile_path=candidate["source_tile_path"], + properties_json=candidate["properties"], + ) + db.add(detection) + persisted.append(detection) db.commit() for detection in persisted: db.refresh(detection) - return persisted + return persisted, { + "raw_detection_count": len(candidates), + "suppressed_detection_count": len(candidates) - len(filtered_candidates), + "duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold), + } @staticmethod def _canonical_class_name(value: Any) -> str: return str(value or "").strip().casefold() + @staticmethod + def _suppress_duplicate_candidates(candidates: list[dict[str, Any]], iou_threshold: float) -> list[dict[str, Any]]: + if iou_threshold <= 0 or len(candidates) < 2: + return candidates + kept: list[dict[str, Any]] = [] + for candidate in sorted(candidates, key=lambda item: float(item["confidence"]), reverse=True): + duplicate = False + for kept_candidate in kept: + if candidate["class_name"] != kept_candidate["class_name"]: + continue + if DetectionService._geometry_iou(candidate["geometry"], kept_candidate["geometry"]) >= iou_threshold: + duplicate = True + break + if not duplicate: + kept.append(candidate) + return kept + + @staticmethod + def _geometry_iou(left, right) -> float: + if left.is_empty or right.is_empty: + return 0.0 + intersection_area = left.intersection(right).area + if intersection_area <= 0: + return 0.0 + union_area = left.union(right).area + if union_area <= 0: + return 0.0 + return intersection_area / union_area + @staticmethod def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int) -> dict[str, Any]: if not tile_manifest_path: diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py index a79c0721..31ef2b77 100644 --- a/backend/tests/test_docker_runtime_config.py +++ b/backend/tests/test_docker_runtime_config.py @@ -94,6 +94,7 @@ def test_env_example_uses_runtime_env_names_read_by_backend_and_frontend() -> No assert "YOLO_CONFIG_DIR=./storage/ultralytics" in env_example assert "YOLO_MAX_TILES=100" in env_example assert "YOLO_MAX_DETECTIONS=1000" in env_example + assert "YOLO_DUPLICATE_IOU_THRESHOLD=0.5" in env_example assert "ENABLE_YOLO" not in env_example assert "ENABLE_SAM" not in env_example assert "VITE_API_BASE_URL=" in env_example @@ -247,4 +248,5 @@ def test_unraid_deploy_passes_ai_build_arg_and_yolo_runtime_env() -> None: assert '-e YOLO_MODEL_PATH="$YOLO_MODEL_PATH"' in run_script assert '-e YOLO_MAX_TILES="$YOLO_MAX_TILES"' in run_script assert '-e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS"' in run_script + assert '-e YOLO_DUPLICATE_IOU_THRESHOLD="$YOLO_DUPLICATE_IOU_THRESHOLD"' in run_script assert "-v \"${GEOINTEL_MODELS_PATH}:/app/models\"" in run_script diff --git a/backend/tests/test_sprint124_detection_calibration_sweep.py b/backend/tests/test_sprint124_detection_calibration_sweep.py index 777bc7d7..f81f8ced 100644 --- a/backend/tests/test_sprint124_detection_calibration_sweep.py +++ b/backend/tests/test_sprint124_detection_calibration_sweep.py @@ -18,7 +18,11 @@ def test_detection_calibration_sweep_reuses_real_data_workflow_and_reports_qa_me assert "REAL_RASTER_PATH" in script assert "REAL_REFERENCE_VECTOR_PATH" in script assert "/api/v1/projects/${project_id}/quality-checks" in script + assert "/api/v1/detection/runs/${analysis_run_id}" in script assert "quality_check_id" in script + assert "raw_detection_count" in script + assert "suppressed_detection_count" in script + assert "duplicate_iou_threshold" in script assert "false_positives" in script assert "false_negatives" in script assert "quality_score" in script diff --git a/backend/tests/test_sprint126_detection_quality_matrix.py b/backend/tests/test_sprint126_detection_quality_matrix.py index f73f657f..61fa995e 100644 --- a/backend/tests/test_sprint126_detection_quality_matrix.py +++ b/backend/tests/test_sprint126_detection_quality_matrix.py @@ -24,7 +24,11 @@ def test_detection_quality_matrix_compares_models_tiles_and_thresholds() -> None assert "REAL_RASTER_PATH" in script assert "REAL_REFERENCE_VECTOR_PATH" in script assert "/api/v1/projects/${project_id}/quality-checks" in script + assert "/api/v1/detection/runs/${analysis_run_id}" in script assert "quality_matrix_summary.json" in script + assert "raw_detection_count" in script + assert "suppressed_detection_count" in script + assert "duplicate_iou_threshold" in script assert "best_by_score" in script assert "best_by_recall" in script assert "best_by_precision" in script diff --git a/backend/tests/test_sprint127_operator_sample_quality_matrix.py b/backend/tests/test_sprint127_operator_sample_quality_matrix.py index 13e909a3..ec22e4cd 100644 --- a/backend/tests/test_sprint127_operator_sample_quality_matrix.py +++ b/backend/tests/test_sprint127_operator_sample_quality_matrix.py @@ -66,6 +66,8 @@ def test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sam assert "best_overall_by_score" in script assert "best_by_sample" in script assert "sample_slug" in script + assert "raw_detection_count" in script + assert "suppressed_detection_count" in script assert "demo/workflow" not in script assert "fixture_mode" not in script assert "will_download_models" not in script diff --git a/backend/tests/test_sprint8b_yolo_foundation.py b/backend/tests/test_sprint8b_yolo_foundation.py index 344fb3a5..49222dd5 100644 --- a/backend/tests/test_sprint8b_yolo_foundation.py +++ b/backend/tests/test_sprint8b_yolo_foundation.py @@ -89,6 +89,25 @@ class MixedCaseYoloAdapter(MockYoloAdapter): ] +class OverlappingTileYoloAdapter(MockYoloAdapter): + def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]: + tile_index = int(tile_path.stem.split("_")[-1]) + if tile_index == 0: + bbox = [10.0, 20.0, 30.0, 40.0] + confidence = 0.82 + else: + bbox = [11.0, 21.0, 31.0, 41.0] + confidence = 0.91 + return [ + { + "class_name": "building", + "confidence": confidence, + "bbox": bbox, + "properties": {"adapter": "overlap"}, + } + ] + + class RecordingPredictModel: def __init__(self) -> None: self.seen_sources: list[dict] = [] @@ -383,6 +402,37 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_ assert detections[0].properties_json["model_class_name"] == "Building" +def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) -> None: + db, project_id, dataset_id = _project_and_dataset() + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"local weights") + settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5) + manifest_path = _manifest(tmp_path, tile_count=2) + + result = DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-configured", + confidence_threshold=0.5, + class_filter=["building"], + tile_manifest_path=str(manifest_path), + settings=settings, + yolo_adapter_class=OverlappingTileYoloAdapter, + ) + + detections = [item for item in db.added if isinstance(item, Detection)] + runs = [item for item in db.added if isinstance(item, AnalysisRun)] + + assert result.status == "success" + assert result.detection_count == 1 + assert detections[0].confidence == 0.91 + assert detections[0].source_tile_path.endswith("tile_0001.tif") + assert runs[0].result_json["raw_detection_count"] == 2 + assert runs[0].result_json["suppressed_detection_count"] == 1 + assert runs[0].result_json["duplicate_iou_threshold"] == 0.5 + + def test_yolo_adapter_converts_single_band_tiles_to_rgb_before_prediction(tmp_path: Path) -> None: Image = pytest.importorskip("PIL.Image") tile_path = tmp_path / "single_band_tile.tif" diff --git a/deploy/unraid/geointel.env.example b/deploy/unraid/geointel.env.example index 5855b8d1..88d9c00f 100644 --- a/deploy/unraid/geointel.env.example +++ b/deploy/unraid/geointel.env.example @@ -37,4 +37,5 @@ YOLO_DEVICE=cpu YOLO_IMAGE_SIZE=640 YOLO_MAX_TILES=100 YOLO_MAX_DETECTIONS=1000 +YOLO_DUPLICATE_IOU_THRESHOLD=0.5 YOLO_BATCH_SIZE=1 diff --git a/deploy/unraid/run-dockerman-container.sh b/deploy/unraid/run-dockerman-container.sh index a5db4323..e4d25124 100644 --- a/deploy/unraid/run-dockerman-container.sh +++ b/deploy/unraid/run-dockerman-container.sh @@ -31,6 +31,7 @@ YOLO_DEVICE="${YOLO_DEVICE:-cpu}" YOLO_IMAGE_SIZE="${YOLO_IMAGE_SIZE:-640}" YOLO_MAX_TILES="${YOLO_MAX_TILES:-100}" YOLO_MAX_DETECTIONS="${YOLO_MAX_DETECTIONS:-1000}" +YOLO_DUPLICATE_IOU_THRESHOLD="${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5}" YOLO_BATCH_SIZE="${YOLO_BATCH_SIZE:-1}" install_dockerman_metadata() { @@ -92,6 +93,7 @@ docker run -d \ -e YOLO_IMAGE_SIZE="$YOLO_IMAGE_SIZE" \ -e YOLO_MAX_TILES="$YOLO_MAX_TILES" \ -e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS" \ + -e YOLO_DUPLICATE_IOU_THRESHOLD="$YOLO_DUPLICATE_IOU_THRESHOLD" \ -e YOLO_BATCH_SIZE="$YOLO_BATCH_SIZE" \ -v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data" \ -v "${GEOINTEL_STORAGE_PATH}:/app/storage" \ diff --git a/docker-compose.unraid.yml b/docker-compose.unraid.yml index 368fb299..fc88ef41 100644 --- a/docker-compose.unraid.yml +++ b/docker-compose.unraid.yml @@ -29,6 +29,7 @@ services: YOLO_IMAGE_SIZE: ${YOLO_IMAGE_SIZE:-640} YOLO_MAX_TILES: ${YOLO_MAX_TILES:-100} YOLO_MAX_DETECTIONS: ${YOLO_MAX_DETECTIONS:-1000} + YOLO_DUPLICATE_IOU_THRESHOLD: ${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5} YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1} ports: - "${GEOINTEL_FRONTEND_PORT:-1202}:80" diff --git a/docker-compose.yml b/docker-compose.yml index a2877c3c..7916532a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,7 @@ services: YOLO_IMAGE_SIZE: ${YOLO_IMAGE_SIZE:-640} YOLO_MAX_TILES: ${YOLO_MAX_TILES:-100} YOLO_MAX_DETECTIONS: ${YOLO_MAX_DETECTIONS:-1000} + YOLO_DUPLICATE_IOU_THRESHOLD: ${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5} YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1} ports: - "${GEOINTEL_BACKEND_PORT:-8000}:8000" diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 43a8abe2..90f077bd 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -110,6 +110,7 @@ Environment variables: - `YOLO_IMAGE_SIZE` - `YOLO_MAX_TILES` - `YOLO_MAX_DETECTIONS` +- `YOLO_DUPLICATE_IOU_THRESHOLD` - `YOLO_BATCH_SIZE` `YOLO_MAX_DETECTIONS` is forwarded to Ultralytics as `max_det` for each @@ -119,6 +120,15 @@ the upstream default would cap recall before QA/QC begins. Operators may lower the value for small rasters or raise it for dense urban tiles after reviewing runtime and false-positive behavior. +After YOLO boxes are georeferenced, configured-YOLO runs apply a GeoIntel +cross-tile duplicate suppression pass before persistence. Candidates are grouped +by canonical class and sorted by confidence; lower-confidence same-class +candidates with EPSG:4326 geometry IoU greater than or equal to +`YOLO_DUPLICATE_IOU_THRESHOLD` are suppressed. The default is `0.5`; set it to +`0` to disable this post-processing for operator debugging. Run summaries record +raw, persisted and suppressed detection counts so calibration evidence remains +auditable. + ### Local model asset catalog GeoIntel can list local runtime model files mounted into the backend model @@ -205,6 +215,9 @@ persisted project quality-check list to build `calibration_summary.json`. Results are honest QA/QC evidence from persisted detections and persisted reference `vector_features`; no demo detections, live provider fetches or model downloads are introduced by the calibration tool. +Summaries include raw detection candidate count, persisted detection count and +suppressed duplicate count so operators can distinguish model output volume from +GeoIntel post-processing. For model/tile/threshold selection, use the quality matrix wrapper: diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 5f1575e2..17075cef 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -785,6 +785,12 @@ Validation errors: - Configured YOLO inference forwards `YOLO_MAX_DETECTIONS` to Ultralytics `max_det` and defaults to `1000` so dense building AOIs are not silently limited by the upstream default of 300 detections before persisted QA/QC. +- Configured YOLO applies cross-tile duplicate suppression after pixel boxes are + converted to EPSG:4326 geometries and before `Detection` rows are persisted. + Same-class candidates are confidence-sorted and lower-confidence candidates + with geometry IoU greater than or equal to + `YOLO_DUPLICATE_IOU_THRESHOLD` are suppressed. The default is `0.5`; `0` + disables this GeoIntel-side post-processing for debugging. - `DETECTION_DEPENDENCY_UNAVAILABLE` when YOLO dependencies are not installed. - `DETECTION_MODEL_LOAD_FAILED` when the local model file exists but cannot be loaded. diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index e25290fd..5c9e6704 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,35 @@ +## Sprint 149 YOLO duplicate suppression evidence (2026-07-09) + +Changed: +- Added configured-YOLO cross-tile duplicate suppression after pixel boxes are converted to EPSG:4326 geometries and before `Detection` rows are persisted. +- Added backend setting `YOLO_DUPLICATE_IOU_THRESHOLD` with default `0.5`; `0` disables the GeoIntel-side pass for debugging. +- Detection run `result_json` now records: + - `raw_detection_count` + - `suppressed_detection_count` + - `duplicate_iou_threshold` +- Updated `.env.example`, Docker Compose, Unraid env examples and the Dockerman run script. +- Updated calibration and quality matrix scripts to fetch detection run details and include raw/suppressed counts in per-run and aggregate summaries. +- Updated backend/API/AI/pipeline documentation. + +Why: +- Dense overlapping tile inference can produce duplicate candidate buildings, which inflates persisted false positives before QA/QC. +- The previous Sprint 148 cap fix allowed dense AOIs to persist more candidates, but made duplicate pressure more visible. +- This pass keeps all outputs honest: no model activation, no fake detections and no migration. It only removes lower-confidence same-class geometric duplicates before persistence. + +Tested: +- Red step: `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py::test_yolo_run_suppresses_cross_tile_duplicate_detections -q` failed with `detection_count == 2`. +- `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py::test_yolo_run_suppresses_cross_tile_duplicate_detections -q` (`1 passed`) +- Red step: Docker runtime config tests failed before `.env.example` and Unraid runner exposed `YOLO_DUPLICATE_IOU_THRESHOLD`. +- `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py backend\tests\test_docker_runtime_config.py::test_env_example_uses_runtime_env_names_read_by_backend_and_frontend backend\tests\test_docker_runtime_config.py::test_unraid_deploy_passes_ai_build_arg_and_yolo_runtime_env -q` (`17 passed`) +- Red step: calibration/matrix script tests failed before detection-run raw/suppressed metadata was included. +- `python -m pytest backend\tests\test_sprint124_detection_calibration_sweep.py backend\tests\test_sprint126_detection_quality_matrix.py backend\tests\test_sprint127_operator_sample_quality_matrix.py::test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sample -q` (`3 passed`) +- `bash -n scripts/run_detection_calibration_sweep.sh` +- `bash -n scripts/run_detection_quality_matrix.sh` +- `bash -n scripts/run_multi_sample_detection_quality_matrix.sh` + +Next: +- Run full readiness, then deploy Tower and rerun the dense Westerlo/Turnhout sweeps with duplicate suppression enabled to quantify quality impact versus the Sprint 148 uncapped baseline. + ## Sprint 148 YOLO max-detection cap hardening (2026-07-09) Changed: diff --git a/docs/DETECTION_PIPELINE_SPEC.md b/docs/DETECTION_PIPELINE_SPEC.md index 70afee6d..7c7b3077 100644 --- a/docs/DETECTION_PIPELINE_SPEC.md +++ b/docs/DETECTION_PIPELINE_SPEC.md @@ -110,6 +110,9 @@ Default method: - calculate IoU between overlapping geospatial bboxes - apply non-maximum suppression by confidence - default IoU merge threshold: 0.5 +- implemented for configured-YOLO as EPSG:4326 cross-tile duplicate suppression + before `Detection` persistence; `YOLO_DUPLICATE_IOU_THRESHOLD=0` disables the + GeoIntel-side pass for debugging. ## Step 6 — Storage diff --git a/docs/ENVIRONMENT_SPEC.md b/docs/ENVIRONMENT_SPEC.md index b0ec07f2..bda34307 100644 --- a/docs/ENVIRONMENT_SPEC.md +++ b/docs/ENVIRONMENT_SPEC.md @@ -22,6 +22,7 @@ YOLO_MODEL_PATH= YOLO_MODEL_VERSION= YOLO_MAX_TILES=100 YOLO_MAX_DETECTIONS=1000 +YOLO_DUPLICATE_IOU_THRESHOLD=0.5 ENABLE_GRB_WFS=false GRB_WFS_URL= OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter diff --git a/docs/TODO.md b/docs/TODO.md index 0caae68a..6b7f91a5 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -112,10 +112,11 @@ This file now starts with the current implementation status. Older preparation/b - [x] Train and gate an AOI-scale `aoi512e80` YOLOv8s candidate to test the 160px training-scale hypothesis. - [x] Raise configured-YOLO `max_det` through `YOLO_MAX_DETECTIONS` so dense AOIs are not capped at 300 detections before QA/QC. - [x] Rerun live dense-AOI calibration after redeploy with `YOLO_MAX_DETECTIONS=1000`; Westerlo reached 523/1000 detections at lower thresholds and Turnhout reached 822/1000, confirming the old 300 cap is removed. +- [x] Add configured-YOLO cross-tile duplicate suppression and raw/suppressed calibration evidence fields. - [ ] Find or train a materially stronger aerial/Kempen building model candidate; `geointel-building-yolov8n-expanded160e50-pt` is the best current dense-AOI candidate but still too weak and too noisy for a V1 default. - [ ] Train a higher-capacity local aerial-building detector with stronger positive recall while preserving the hard-negative false-positive gate. - [ ] Add more diverse positive AOIs and revisit geometry-to-box label strategy before the next default-model training attempt. -- [ ] Add operator-side duplicate suppression/post-processing analysis for dense overlapping tile detections before the next promotion gate. +- [ ] Rerun live dense-AOI calibration after redeploy with `YOLO_DUPLICATE_IOU_THRESHOLD=0.5` to measure post-processing impact versus the Sprint 148 uncapped baseline. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index 12d704ed..30f4b979 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -226,6 +226,9 @@ logs plus `calibration_summary.json` under `CALIBRATION_OUTPUT_DIR` is set. This is a calibration/benchmarking tool only: it does not seed demo data, enable fixture detections, fetch external data or download model weights. +Per-threshold summaries include persisted detection count, raw candidate count +before GeoIntel duplicate suppression, suppressed duplicate count and the +configured duplicate IoU threshold. Run a broader model/tile/threshold quality matrix when multiple local model assets or tile settings need to be compared: diff --git a/scripts/run_detection_calibration_sweep.sh b/scripts/run_detection_calibration_sweep.sh index ccab3ae5..14ed2c8b 100644 --- a/scripts/run_detection_calibration_sweep.sh +++ b/scripts/run_detection_calibration_sweep.sh @@ -98,6 +98,7 @@ for threshold in ${thresholds_normalized}; do threshold_label="$(printf '%s' "${threshold}" | tr '.-' 'pm')" run_log="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}.log" quality_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_quality_checks.json" + detection_run_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_detection_run.json" run_summary="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_summary.json" echo "-- Threshold ${threshold} (${run_index}) --" @@ -121,9 +122,11 @@ for threshold in ${thresholds_normalized}; do fi curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}" + curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}" > "${detection_run_response}" "${PYTHON_BIN}" - \ "${quality_response}" \ + "${detection_run_response}" \ "${run_summary}" \ "${threshold}" \ "${project_id}" \ @@ -134,11 +137,17 @@ for threshold in ${thresholds_normalized}; do import json import sys -quality_path, output_path, threshold, project_id, analysis_run_id, quality_check_id, detection_count, run_log = sys.argv[1:9] +quality_path, detection_run_path, output_path, threshold, project_id, analysis_run_id, quality_check_id, detection_count, run_log = sys.argv[1:10] with open(quality_path, "r", encoding="utf-8") as handle: payload = json.load(handle) if "data" not in payload: raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope") +with open(detection_run_path, "r", encoding="utf-8") as handle: + detection_run_payload = json.load(handle) +if "data" not in detection_run_payload: + raise SystemExit("Detection run detail is not a canonical GeoIntel data envelope") +detection_run = detection_run_payload["data"] +run_result = detection_run.get("result_json") or {} items = payload["data"].get("items") or [] quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None) if quality_check is None: @@ -157,6 +166,9 @@ summary = { "detection_count": int(detection_count), "quality_status": quality_check.get("status"), "quality_score": quality_check.get("score"), + "raw_detection_count": run_result.get("raw_detection_count", int(detection_count)), + "suppressed_detection_count": run_result.get("suppressed_detection_count", 0), + "duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"), "precision": metrics.get("precision"), "recall": metrics.get("recall"), "f1_score": metrics.get("f1_score", metrics.get("f1")), @@ -169,10 +181,12 @@ summary = { with open(output_path, "w", encoding="utf-8") as handle: json.dump(summary, handle, indent=2, sort_keys=True) print( - "threshold={threshold} detections={detections} score={score} " + "threshold={threshold} detections={detections} raw={raw} suppressed={suppressed} score={score} " "precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format( threshold=summary["threshold"], detections=summary["detection_count"], + raw=summary["raw_detection_count"], + suppressed=summary["suppressed_detection_count"], score=summary["quality_score"], precision=summary["precision"], recall=summary["recall"], @@ -217,10 +231,10 @@ with open(summary_path, "w", encoding="utf-8") as handle: print("") print("Detection calibration summary") -print("threshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn") +print("threshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn") for item in items: print( - "{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format( + "{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format( **item ) ) diff --git a/scripts/run_detection_quality_matrix.sh b/scripts/run_detection_quality_matrix.sh index 8906a1df..8b643373 100644 --- a/scripts/run_detection_quality_matrix.sh +++ b/scripts/run_detection_quality_matrix.sh @@ -168,6 +168,7 @@ while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label run_index=$((run_index + 1)) run_log="${QUALITY_OUTPUT_DIR}/${run_label}.log" quality_response="${QUALITY_OUTPUT_DIR}/${run_label}_quality_checks.json" + detection_run_response="${QUALITY_OUTPUT_DIR}/${run_label}_detection_run.json" run_summary="${QUALITY_OUTPUT_DIR}/${run_label}_summary.json" model_env="${model_request}" if [ "${model_request}" = "__active__" ]; then @@ -203,9 +204,11 @@ while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label fi curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}" + curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}" > "${detection_run_response}" "${PYTHON_BIN}" - \ "${quality_response}" \ + "${detection_run_response}" \ "${run_summary}" \ "${model_request}" \ "${selected_model_asset_id}" \ @@ -226,6 +229,7 @@ import sys ( quality_path, + detection_run_path, output_path, model_request, selected_model_asset_id, @@ -241,12 +245,18 @@ import sys detection_count, export_id, run_log, -) = sys.argv[1:17] +) = sys.argv[1:18] with open(quality_path, "r", encoding="utf-8") as handle: payload = json.load(handle) if "data" not in payload: raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope") +with open(detection_run_path, "r", encoding="utf-8") as handle: + detection_run_payload = json.load(handle) +if "data" not in detection_run_payload: + raise SystemExit("Detection run detail is not a canonical GeoIntel data envelope") +detection_run = detection_run_payload["data"] +run_result = detection_run.get("result_json") or {} items = payload["data"].get("items") or [] quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None) if quality_check is None: @@ -270,6 +280,9 @@ summary = { "analysis_run_id": analysis_run_id, "quality_check_id": quality_check_id, "detection_count": int(detection_count), + "raw_detection_count": run_result.get("raw_detection_count", int(detection_count)), + "suppressed_detection_count": run_result.get("suppressed_detection_count", 0), + "duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"), "quality_status": quality_check.get("status"), "quality_score": quality_check.get("score"), "precision": metrics.get("precision"), @@ -285,13 +298,15 @@ summary = { with open(output_path, "w", encoding="utf-8") as handle: json.dump(summary, handle, indent=2, sort_keys=True) print( - "model={model} tile={tile} overlap={overlap} threshold={threshold} detections={detections} " + "model={model} tile={tile} overlap={overlap} threshold={threshold} detections={detections} raw={raw} suppressed={suppressed} " "score={score} precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format( model=summary["model_asset_id"], tile=summary["tile_size"], overlap=summary["tile_overlap"], threshold=summary["threshold"], detections=summary["detection_count"], + raw=summary["raw_detection_count"], + suppressed=summary["suppressed_detection_count"], score=summary["quality_score"], precision=summary["precision"], recall=summary["recall"], @@ -346,10 +361,10 @@ with open(summary_path, "w", encoding="utf-8") as handle: print("") print("Detection quality matrix summary") -print("model\ttile\toverlap\tthreshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn") +print("model\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn") for item in items: print( - "{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format( + "{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format( **item ) ) diff --git a/scripts/run_multi_sample_detection_quality_matrix.sh b/scripts/run_multi_sample_detection_quality_matrix.sh index 0db4ca12..138b4150 100644 --- a/scripts/run_multi_sample_detection_quality_matrix.sh +++ b/scripts/run_multi_sample_detection_quality_matrix.sh @@ -205,10 +205,10 @@ summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding= print("") print("Multi-sample detection quality summary") -print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn") +print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn") for item in flat_items: print( - "{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format( + "{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format( **item ) )