Suppress duplicate YOLO tile detections
This commit is contained in:
@@ -16,6 +16,7 @@ YOLO_DEVICE=cpu
|
|||||||
YOLO_IMAGE_SIZE=640
|
YOLO_IMAGE_SIZE=640
|
||||||
YOLO_MAX_TILES=100
|
YOLO_MAX_TILES=100
|
||||||
YOLO_MAX_DETECTIONS=1000
|
YOLO_MAX_DETECTIONS=1000
|
||||||
|
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
|
||||||
YOLO_BATCH_SIZE=1
|
YOLO_BATCH_SIZE=1
|
||||||
ENABLE_GRB_WFS=false
|
ENABLE_GRB_WFS=false
|
||||||
GRB_WFS_URL=
|
GRB_WFS_URL=
|
||||||
|
|||||||
@@ -7,6 +7,15 @@
|
|||||||
|
|
||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## Sprint 149 YOLO duplicate suppression evidence (2026-07-09)
|
||||||
|
|
||||||
|
- Added configured-YOLO cross-tile duplicate suppression before `Detection` rows are persisted.
|
||||||
|
- Added `YOLO_DUPLICATE_IOU_THRESHOLD` with default `0.5`; `0` disables the GeoIntel-side pass for debugging.
|
||||||
|
- Detection run result metadata now records raw candidate count, suppressed duplicate count and duplicate IoU threshold.
|
||||||
|
- Calibration and quality matrix scripts now fetch detection run details and include raw/suppressed counts in summaries.
|
||||||
|
- Updated Docker/Unraid env examples, API/AI/backend docs and detection pipeline notes.
|
||||||
|
- No model was activated, no detections were faked, and no migration changed.
|
||||||
|
|
||||||
## Sprint 148 YOLO max-detection cap hardening (2026-07-09)
|
## Sprint 148 YOLO max-detection cap hardening (2026-07-09)
|
||||||
|
|
||||||
- Added `YOLO_MAX_DETECTIONS` with default `1000` and forward it to Ultralytics as `max_det`.
|
- Added `YOLO_MAX_DETECTIONS` with default `1000` and forward it to Ultralytics as `max_det`.
|
||||||
|
|||||||
+10
-1
@@ -360,6 +360,7 @@ YOLO_DEVICE=cpu
|
|||||||
YOLO_IMAGE_SIZE=640
|
YOLO_IMAGE_SIZE=640
|
||||||
YOLO_MAX_TILES=100
|
YOLO_MAX_TILES=100
|
||||||
YOLO_MAX_DETECTIONS=1000
|
YOLO_MAX_DETECTIONS=1000
|
||||||
|
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
|
||||||
YOLO_BATCH_SIZE=1
|
YOLO_BATCH_SIZE=1
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -389,6 +390,13 @@ The preflight checks configuration, dependency availability, local model file ex
|
|||||||
`1000` because dense building AOIs can exceed the upstream default cap of 300
|
`1000` because dense building AOIs can exceed the upstream default cap of 300
|
||||||
detections before QA/QC can measure recall honestly.
|
detections before QA/QC can measure recall honestly.
|
||||||
|
|
||||||
|
`YOLO_DUPLICATE_IOU_THRESHOLD` controls GeoIntel-side cross-tile duplicate
|
||||||
|
suppression after YOLO pixel boxes are converted to EPSG:4326 polygons and
|
||||||
|
before `Detection` rows are persisted. Candidates are sorted by confidence per
|
||||||
|
class; lower-confidence same-class candidates with geometry IoU greater than or
|
||||||
|
equal to the threshold are suppressed. The default is `0.5`; set `0` to disable
|
||||||
|
this post-processing for debugging.
|
||||||
|
|
||||||
The same read-only status is available through the API and Detection Lab UI:
|
The same read-only status is available through the API and Detection Lab UI:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -458,7 +466,8 @@ bash scripts/run_detection_calibration_sweep.sh http://192.168.10.150:1202
|
|||||||
|
|
||||||
The sweep creates one real persisted workflow run per threshold, fetches the
|
The sweep creates one real persisted workflow run per threshold, fetches the
|
||||||
persisted `QualityCheck`/`Metric` rows and writes a `calibration_summary.json`
|
persisted `QualityCheck`/`Metric` rows and writes a `calibration_summary.json`
|
||||||
with detection count, score, precision, recall, F1, mean IoU and false
|
with persisted detection count, raw candidate count, suppressed duplicate count,
|
||||||
|
duplicate IoU threshold, score, precision, recall, F1, mean IoU and false
|
||||||
positive/negative counts. It is intended to tune confidence/IoU/model choices,
|
positive/negative counts. It is intended to tune confidence/IoU/model choices,
|
||||||
not to add new inference behavior.
|
not to add new inference behavior.
|
||||||
|
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ class Settings(BaseSettings):
|
|||||||
yolo_image_size: int = Field(default=640, validation_alias="YOLO_IMAGE_SIZE")
|
yolo_image_size: int = Field(default=640, validation_alias="YOLO_IMAGE_SIZE")
|
||||||
yolo_max_tiles: int = Field(default=100, validation_alias="YOLO_MAX_TILES")
|
yolo_max_tiles: int = Field(default=100, validation_alias="YOLO_MAX_TILES")
|
||||||
yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS")
|
yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS")
|
||||||
|
yolo_duplicate_iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0, validation_alias="YOLO_DUPLICATE_IOU_THRESHOLD")
|
||||||
yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE")
|
yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE")
|
||||||
cors_origins: list[str] | str = Field(
|
cors_origins: list[str] | str = Field(
|
||||||
default=["http://localhost:5173", "http://127.0.0.1:5173"],
|
default=["http://localhost:5173", "http://127.0.0.1:5173"],
|
||||||
|
|||||||
@@ -138,7 +138,7 @@ class DetectionService:
|
|||||||
|
|
||||||
if model.model_id == resolved_settings.yolo_model_id:
|
if model.model_id == resolved_settings.yolo_model_id:
|
||||||
try:
|
try:
|
||||||
detections = DetectionService._run_configured_yolo(
|
detections, postprocess_summary = DetectionService._run_configured_yolo(
|
||||||
db=db,
|
db=db,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
@@ -165,7 +165,7 @@ class DetectionService:
|
|||||||
error_code=exc.code,
|
error_code=exc.code,
|
||||||
message=exc.message,
|
message=exc.message,
|
||||||
)
|
)
|
||||||
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections))
|
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections), extra_result=postprocess_summary)
|
||||||
return DetectionRunResponse(
|
return DetectionRunResponse(
|
||||||
analysis_run_id=analysis_run.id,
|
analysis_run_id=analysis_run.id,
|
||||||
job_id=job.id,
|
job_id=job.id,
|
||||||
@@ -454,8 +454,10 @@ class DetectionService:
|
|||||||
db.refresh(job)
|
db.refresh(job)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _mark_success(db, analysis_run: AnalysisRun, job: Job, detection_count: int) -> None:
|
def _mark_success(db, analysis_run: AnalysisRun, job: Job, detection_count: int, extra_result: dict[str, Any] | None = None) -> None:
|
||||||
result = {"detection_count": detection_count}
|
result = {"detection_count": detection_count}
|
||||||
|
if extra_result:
|
||||||
|
result.update(extra_result)
|
||||||
analysis_run.status = "success"
|
analysis_run.status = "success"
|
||||||
analysis_run.finished_at = DetectionService._now()
|
analysis_run.finished_at = DetectionService._now()
|
||||||
analysis_run.result_json = result
|
analysis_run.result_json = result
|
||||||
@@ -536,13 +538,13 @@ class DetectionService:
|
|||||||
class_filter: list[str],
|
class_filter: list[str],
|
||||||
settings: Settings,
|
settings: Settings,
|
||||||
yolo_adapter_class: Type[YoloDetectionAdapter],
|
yolo_adapter_class: Type[YoloDetectionAdapter],
|
||||||
) -> list[Detection]:
|
) -> tuple[list[Detection], dict[str, Any]]:
|
||||||
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
|
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
|
||||||
model_path = Path(settings.yolo_model_path or "").expanduser()
|
model_path = Path(settings.yolo_model_path or "").expanduser()
|
||||||
adapter = yolo_adapter_class(settings)
|
adapter = yolo_adapter_class(settings)
|
||||||
model = adapter.load_model(model_path)
|
model = adapter.load_model(model_path)
|
||||||
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
|
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
|
||||||
persisted: list[Detection] = []
|
candidates: list[dict[str, Any]] = []
|
||||||
manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") or "EPSG:4326"
|
manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") or "EPSG:4326"
|
||||||
for tile in manifest["tiles"]:
|
for tile in manifest["tiles"]:
|
||||||
tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser())
|
tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser())
|
||||||
@@ -561,37 +563,87 @@ class DetectionService:
|
|||||||
properties = dict(raw.get("properties") or {})
|
properties = dict(raw.get("properties") or {})
|
||||||
if model_class_name and model_class_name != class_name:
|
if model_class_name and model_class_name != class_name:
|
||||||
properties.setdefault("model_class_name", model_class_name)
|
properties.setdefault("model_class_name", model_class_name)
|
||||||
detection = Detection(
|
candidates.append(
|
||||||
id=uuid.uuid4(),
|
{
|
||||||
project_id=project_id,
|
"class_name": class_name,
|
||||||
dataset_id=dataset_id,
|
"confidence": confidence,
|
||||||
analysis_run_id=analysis_run.id,
|
"geometry": geometry,
|
||||||
job_id=job.id,
|
"bbox": bbox,
|
||||||
model_name=model_name,
|
"source_tile_path": str(tile_path),
|
||||||
model_version=model_version,
|
"properties": {**properties, "tile_index": tile.get("index")},
|
||||||
class_name=class_name,
|
}
|
||||||
confidence=confidence,
|
|
||||||
geometry=from_shape(geometry, srid=4326),
|
|
||||||
bbox_json={
|
|
||||||
"x_min": float(bbox[0]),
|
|
||||||
"y_min": float(bbox[1]),
|
|
||||||
"x_max": float(bbox[2]),
|
|
||||||
"y_max": float(bbox[3]),
|
|
||||||
},
|
|
||||||
source_tile_path=str(tile_path),
|
|
||||||
properties_json={**properties, "tile_index": tile.get("index")},
|
|
||||||
)
|
)
|
||||||
db.add(detection)
|
filtered_candidates = DetectionService._suppress_duplicate_candidates(
|
||||||
persisted.append(detection)
|
candidates,
|
||||||
|
iou_threshold=float(settings.yolo_duplicate_iou_threshold),
|
||||||
|
)
|
||||||
|
persisted: list[Detection] = []
|
||||||
|
for candidate in filtered_candidates:
|
||||||
|
bbox = candidate["bbox"]
|
||||||
|
detection = Detection(
|
||||||
|
id=uuid.uuid4(),
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
|
analysis_run_id=analysis_run.id,
|
||||||
|
job_id=job.id,
|
||||||
|
model_name=model_name,
|
||||||
|
model_version=model_version,
|
||||||
|
class_name=candidate["class_name"],
|
||||||
|
confidence=candidate["confidence"],
|
||||||
|
geometry=from_shape(candidate["geometry"], srid=4326),
|
||||||
|
bbox_json={
|
||||||
|
"x_min": float(bbox[0]),
|
||||||
|
"y_min": float(bbox[1]),
|
||||||
|
"x_max": float(bbox[2]),
|
||||||
|
"y_max": float(bbox[3]),
|
||||||
|
},
|
||||||
|
source_tile_path=candidate["source_tile_path"],
|
||||||
|
properties_json=candidate["properties"],
|
||||||
|
)
|
||||||
|
db.add(detection)
|
||||||
|
persisted.append(detection)
|
||||||
db.commit()
|
db.commit()
|
||||||
for detection in persisted:
|
for detection in persisted:
|
||||||
db.refresh(detection)
|
db.refresh(detection)
|
||||||
return persisted
|
return persisted, {
|
||||||
|
"raw_detection_count": len(candidates),
|
||||||
|
"suppressed_detection_count": len(candidates) - len(filtered_candidates),
|
||||||
|
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
|
||||||
|
}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _canonical_class_name(value: Any) -> str:
|
def _canonical_class_name(value: Any) -> str:
|
||||||
return str(value or "").strip().casefold()
|
return str(value or "").strip().casefold()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _suppress_duplicate_candidates(candidates: list[dict[str, Any]], iou_threshold: float) -> list[dict[str, Any]]:
|
||||||
|
if iou_threshold <= 0 or len(candidates) < 2:
|
||||||
|
return candidates
|
||||||
|
kept: list[dict[str, Any]] = []
|
||||||
|
for candidate in sorted(candidates, key=lambda item: float(item["confidence"]), reverse=True):
|
||||||
|
duplicate = False
|
||||||
|
for kept_candidate in kept:
|
||||||
|
if candidate["class_name"] != kept_candidate["class_name"]:
|
||||||
|
continue
|
||||||
|
if DetectionService._geometry_iou(candidate["geometry"], kept_candidate["geometry"]) >= iou_threshold:
|
||||||
|
duplicate = True
|
||||||
|
break
|
||||||
|
if not duplicate:
|
||||||
|
kept.append(candidate)
|
||||||
|
return kept
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _geometry_iou(left, right) -> float:
|
||||||
|
if left.is_empty or right.is_empty:
|
||||||
|
return 0.0
|
||||||
|
intersection_area = left.intersection(right).area
|
||||||
|
if intersection_area <= 0:
|
||||||
|
return 0.0
|
||||||
|
union_area = left.union(right).area
|
||||||
|
if union_area <= 0:
|
||||||
|
return 0.0
|
||||||
|
return intersection_area / union_area
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int) -> dict[str, Any]:
|
def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int) -> dict[str, Any]:
|
||||||
if not tile_manifest_path:
|
if not tile_manifest_path:
|
||||||
|
|||||||
@@ -94,6 +94,7 @@ def test_env_example_uses_runtime_env_names_read_by_backend_and_frontend() -> No
|
|||||||
assert "YOLO_CONFIG_DIR=./storage/ultralytics" in env_example
|
assert "YOLO_CONFIG_DIR=./storage/ultralytics" in env_example
|
||||||
assert "YOLO_MAX_TILES=100" in env_example
|
assert "YOLO_MAX_TILES=100" in env_example
|
||||||
assert "YOLO_MAX_DETECTIONS=1000" in env_example
|
assert "YOLO_MAX_DETECTIONS=1000" in env_example
|
||||||
|
assert "YOLO_DUPLICATE_IOU_THRESHOLD=0.5" in env_example
|
||||||
assert "ENABLE_YOLO" not in env_example
|
assert "ENABLE_YOLO" not in env_example
|
||||||
assert "ENABLE_SAM" not in env_example
|
assert "ENABLE_SAM" not in env_example
|
||||||
assert "VITE_API_BASE_URL=" in env_example
|
assert "VITE_API_BASE_URL=" in env_example
|
||||||
@@ -247,4 +248,5 @@ def test_unraid_deploy_passes_ai_build_arg_and_yolo_runtime_env() -> None:
|
|||||||
assert '-e YOLO_MODEL_PATH="$YOLO_MODEL_PATH"' in run_script
|
assert '-e YOLO_MODEL_PATH="$YOLO_MODEL_PATH"' in run_script
|
||||||
assert '-e YOLO_MAX_TILES="$YOLO_MAX_TILES"' in run_script
|
assert '-e YOLO_MAX_TILES="$YOLO_MAX_TILES"' in run_script
|
||||||
assert '-e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS"' in run_script
|
assert '-e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS"' in run_script
|
||||||
|
assert '-e YOLO_DUPLICATE_IOU_THRESHOLD="$YOLO_DUPLICATE_IOU_THRESHOLD"' in run_script
|
||||||
assert "-v \"${GEOINTEL_MODELS_PATH}:/app/models\"" in run_script
|
assert "-v \"${GEOINTEL_MODELS_PATH}:/app/models\"" in run_script
|
||||||
|
|||||||
@@ -18,7 +18,11 @@ def test_detection_calibration_sweep_reuses_real_data_workflow_and_reports_qa_me
|
|||||||
assert "REAL_RASTER_PATH" in script
|
assert "REAL_RASTER_PATH" in script
|
||||||
assert "REAL_REFERENCE_VECTOR_PATH" in script
|
assert "REAL_REFERENCE_VECTOR_PATH" in script
|
||||||
assert "/api/v1/projects/${project_id}/quality-checks" in script
|
assert "/api/v1/projects/${project_id}/quality-checks" in script
|
||||||
|
assert "/api/v1/detection/runs/${analysis_run_id}" in script
|
||||||
assert "quality_check_id" in script
|
assert "quality_check_id" in script
|
||||||
|
assert "raw_detection_count" in script
|
||||||
|
assert "suppressed_detection_count" in script
|
||||||
|
assert "duplicate_iou_threshold" in script
|
||||||
assert "false_positives" in script
|
assert "false_positives" in script
|
||||||
assert "false_negatives" in script
|
assert "false_negatives" in script
|
||||||
assert "quality_score" in script
|
assert "quality_score" in script
|
||||||
|
|||||||
@@ -24,7 +24,11 @@ def test_detection_quality_matrix_compares_models_tiles_and_thresholds() -> None
|
|||||||
assert "REAL_RASTER_PATH" in script
|
assert "REAL_RASTER_PATH" in script
|
||||||
assert "REAL_REFERENCE_VECTOR_PATH" in script
|
assert "REAL_REFERENCE_VECTOR_PATH" in script
|
||||||
assert "/api/v1/projects/${project_id}/quality-checks" in script
|
assert "/api/v1/projects/${project_id}/quality-checks" in script
|
||||||
|
assert "/api/v1/detection/runs/${analysis_run_id}" in script
|
||||||
assert "quality_matrix_summary.json" in script
|
assert "quality_matrix_summary.json" in script
|
||||||
|
assert "raw_detection_count" in script
|
||||||
|
assert "suppressed_detection_count" in script
|
||||||
|
assert "duplicate_iou_threshold" in script
|
||||||
assert "best_by_score" in script
|
assert "best_by_score" in script
|
||||||
assert "best_by_recall" in script
|
assert "best_by_recall" in script
|
||||||
assert "best_by_precision" in script
|
assert "best_by_precision" in script
|
||||||
|
|||||||
@@ -66,6 +66,8 @@ def test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sam
|
|||||||
assert "best_overall_by_score" in script
|
assert "best_overall_by_score" in script
|
||||||
assert "best_by_sample" in script
|
assert "best_by_sample" in script
|
||||||
assert "sample_slug" in script
|
assert "sample_slug" in script
|
||||||
|
assert "raw_detection_count" in script
|
||||||
|
assert "suppressed_detection_count" in script
|
||||||
assert "demo/workflow" not in script
|
assert "demo/workflow" not in script
|
||||||
assert "fixture_mode" not in script
|
assert "fixture_mode" not in script
|
||||||
assert "will_download_models" not in script
|
assert "will_download_models" not in script
|
||||||
|
|||||||
@@ -89,6 +89,25 @@ class MixedCaseYoloAdapter(MockYoloAdapter):
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class OverlappingTileYoloAdapter(MockYoloAdapter):
|
||||||
|
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
|
||||||
|
tile_index = int(tile_path.stem.split("_")[-1])
|
||||||
|
if tile_index == 0:
|
||||||
|
bbox = [10.0, 20.0, 30.0, 40.0]
|
||||||
|
confidence = 0.82
|
||||||
|
else:
|
||||||
|
bbox = [11.0, 21.0, 31.0, 41.0]
|
||||||
|
confidence = 0.91
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"class_name": "building",
|
||||||
|
"confidence": confidence,
|
||||||
|
"bbox": bbox,
|
||||||
|
"properties": {"adapter": "overlap"},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
class RecordingPredictModel:
|
class RecordingPredictModel:
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self.seen_sources: list[dict] = []
|
self.seen_sources: list[dict] = []
|
||||||
@@ -383,6 +402,37 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_
|
|||||||
assert detections[0].properties_json["model_class_name"] == "Building"
|
assert detections[0].properties_json["model_class_name"] == "Building"
|
||||||
|
|
||||||
|
|
||||||
|
def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) -> None:
|
||||||
|
db, project_id, dataset_id = _project_and_dataset()
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"local weights")
|
||||||
|
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5)
|
||||||
|
manifest_path = _manifest(tmp_path, tile_count=2)
|
||||||
|
|
||||||
|
result = DetectionService.run_detection(
|
||||||
|
db=db,
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
confidence_threshold=0.5,
|
||||||
|
class_filter=["building"],
|
||||||
|
tile_manifest_path=str(manifest_path),
|
||||||
|
settings=settings,
|
||||||
|
yolo_adapter_class=OverlappingTileYoloAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
detections = [item for item in db.added if isinstance(item, Detection)]
|
||||||
|
runs = [item for item in db.added if isinstance(item, AnalysisRun)]
|
||||||
|
|
||||||
|
assert result.status == "success"
|
||||||
|
assert result.detection_count == 1
|
||||||
|
assert detections[0].confidence == 0.91
|
||||||
|
assert detections[0].source_tile_path.endswith("tile_0001.tif")
|
||||||
|
assert runs[0].result_json["raw_detection_count"] == 2
|
||||||
|
assert runs[0].result_json["suppressed_detection_count"] == 1
|
||||||
|
assert runs[0].result_json["duplicate_iou_threshold"] == 0.5
|
||||||
|
|
||||||
|
|
||||||
def test_yolo_adapter_converts_single_band_tiles_to_rgb_before_prediction(tmp_path: Path) -> None:
|
def test_yolo_adapter_converts_single_band_tiles_to_rgb_before_prediction(tmp_path: Path) -> None:
|
||||||
Image = pytest.importorskip("PIL.Image")
|
Image = pytest.importorskip("PIL.Image")
|
||||||
tile_path = tmp_path / "single_band_tile.tif"
|
tile_path = tmp_path / "single_band_tile.tif"
|
||||||
|
|||||||
@@ -37,4 +37,5 @@ YOLO_DEVICE=cpu
|
|||||||
YOLO_IMAGE_SIZE=640
|
YOLO_IMAGE_SIZE=640
|
||||||
YOLO_MAX_TILES=100
|
YOLO_MAX_TILES=100
|
||||||
YOLO_MAX_DETECTIONS=1000
|
YOLO_MAX_DETECTIONS=1000
|
||||||
|
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
|
||||||
YOLO_BATCH_SIZE=1
|
YOLO_BATCH_SIZE=1
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ YOLO_DEVICE="${YOLO_DEVICE:-cpu}"
|
|||||||
YOLO_IMAGE_SIZE="${YOLO_IMAGE_SIZE:-640}"
|
YOLO_IMAGE_SIZE="${YOLO_IMAGE_SIZE:-640}"
|
||||||
YOLO_MAX_TILES="${YOLO_MAX_TILES:-100}"
|
YOLO_MAX_TILES="${YOLO_MAX_TILES:-100}"
|
||||||
YOLO_MAX_DETECTIONS="${YOLO_MAX_DETECTIONS:-1000}"
|
YOLO_MAX_DETECTIONS="${YOLO_MAX_DETECTIONS:-1000}"
|
||||||
|
YOLO_DUPLICATE_IOU_THRESHOLD="${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5}"
|
||||||
YOLO_BATCH_SIZE="${YOLO_BATCH_SIZE:-1}"
|
YOLO_BATCH_SIZE="${YOLO_BATCH_SIZE:-1}"
|
||||||
|
|
||||||
install_dockerman_metadata() {
|
install_dockerman_metadata() {
|
||||||
@@ -92,6 +93,7 @@ docker run -d \
|
|||||||
-e YOLO_IMAGE_SIZE="$YOLO_IMAGE_SIZE" \
|
-e YOLO_IMAGE_SIZE="$YOLO_IMAGE_SIZE" \
|
||||||
-e YOLO_MAX_TILES="$YOLO_MAX_TILES" \
|
-e YOLO_MAX_TILES="$YOLO_MAX_TILES" \
|
||||||
-e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS" \
|
-e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS" \
|
||||||
|
-e YOLO_DUPLICATE_IOU_THRESHOLD="$YOLO_DUPLICATE_IOU_THRESHOLD" \
|
||||||
-e YOLO_BATCH_SIZE="$YOLO_BATCH_SIZE" \
|
-e YOLO_BATCH_SIZE="$YOLO_BATCH_SIZE" \
|
||||||
-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data" \
|
-v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data" \
|
||||||
-v "${GEOINTEL_STORAGE_PATH}:/app/storage" \
|
-v "${GEOINTEL_STORAGE_PATH}:/app/storage" \
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ services:
|
|||||||
YOLO_IMAGE_SIZE: ${YOLO_IMAGE_SIZE:-640}
|
YOLO_IMAGE_SIZE: ${YOLO_IMAGE_SIZE:-640}
|
||||||
YOLO_MAX_TILES: ${YOLO_MAX_TILES:-100}
|
YOLO_MAX_TILES: ${YOLO_MAX_TILES:-100}
|
||||||
YOLO_MAX_DETECTIONS: ${YOLO_MAX_DETECTIONS:-1000}
|
YOLO_MAX_DETECTIONS: ${YOLO_MAX_DETECTIONS:-1000}
|
||||||
|
YOLO_DUPLICATE_IOU_THRESHOLD: ${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5}
|
||||||
YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1}
|
YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1}
|
||||||
ports:
|
ports:
|
||||||
- "${GEOINTEL_FRONTEND_PORT:-1202}:80"
|
- "${GEOINTEL_FRONTEND_PORT:-1202}:80"
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ services:
|
|||||||
YOLO_IMAGE_SIZE: ${YOLO_IMAGE_SIZE:-640}
|
YOLO_IMAGE_SIZE: ${YOLO_IMAGE_SIZE:-640}
|
||||||
YOLO_MAX_TILES: ${YOLO_MAX_TILES:-100}
|
YOLO_MAX_TILES: ${YOLO_MAX_TILES:-100}
|
||||||
YOLO_MAX_DETECTIONS: ${YOLO_MAX_DETECTIONS:-1000}
|
YOLO_MAX_DETECTIONS: ${YOLO_MAX_DETECTIONS:-1000}
|
||||||
|
YOLO_DUPLICATE_IOU_THRESHOLD: ${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5}
|
||||||
YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1}
|
YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1}
|
||||||
ports:
|
ports:
|
||||||
- "${GEOINTEL_BACKEND_PORT:-8000}:8000"
|
- "${GEOINTEL_BACKEND_PORT:-8000}:8000"
|
||||||
|
|||||||
@@ -110,6 +110,7 @@ Environment variables:
|
|||||||
- `YOLO_IMAGE_SIZE`
|
- `YOLO_IMAGE_SIZE`
|
||||||
- `YOLO_MAX_TILES`
|
- `YOLO_MAX_TILES`
|
||||||
- `YOLO_MAX_DETECTIONS`
|
- `YOLO_MAX_DETECTIONS`
|
||||||
|
- `YOLO_DUPLICATE_IOU_THRESHOLD`
|
||||||
- `YOLO_BATCH_SIZE`
|
- `YOLO_BATCH_SIZE`
|
||||||
|
|
||||||
`YOLO_MAX_DETECTIONS` is forwarded to Ultralytics as `max_det` for each
|
`YOLO_MAX_DETECTIONS` is forwarded to Ultralytics as `max_det` for each
|
||||||
@@ -119,6 +120,15 @@ the upstream default would cap recall before QA/QC begins. Operators may lower
|
|||||||
the value for small rasters or raise it for dense urban tiles after reviewing
|
the value for small rasters or raise it for dense urban tiles after reviewing
|
||||||
runtime and false-positive behavior.
|
runtime and false-positive behavior.
|
||||||
|
|
||||||
|
After YOLO boxes are georeferenced, configured-YOLO runs apply a GeoIntel
|
||||||
|
cross-tile duplicate suppression pass before persistence. Candidates are grouped
|
||||||
|
by canonical class and sorted by confidence; lower-confidence same-class
|
||||||
|
candidates with EPSG:4326 geometry IoU greater than or equal to
|
||||||
|
`YOLO_DUPLICATE_IOU_THRESHOLD` are suppressed. The default is `0.5`; set it to
|
||||||
|
`0` to disable this post-processing for operator debugging. Run summaries record
|
||||||
|
raw, persisted and suppressed detection counts so calibration evidence remains
|
||||||
|
auditable.
|
||||||
|
|
||||||
### Local model asset catalog
|
### Local model asset catalog
|
||||||
|
|
||||||
GeoIntel can list local runtime model files mounted into the backend model
|
GeoIntel can list local runtime model files mounted into the backend model
|
||||||
@@ -205,6 +215,9 @@ persisted project quality-check list to build `calibration_summary.json`.
|
|||||||
Results are honest QA/QC evidence from persisted detections and persisted
|
Results are honest QA/QC evidence from persisted detections and persisted
|
||||||
reference `vector_features`; no demo detections, live provider fetches or model
|
reference `vector_features`; no demo detections, live provider fetches or model
|
||||||
downloads are introduced by the calibration tool.
|
downloads are introduced by the calibration tool.
|
||||||
|
Summaries include raw detection candidate count, persisted detection count and
|
||||||
|
suppressed duplicate count so operators can distinguish model output volume from
|
||||||
|
GeoIntel post-processing.
|
||||||
|
|
||||||
For model/tile/threshold selection, use the quality matrix wrapper:
|
For model/tile/threshold selection, use the quality matrix wrapper:
|
||||||
|
|
||||||
|
|||||||
@@ -785,6 +785,12 @@ Validation errors:
|
|||||||
- Configured YOLO inference forwards `YOLO_MAX_DETECTIONS` to Ultralytics
|
- Configured YOLO inference forwards `YOLO_MAX_DETECTIONS` to Ultralytics
|
||||||
`max_det` and defaults to `1000` so dense building AOIs are not silently
|
`max_det` and defaults to `1000` so dense building AOIs are not silently
|
||||||
limited by the upstream default of 300 detections before persisted QA/QC.
|
limited by the upstream default of 300 detections before persisted QA/QC.
|
||||||
|
- Configured YOLO applies cross-tile duplicate suppression after pixel boxes are
|
||||||
|
converted to EPSG:4326 geometries and before `Detection` rows are persisted.
|
||||||
|
Same-class candidates are confidence-sorted and lower-confidence candidates
|
||||||
|
with geometry IoU greater than or equal to
|
||||||
|
`YOLO_DUPLICATE_IOU_THRESHOLD` are suppressed. The default is `0.5`; `0`
|
||||||
|
disables this GeoIntel-side post-processing for debugging.
|
||||||
- `DETECTION_DEPENDENCY_UNAVAILABLE` when YOLO dependencies are not installed.
|
- `DETECTION_DEPENDENCY_UNAVAILABLE` when YOLO dependencies are not installed.
|
||||||
- `DETECTION_MODEL_LOAD_FAILED` when the local model file exists but cannot be loaded.
|
- `DETECTION_MODEL_LOAD_FAILED` when the local model file exists but cannot be loaded.
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,35 @@
|
|||||||
|
## Sprint 149 YOLO duplicate suppression evidence (2026-07-09)
|
||||||
|
|
||||||
|
Changed:
|
||||||
|
- Added configured-YOLO cross-tile duplicate suppression after pixel boxes are converted to EPSG:4326 geometries and before `Detection` rows are persisted.
|
||||||
|
- Added backend setting `YOLO_DUPLICATE_IOU_THRESHOLD` with default `0.5`; `0` disables the GeoIntel-side pass for debugging.
|
||||||
|
- Detection run `result_json` now records:
|
||||||
|
- `raw_detection_count`
|
||||||
|
- `suppressed_detection_count`
|
||||||
|
- `duplicate_iou_threshold`
|
||||||
|
- Updated `.env.example`, Docker Compose, Unraid env examples and the Dockerman run script.
|
||||||
|
- Updated calibration and quality matrix scripts to fetch detection run details and include raw/suppressed counts in per-run and aggregate summaries.
|
||||||
|
- Updated backend/API/AI/pipeline documentation.
|
||||||
|
|
||||||
|
Why:
|
||||||
|
- Dense overlapping tile inference can produce duplicate candidate buildings, which inflates persisted false positives before QA/QC.
|
||||||
|
- The previous Sprint 148 cap fix allowed dense AOIs to persist more candidates, but made duplicate pressure more visible.
|
||||||
|
- This pass keeps all outputs honest: no model activation, no fake detections and no migration. It only removes lower-confidence same-class geometric duplicates before persistence.
|
||||||
|
|
||||||
|
Tested:
|
||||||
|
- Red step: `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py::test_yolo_run_suppresses_cross_tile_duplicate_detections -q` failed with `detection_count == 2`.
|
||||||
|
- `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py::test_yolo_run_suppresses_cross_tile_duplicate_detections -q` (`1 passed`)
|
||||||
|
- Red step: Docker runtime config tests failed before `.env.example` and Unraid runner exposed `YOLO_DUPLICATE_IOU_THRESHOLD`.
|
||||||
|
- `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py backend\tests\test_docker_runtime_config.py::test_env_example_uses_runtime_env_names_read_by_backend_and_frontend backend\tests\test_docker_runtime_config.py::test_unraid_deploy_passes_ai_build_arg_and_yolo_runtime_env -q` (`17 passed`)
|
||||||
|
- Red step: calibration/matrix script tests failed before detection-run raw/suppressed metadata was included.
|
||||||
|
- `python -m pytest backend\tests\test_sprint124_detection_calibration_sweep.py backend\tests\test_sprint126_detection_quality_matrix.py backend\tests\test_sprint127_operator_sample_quality_matrix.py::test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sample -q` (`3 passed`)
|
||||||
|
- `bash -n scripts/run_detection_calibration_sweep.sh`
|
||||||
|
- `bash -n scripts/run_detection_quality_matrix.sh`
|
||||||
|
- `bash -n scripts/run_multi_sample_detection_quality_matrix.sh`
|
||||||
|
|
||||||
|
Next:
|
||||||
|
- Run full readiness, then deploy Tower and rerun the dense Westerlo/Turnhout sweeps with duplicate suppression enabled to quantify quality impact versus the Sprint 148 uncapped baseline.
|
||||||
|
|
||||||
## Sprint 148 YOLO max-detection cap hardening (2026-07-09)
|
## Sprint 148 YOLO max-detection cap hardening (2026-07-09)
|
||||||
|
|
||||||
Changed:
|
Changed:
|
||||||
|
|||||||
@@ -110,6 +110,9 @@ Default method:
|
|||||||
- calculate IoU between overlapping geospatial bboxes
|
- calculate IoU between overlapping geospatial bboxes
|
||||||
- apply non-maximum suppression by confidence
|
- apply non-maximum suppression by confidence
|
||||||
- default IoU merge threshold: 0.5
|
- default IoU merge threshold: 0.5
|
||||||
|
- implemented for configured-YOLO as EPSG:4326 cross-tile duplicate suppression
|
||||||
|
before `Detection` persistence; `YOLO_DUPLICATE_IOU_THRESHOLD=0` disables the
|
||||||
|
GeoIntel-side pass for debugging.
|
||||||
|
|
||||||
## Step 6 — Storage
|
## Step 6 — Storage
|
||||||
|
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ YOLO_MODEL_PATH=
|
|||||||
YOLO_MODEL_VERSION=
|
YOLO_MODEL_VERSION=
|
||||||
YOLO_MAX_TILES=100
|
YOLO_MAX_TILES=100
|
||||||
YOLO_MAX_DETECTIONS=1000
|
YOLO_MAX_DETECTIONS=1000
|
||||||
|
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
|
||||||
ENABLE_GRB_WFS=false
|
ENABLE_GRB_WFS=false
|
||||||
GRB_WFS_URL=
|
GRB_WFS_URL=
|
||||||
OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter
|
OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter
|
||||||
|
|||||||
+2
-1
@@ -112,10 +112,11 @@ This file now starts with the current implementation status. Older preparation/b
|
|||||||
- [x] Train and gate an AOI-scale `aoi512e80` YOLOv8s candidate to test the 160px training-scale hypothesis.
|
- [x] Train and gate an AOI-scale `aoi512e80` YOLOv8s candidate to test the 160px training-scale hypothesis.
|
||||||
- [x] Raise configured-YOLO `max_det` through `YOLO_MAX_DETECTIONS` so dense AOIs are not capped at 300 detections before QA/QC.
|
- [x] Raise configured-YOLO `max_det` through `YOLO_MAX_DETECTIONS` so dense AOIs are not capped at 300 detections before QA/QC.
|
||||||
- [x] Rerun live dense-AOI calibration after redeploy with `YOLO_MAX_DETECTIONS=1000`; Westerlo reached 523/1000 detections at lower thresholds and Turnhout reached 822/1000, confirming the old 300 cap is removed.
|
- [x] Rerun live dense-AOI calibration after redeploy with `YOLO_MAX_DETECTIONS=1000`; Westerlo reached 523/1000 detections at lower thresholds and Turnhout reached 822/1000, confirming the old 300 cap is removed.
|
||||||
|
- [x] Add configured-YOLO cross-tile duplicate suppression and raw/suppressed calibration evidence fields.
|
||||||
- [ ] Find or train a materially stronger aerial/Kempen building model candidate; `geointel-building-yolov8n-expanded160e50-pt` is the best current dense-AOI candidate but still too weak and too noisy for a V1 default.
|
- [ ] Find or train a materially stronger aerial/Kempen building model candidate; `geointel-building-yolov8n-expanded160e50-pt` is the best current dense-AOI candidate but still too weak and too noisy for a V1 default.
|
||||||
- [ ] Train a higher-capacity local aerial-building detector with stronger positive recall while preserving the hard-negative false-positive gate.
|
- [ ] Train a higher-capacity local aerial-building detector with stronger positive recall while preserving the hard-negative false-positive gate.
|
||||||
- [ ] Add more diverse positive AOIs and revisit geometry-to-box label strategy before the next default-model training attempt.
|
- [ ] Add more diverse positive AOIs and revisit geometry-to-box label strategy before the next default-model training attempt.
|
||||||
- [ ] Add operator-side duplicate suppression/post-processing analysis for dense overlapping tile detections before the next promotion gate.
|
- [ ] Rerun live dense-AOI calibration after redeploy with `YOLO_DUPLICATE_IOU_THRESHOLD=0.5` to measure post-processing impact versus the Sprint 148 uncapped baseline.
|
||||||
|
|
||||||
## Sprint 8 status
|
## Sprint 8 status
|
||||||
|
|
||||||
|
|||||||
@@ -226,6 +226,9 @@ logs plus `calibration_summary.json` under
|
|||||||
`CALIBRATION_OUTPUT_DIR` is set. This is a calibration/benchmarking tool only:
|
`CALIBRATION_OUTPUT_DIR` is set. This is a calibration/benchmarking tool only:
|
||||||
it does not seed demo data, enable fixture detections, fetch external data or
|
it does not seed demo data, enable fixture detections, fetch external data or
|
||||||
download model weights.
|
download model weights.
|
||||||
|
Per-threshold summaries include persisted detection count, raw candidate count
|
||||||
|
before GeoIntel duplicate suppression, suppressed duplicate count and the
|
||||||
|
configured duplicate IoU threshold.
|
||||||
|
|
||||||
Run a broader model/tile/threshold quality matrix when multiple local model
|
Run a broader model/tile/threshold quality matrix when multiple local model
|
||||||
assets or tile settings need to be compared:
|
assets or tile settings need to be compared:
|
||||||
|
|||||||
@@ -98,6 +98,7 @@ for threshold in ${thresholds_normalized}; do
|
|||||||
threshold_label="$(printf '%s' "${threshold}" | tr '.-' 'pm')"
|
threshold_label="$(printf '%s' "${threshold}" | tr '.-' 'pm')"
|
||||||
run_log="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}.log"
|
run_log="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}.log"
|
||||||
quality_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_quality_checks.json"
|
quality_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_quality_checks.json"
|
||||||
|
detection_run_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_detection_run.json"
|
||||||
run_summary="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_summary.json"
|
run_summary="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_summary.json"
|
||||||
|
|
||||||
echo "-- Threshold ${threshold} (${run_index}) --"
|
echo "-- Threshold ${threshold} (${run_index}) --"
|
||||||
@@ -121,9 +122,11 @@ for threshold in ${thresholds_normalized}; do
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}"
|
curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}"
|
||||||
|
curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}" > "${detection_run_response}"
|
||||||
|
|
||||||
"${PYTHON_BIN}" - \
|
"${PYTHON_BIN}" - \
|
||||||
"${quality_response}" \
|
"${quality_response}" \
|
||||||
|
"${detection_run_response}" \
|
||||||
"${run_summary}" \
|
"${run_summary}" \
|
||||||
"${threshold}" \
|
"${threshold}" \
|
||||||
"${project_id}" \
|
"${project_id}" \
|
||||||
@@ -134,11 +137,17 @@ for threshold in ${thresholds_normalized}; do
|
|||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
quality_path, output_path, threshold, project_id, analysis_run_id, quality_check_id, detection_count, run_log = sys.argv[1:9]
|
quality_path, detection_run_path, output_path, threshold, project_id, analysis_run_id, quality_check_id, detection_count, run_log = sys.argv[1:10]
|
||||||
with open(quality_path, "r", encoding="utf-8") as handle:
|
with open(quality_path, "r", encoding="utf-8") as handle:
|
||||||
payload = json.load(handle)
|
payload = json.load(handle)
|
||||||
if "data" not in payload:
|
if "data" not in payload:
|
||||||
raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope")
|
raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope")
|
||||||
|
with open(detection_run_path, "r", encoding="utf-8") as handle:
|
||||||
|
detection_run_payload = json.load(handle)
|
||||||
|
if "data" not in detection_run_payload:
|
||||||
|
raise SystemExit("Detection run detail is not a canonical GeoIntel data envelope")
|
||||||
|
detection_run = detection_run_payload["data"]
|
||||||
|
run_result = detection_run.get("result_json") or {}
|
||||||
items = payload["data"].get("items") or []
|
items = payload["data"].get("items") or []
|
||||||
quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None)
|
quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None)
|
||||||
if quality_check is None:
|
if quality_check is None:
|
||||||
@@ -157,6 +166,9 @@ summary = {
|
|||||||
"detection_count": int(detection_count),
|
"detection_count": int(detection_count),
|
||||||
"quality_status": quality_check.get("status"),
|
"quality_status": quality_check.get("status"),
|
||||||
"quality_score": quality_check.get("score"),
|
"quality_score": quality_check.get("score"),
|
||||||
|
"raw_detection_count": run_result.get("raw_detection_count", int(detection_count)),
|
||||||
|
"suppressed_detection_count": run_result.get("suppressed_detection_count", 0),
|
||||||
|
"duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"),
|
||||||
"precision": metrics.get("precision"),
|
"precision": metrics.get("precision"),
|
||||||
"recall": metrics.get("recall"),
|
"recall": metrics.get("recall"),
|
||||||
"f1_score": metrics.get("f1_score", metrics.get("f1")),
|
"f1_score": metrics.get("f1_score", metrics.get("f1")),
|
||||||
@@ -169,10 +181,12 @@ summary = {
|
|||||||
with open(output_path, "w", encoding="utf-8") as handle:
|
with open(output_path, "w", encoding="utf-8") as handle:
|
||||||
json.dump(summary, handle, indent=2, sort_keys=True)
|
json.dump(summary, handle, indent=2, sort_keys=True)
|
||||||
print(
|
print(
|
||||||
"threshold={threshold} detections={detections} score={score} "
|
"threshold={threshold} detections={detections} raw={raw} suppressed={suppressed} score={score} "
|
||||||
"precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format(
|
"precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format(
|
||||||
threshold=summary["threshold"],
|
threshold=summary["threshold"],
|
||||||
detections=summary["detection_count"],
|
detections=summary["detection_count"],
|
||||||
|
raw=summary["raw_detection_count"],
|
||||||
|
suppressed=summary["suppressed_detection_count"],
|
||||||
score=summary["quality_score"],
|
score=summary["quality_score"],
|
||||||
precision=summary["precision"],
|
precision=summary["precision"],
|
||||||
recall=summary["recall"],
|
recall=summary["recall"],
|
||||||
@@ -217,10 +231,10 @@ with open(summary_path, "w", encoding="utf-8") as handle:
|
|||||||
|
|
||||||
print("")
|
print("")
|
||||||
print("Detection calibration summary")
|
print("Detection calibration summary")
|
||||||
print("threshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
|
print("threshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
|
||||||
for item in items:
|
for item in items:
|
||||||
print(
|
print(
|
||||||
"{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
|
"{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
|
||||||
**item
|
**item
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -168,6 +168,7 @@ while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label
|
|||||||
run_index=$((run_index + 1))
|
run_index=$((run_index + 1))
|
||||||
run_log="${QUALITY_OUTPUT_DIR}/${run_label}.log"
|
run_log="${QUALITY_OUTPUT_DIR}/${run_label}.log"
|
||||||
quality_response="${QUALITY_OUTPUT_DIR}/${run_label}_quality_checks.json"
|
quality_response="${QUALITY_OUTPUT_DIR}/${run_label}_quality_checks.json"
|
||||||
|
detection_run_response="${QUALITY_OUTPUT_DIR}/${run_label}_detection_run.json"
|
||||||
run_summary="${QUALITY_OUTPUT_DIR}/${run_label}_summary.json"
|
run_summary="${QUALITY_OUTPUT_DIR}/${run_label}_summary.json"
|
||||||
model_env="${model_request}"
|
model_env="${model_request}"
|
||||||
if [ "${model_request}" = "__active__" ]; then
|
if [ "${model_request}" = "__active__" ]; then
|
||||||
@@ -203,9 +204,11 @@ while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}"
|
curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}"
|
||||||
|
curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}" > "${detection_run_response}"
|
||||||
|
|
||||||
"${PYTHON_BIN}" - \
|
"${PYTHON_BIN}" - \
|
||||||
"${quality_response}" \
|
"${quality_response}" \
|
||||||
|
"${detection_run_response}" \
|
||||||
"${run_summary}" \
|
"${run_summary}" \
|
||||||
"${model_request}" \
|
"${model_request}" \
|
||||||
"${selected_model_asset_id}" \
|
"${selected_model_asset_id}" \
|
||||||
@@ -226,6 +229,7 @@ import sys
|
|||||||
|
|
||||||
(
|
(
|
||||||
quality_path,
|
quality_path,
|
||||||
|
detection_run_path,
|
||||||
output_path,
|
output_path,
|
||||||
model_request,
|
model_request,
|
||||||
selected_model_asset_id,
|
selected_model_asset_id,
|
||||||
@@ -241,12 +245,18 @@ import sys
|
|||||||
detection_count,
|
detection_count,
|
||||||
export_id,
|
export_id,
|
||||||
run_log,
|
run_log,
|
||||||
) = sys.argv[1:17]
|
) = sys.argv[1:18]
|
||||||
|
|
||||||
with open(quality_path, "r", encoding="utf-8") as handle:
|
with open(quality_path, "r", encoding="utf-8") as handle:
|
||||||
payload = json.load(handle)
|
payload = json.load(handle)
|
||||||
if "data" not in payload:
|
if "data" not in payload:
|
||||||
raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope")
|
raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope")
|
||||||
|
with open(detection_run_path, "r", encoding="utf-8") as handle:
|
||||||
|
detection_run_payload = json.load(handle)
|
||||||
|
if "data" not in detection_run_payload:
|
||||||
|
raise SystemExit("Detection run detail is not a canonical GeoIntel data envelope")
|
||||||
|
detection_run = detection_run_payload["data"]
|
||||||
|
run_result = detection_run.get("result_json") or {}
|
||||||
items = payload["data"].get("items") or []
|
items = payload["data"].get("items") or []
|
||||||
quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None)
|
quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None)
|
||||||
if quality_check is None:
|
if quality_check is None:
|
||||||
@@ -270,6 +280,9 @@ summary = {
|
|||||||
"analysis_run_id": analysis_run_id,
|
"analysis_run_id": analysis_run_id,
|
||||||
"quality_check_id": quality_check_id,
|
"quality_check_id": quality_check_id,
|
||||||
"detection_count": int(detection_count),
|
"detection_count": int(detection_count),
|
||||||
|
"raw_detection_count": run_result.get("raw_detection_count", int(detection_count)),
|
||||||
|
"suppressed_detection_count": run_result.get("suppressed_detection_count", 0),
|
||||||
|
"duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"),
|
||||||
"quality_status": quality_check.get("status"),
|
"quality_status": quality_check.get("status"),
|
||||||
"quality_score": quality_check.get("score"),
|
"quality_score": quality_check.get("score"),
|
||||||
"precision": metrics.get("precision"),
|
"precision": metrics.get("precision"),
|
||||||
@@ -285,13 +298,15 @@ summary = {
|
|||||||
with open(output_path, "w", encoding="utf-8") as handle:
|
with open(output_path, "w", encoding="utf-8") as handle:
|
||||||
json.dump(summary, handle, indent=2, sort_keys=True)
|
json.dump(summary, handle, indent=2, sort_keys=True)
|
||||||
print(
|
print(
|
||||||
"model={model} tile={tile} overlap={overlap} threshold={threshold} detections={detections} "
|
"model={model} tile={tile} overlap={overlap} threshold={threshold} detections={detections} raw={raw} suppressed={suppressed} "
|
||||||
"score={score} precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format(
|
"score={score} precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format(
|
||||||
model=summary["model_asset_id"],
|
model=summary["model_asset_id"],
|
||||||
tile=summary["tile_size"],
|
tile=summary["tile_size"],
|
||||||
overlap=summary["tile_overlap"],
|
overlap=summary["tile_overlap"],
|
||||||
threshold=summary["threshold"],
|
threshold=summary["threshold"],
|
||||||
detections=summary["detection_count"],
|
detections=summary["detection_count"],
|
||||||
|
raw=summary["raw_detection_count"],
|
||||||
|
suppressed=summary["suppressed_detection_count"],
|
||||||
score=summary["quality_score"],
|
score=summary["quality_score"],
|
||||||
precision=summary["precision"],
|
precision=summary["precision"],
|
||||||
recall=summary["recall"],
|
recall=summary["recall"],
|
||||||
@@ -346,10 +361,10 @@ with open(summary_path, "w", encoding="utf-8") as handle:
|
|||||||
|
|
||||||
print("")
|
print("")
|
||||||
print("Detection quality matrix summary")
|
print("Detection quality matrix summary")
|
||||||
print("model\ttile\toverlap\tthreshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
|
print("model\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
|
||||||
for item in items:
|
for item in items:
|
||||||
print(
|
print(
|
||||||
"{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
|
"{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
|
||||||
**item
|
**item
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -205,10 +205,10 @@ summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding=
|
|||||||
|
|
||||||
print("")
|
print("")
|
||||||
print("Multi-sample detection quality summary")
|
print("Multi-sample detection quality summary")
|
||||||
print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
|
print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
|
||||||
for item in flat_items:
|
for item in flat_items:
|
||||||
print(
|
print(
|
||||||
"{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
|
"{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
|
||||||
**item
|
**item
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user