audit YOLO geometry and overlapping validation rows
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Jens
2026-08-09 19:36:15 +02:00
parent fc18e72c7f
commit 736e773fb8
13 changed files with 789 additions and 16 deletions
+8
View File
@@ -20,6 +20,14 @@ files. Its output retains the flagged source tiles so it can be passed directly
to the contact-sheet renderer. Possible nesting remains review evidence and is
never treated as an automatic label error or rewrite instruction.
`audit_yolo_label_outliers.py` retains row-level evidence for sub-threshold
pixel dimensions, extreme aspect ratios and tile-edge labels. Categories can
be emitted separately with repeated `--category` arguments and passed directly
to the contact-sheet renderer. `audit_yolo_cross_tile_repetition.py`
reconstructs global pixel boxes from exporter tile offsets to quantify exact
interior-object repetition caused by overlap; edge rows remain explicitly
unlinked and no repetition is automatically classified as an error.
Setup-, import-, demo- en maintenance-scripts voor GeoIntel.
## WALOUS source provisioning
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
"""Quantify exact interior-object repetition caused by overlapping YOLO tiles."""
from __future__ import annotations
import argparse
import json
import re
from collections import defaultdict
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
try:
from scripts.audit_yolo_label_relationships import (
Box,
parse_label_file,
resolve_path,
sha256_file,
)
except ModuleNotFoundError: # Standalone operator-tool copy beside the auditor.
from audit_yolo_label_relationships import (
Box,
parse_label_file,
resolve_path,
sha256_file,
)
OFFSET_PATTERN = re.compile(r"_r(?P<row>\d+)_c(?P<column>\d+)$")
def tile_offsets(image_path: str) -> tuple[int, int]:
match = OFFSET_PATTERN.search(Path(image_path).stem)
if match is None:
raise ValueError(f"tile filename has no row/column offsets: {image_path}")
return int(match["row"]), int(match["column"])
def interior_global_key(
box: Box,
*,
row_offset: int,
column_offset: int,
tile_size: int,
edge_tolerance_pixels: float,
) -> tuple[float, float, float, float] | None:
left, top, right, bottom = box.coordinates
pixel_box = (
left * tile_size,
top * tile_size,
right * tile_size,
bottom * tile_size,
)
if (
pixel_box[0] <= edge_tolerance_pixels
or pixel_box[1] <= edge_tolerance_pixels
or tile_size - pixel_box[2] <= edge_tolerance_pixels
or tile_size - pixel_box[3] <= edge_tolerance_pixels
):
return None
return (
round(column_offset + pixel_box[0], 3),
round(row_offset + pixel_box[1], 3),
round(column_offset + pixel_box[2], 3),
round(row_offset + pixel_box[3], 3),
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary-path", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--edge-tolerance-pixels", type=float, default=0.5)
args = parser.parse_args()
if args.output.exists():
parser.error(f"output already exists: {args.output}")
if args.edge_tolerance_pixels < 0:
parser.error("--edge-tolerance-pixels must be non-negative")
summary_path = args.summary_path.expanduser().resolve(strict=True)
summary = json.loads(summary_path.read_text(encoding="utf-8"))
tiles = summary.get("tiles")
tile_size = summary.get("tile_size")
if not isinstance(tiles, list):
raise ValueError("dataset summary must contain a tiles list")
if not isinstance(tile_size, int) or tile_size <= 0:
raise ValueError("dataset summary must contain a positive integer tile_size")
groups: dict[tuple[str, int, float, float, float, float], list[dict[str, Any]]] = (
defaultdict(list)
)
reviewed_label_count = 0
edge_label_count = 0
for tile in tiles:
if not isinstance(tile, dict) or not tile.get("kept", True):
continue
image_path = str(tile.get("image_path") or "")
row_offset, column_offset = tile_offsets(image_path)
label_path = resolve_path(str(tile.get("label_path") or ""), summary_path)
boxes = parse_label_file(label_path)
reviewed_label_count += len(boxes)
for index, box in enumerate(boxes):
global_key = interior_global_key(
box,
row_offset=row_offset,
column_offset=column_offset,
tile_size=tile_size,
edge_tolerance_pixels=args.edge_tolerance_pixels,
)
if global_key is None:
edge_label_count += 1
continue
key = (
str(tile.get("sample_slug") or "unknown"),
box.class_id,
*global_key,
)
groups[key].append(
{
"split": str(tile.get("split") or "unknown"),
"tile_index": int(tile.get("tile_index") or 0),
"label_index": index,
"image_path": image_path,
"label_path": str(label_path),
}
)
repeated_groups = [members for members in groups.values() if len(members) > 1]
split_leakage_groups = [
members
for members in repeated_groups
if len({member["split"] for member in members}) > 1
]
repetition_histogram: dict[str, int] = defaultdict(int)
for members in groups.values():
repetition_histogram[str(len(members))] += 1
sample_stats: dict[str, dict[str, int]] = defaultdict(
lambda: {
"unique_interior_objects": 0,
"interior_label_rows": 0,
"repeated_object_groups": 0,
"extra_repeated_rows": 0,
}
)
for key, members in groups.items():
sample = sample_stats[key[0]]
sample["unique_interior_objects"] += 1
sample["interior_label_rows"] += len(members)
if len(members) > 1:
sample["repeated_object_groups"] += 1
sample["extra_repeated_rows"] += len(members) - 1
interior_label_count = sum(len(members) for members in groups.values())
extra_repeated_rows = sum(len(members) - 1 for members in repeated_groups)
payload = {
"schema_version": 1,
"generated_at": datetime.now(UTC).isoformat(),
"status": "attention" if repeated_groups else "ok",
"claim_boundary": (
"Exact reconstructed repetition among interior boxes only; tile-edge "
"rows remain unlinked and repetition is not automatically an error."
),
"summary_path": str(summary_path),
"summary_sha256": sha256_file(summary_path),
"tile_size": tile_size,
"stride": summary.get("stride"),
"edge_tolerance_pixels": args.edge_tolerance_pixels,
"reviewed_label_count": reviewed_label_count,
"interior_label_count": interior_label_count,
"edge_label_count": edge_label_count,
"unique_interior_object_count": len(groups),
"repeated_object_group_count": len(repeated_groups),
"extra_repeated_label_row_count": extra_repeated_rows,
"max_repetition": max((len(members) for members in groups.values()), default=0),
"repetition_histogram": dict(
sorted(repetition_histogram.items(), key=lambda item: int(item[0]))
),
"cross_split_repetition_group_count": len(split_leakage_groups),
"sample_stats": [
{"sample_slug": slug, **stats}
for slug, stats in sorted(sample_stats.items())
],
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
print(json.dumps(payload, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""Audit individual YOLO rows for geometric training risks without mutation."""
from __future__ import annotations
import argparse
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
try:
from scripts.audit_yolo_label_relationships import (
Box,
parse_label_file,
resolve_path,
sha256_file,
)
except ModuleNotFoundError: # Standalone operator-tool copy beside the auditor.
from audit_yolo_label_relationships import (
Box,
parse_label_file,
resolve_path,
sha256_file,
)
def classify_box(
box: Box,
*,
tile_size: int,
min_dimension_pixels: float,
extreme_aspect_ratio: float,
edge_tolerance_pixels: float,
) -> list[dict[str, Any]]:
width_px = box.width * tile_size
height_px = box.height * tile_size
aspect_ratio = max(width_px / height_px, height_px / width_px)
left, top, right, bottom = box.coordinates
edge_sides = [
side
for side, distance in (
("left", left * tile_size),
("top", top * tile_size),
("right", (1 - right) * tile_size),
("bottom", (1 - bottom) * tile_size),
)
if distance <= edge_tolerance_pixels
]
metrics = {
"width_px": round(width_px, 6),
"height_px": round(height_px, 6),
"area_px2": round(width_px * height_px, 6),
"aspect_ratio": round(aspect_ratio, 6),
}
outliers: list[dict[str, Any]] = []
if min(width_px, height_px) < min_dimension_pixels:
outliers.append({"category": "small_dimension", **metrics})
if aspect_ratio >= extreme_aspect_ratio:
outliers.append({"category": "extreme_aspect_ratio", **metrics})
if edge_sides:
outliers.append({"category": "tile_edge", "edge_sides": edge_sides, **metrics})
return outliers
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary-path", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--tile-size", type=int)
parser.add_argument("--min-dimension-pixels", type=float, default=4.0)
parser.add_argument("--extreme-aspect-ratio", type=float, default=8.0)
parser.add_argument("--edge-tolerance-pixels", type=float, default=0.5)
parser.add_argument(
"--category",
action="append",
choices=("small_dimension", "extreme_aspect_ratio", "tile_edge"),
default=[],
help="Limit output to one risk category; repeat to select multiple.",
)
args = parser.parse_args()
if args.output.exists():
parser.error(f"output already exists: {args.output}")
if args.min_dimension_pixels <= 0:
parser.error("--min-dimension-pixels must be positive")
if args.extreme_aspect_ratio < 1:
parser.error("--extreme-aspect-ratio must be at least one")
if args.edge_tolerance_pixels < 0:
parser.error("--edge-tolerance-pixels must be non-negative")
summary_path = args.summary_path.expanduser().resolve(strict=True)
summary = json.loads(summary_path.read_text(encoding="utf-8"))
tiles = summary.get("tiles")
if not isinstance(tiles, list):
raise ValueError("dataset summary must contain a tiles list")
tile_size = args.tile_size or summary.get("tile_size")
if not isinstance(tile_size, int) or tile_size <= 0:
raise ValueError("a positive integer tile size is required")
selected_categories = set(
args.category or ("small_dimension", "extreme_aspect_ratio", "tile_edge")
)
category_totals = {
"small_dimension": 0,
"extreme_aspect_ratio": 0,
"tile_edge": 0,
}
reviewed_label_count = 0
unique_flagged_rows: set[tuple[str, int]] = set()
flagged_tiles: list[dict[str, Any]] = []
renderable_tiles: list[dict[str, Any]] = []
for tile in tiles:
if not isinstance(tile, dict) or not tile.get("kept", True):
continue
label_path = resolve_path(str(tile.get("label_path") or ""), summary_path)
boxes = parse_label_file(label_path)
reviewed_label_count += len(boxes)
outliers: list[dict[str, Any]] = []
for index, box in enumerate(boxes):
classifications = classify_box(
box,
tile_size=tile_size,
min_dimension_pixels=args.min_dimension_pixels,
extreme_aspect_ratio=args.extreme_aspect_ratio,
edge_tolerance_pixels=args.edge_tolerance_pixels,
)
for classification in classifications:
if classification["category"] not in selected_categories:
continue
category_totals[classification["category"]] += 1
unique_flagged_rows.add((str(label_path), index))
outliers.append(
{
"index": index,
"box": box.as_list(),
**classification,
}
)
if not outliers:
continue
flagged_tiles.append(
{
"sample_slug": str(tile.get("sample_slug") or "unknown"),
"split": str(tile.get("split") or "unknown"),
"tile_index": int(tile.get("tile_index") or 0),
"label_path": str(label_path),
"label_count": len(boxes),
"outlier_count": len(outliers),
"outliers": outliers,
}
)
renderable_tiles.append(tile)
flagged_tiles.sort(
key=lambda item: (
-item["outlier_count"],
item["sample_slug"],
item["split"],
item["tile_index"],
)
)
payload = {
"schema_version": 1,
"generated_at": datetime.now(UTC).isoformat(),
"status": "attention" if flagged_tiles else "ok",
"claim_boundary": (
"Read-only geometric risk triage; a flagged label is not an automatic "
"ground-truth error or rewrite instruction."
),
"summary_path": str(summary_path),
"summary_sha256": sha256_file(summary_path),
"output_dir": summary.get("output_dir"),
"class_names": summary.get("class_names", []),
"tile_size": tile_size,
"thresholds": {
"min_dimension_pixels": args.min_dimension_pixels,
"extreme_aspect_ratio": args.extreme_aspect_ratio,
"edge_tolerance_pixels": args.edge_tolerance_pixels,
},
"selected_categories": sorted(selected_categories),
"reviewed_tile_count": sum(
1 for tile in tiles if isinstance(tile, dict) and tile.get("kept", True)
),
"reviewed_label_count": reviewed_label_count,
"flagged_tile_count": len(flagged_tiles),
"unique_flagged_label_count": len(unique_flagged_rows),
"category_totals": category_totals,
"flagged_tiles": flagged_tiles,
"tiles": renderable_tiles,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
print(
json.dumps(
{
key: payload[key]
for key in (
"status",
"reviewed_tile_count",
"reviewed_label_count",
"flagged_tile_count",
"unique_flagged_label_count",
"category_totals",
)
},
indent=2,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+53 -4
View File
@@ -45,13 +45,50 @@ def validation_images(dataset_yaml: Path) -> list[Path]:
images = sorted(
path
for path in directory.iterdir()
if path.is_file() and path.suffix.casefold() in {".jpg", ".jpeg", ".png", ".tif", ".tiff"}
if path.is_file()
and path.suffix.casefold() in {".jpg", ".jpeg", ".png", ".tif", ".tiff"}
)
if not images:
raise ValueError("validation image directory is empty")
return images
def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
summary_path = dataset_yaml.parent / "yolo_tile_dataset_summary.json"
if not summary_path.is_file():
return {
"status": "unavailable",
"summary_path": str(summary_path),
"validation_rows_independent": None,
}
payload = json.loads(summary_path.read_text(encoding="utf-8"))
tile_size = payload.get("tile_size")
stride = payload.get("stride")
if not isinstance(tile_size, int) or not isinstance(stride, int) or stride <= 0:
return {
"status": "invalid",
"summary_path": str(summary_path),
"summary_sha256": sha256_file(summary_path),
"validation_rows_independent": None,
}
overlap_pixels = max(tile_size - stride, 0)
return {
"status": "overlapping" if overlap_pixels else "non_overlapping",
"summary_path": str(summary_path),
"summary_sha256": sha256_file(summary_path),
"tile_size": tile_size,
"stride": stride,
"overlap_pixels": overlap_pixels,
"validation_rows_independent": overlap_pixels == 0,
"interpretation": (
"Tile metrics can repeat the same source object and are valid for "
"candidate ranking only, not independent object-level uncertainty."
if overlap_pixels
else "Tile rows do not overlap according to the dataset summary."
),
}
def metric_value(metrics: Any, attribute: str) -> float:
value = getattr(metrics.box, attribute)
return float(value)
@@ -68,7 +105,9 @@ def background_detection_count(
) -> tuple[int, int]:
selected = [path for path in images if path.stem.casefold().startswith(prefixes)]
if not selected:
raise ValueError("no validation images match the declared pure-background prefixes")
raise ValueError(
"no validation images match the declared pure-background prefixes"
)
count = 0
for start in range(0, len(selected), 16):
results = model.predict(
@@ -102,6 +141,7 @@ def main() -> int:
parser.error("--background-confidence must be between zero and one")
dataset_yaml = args.dataset_yaml.expanduser().resolve(strict=True)
images = validation_images(dataset_yaml)
overlap_evidence = dataset_overlap_evidence(dataset_yaml)
prefixes = tuple(value.casefold() for value in args.background_prefix)
import torch
@@ -161,7 +201,13 @@ def main() -> int:
}
)
except Exception as exc: # preserve the complete attempted matrix
row.update({"status": "error", "error_type": type(exc).__name__, "error": str(exc)[:1000]})
row.update(
{
"status": "error",
"error_type": type(exc).__name__,
"error": str(exc)[:1000],
}
)
finally:
del model
if torch.cuda.is_available():
@@ -187,6 +233,7 @@ def main() -> int:
"claim_boundary": "Non-protected validation ranking only; no test, challenge or promotion claim.",
"dataset_yaml": str(dataset_yaml),
"dataset_yaml_sha256": sha256_file(dataset_yaml),
"dataset_overlap_evidence": overlap_evidence,
"validation_image_count": len(images),
"pure_background_prefixes": list(prefixes),
"pure_background_confidence": args.background_confidence,
@@ -198,7 +245,9 @@ def main() -> int:
"attempts": rows,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
args.output.write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
return 0 if successful else 2
@@ -287,6 +287,9 @@ def draw_tile_card(
"exact_duplicate": ((0, 220, 255), 5),
"near_duplicate": ((255, 55, 55), 5),
"possible_nested": ((255, 0, 220), 5),
"extreme_aspect_ratio": ((255, 128, 0), 5),
"small_dimension": ((80, 160, 255), 5),
"tile_edge": ((57, 255, 20), 5),
}
for box_index, box in enumerate(boxes):
x_center = box["center_x"] * thumb_size
@@ -317,14 +320,17 @@ def draw_tile_card(
return card
def build_relationship_highlights(
def build_label_highlights(
summary: dict[str, Any],
) -> dict[str, dict[int, str]]:
"""Map audited label rows to their highest-priority visual warning."""
priorities = {
"possible_nested": 1,
"near_duplicate": 2,
"exact_duplicate": 3,
"tile_edge": 1,
"small_dimension": 2,
"extreme_aspect_ratio": 3,
"possible_nested": 4,
"near_duplicate": 5,
"exact_duplicate": 6,
}
highlights: dict[str, dict[int, str]] = {}
flagged_tiles = summary.get("flagged_tiles") or []
@@ -355,9 +361,30 @@ def build_relationship_highlights(
):
tile_highlights[index] = relationship_type
outliers = tile.get("outliers") or []
if not isinstance(outliers, list):
continue
for outlier in outliers:
if not isinstance(outlier, dict):
continue
category = str(outlier.get("category") or "")
index = outlier.get("index")
if category not in priorities or not isinstance(index, int) or index < 0:
continue
existing = tile_highlights.get(index)
if existing is None or priorities[category] > priorities[existing]:
tile_highlights[index] = category
return highlights
def build_relationship_highlights(
summary: dict[str, Any],
) -> dict[str, dict[int, str]]:
"""Backward-compatible name for callers using relationship manifests."""
return build_label_highlights(summary)
def image_has_low_visual_variance(image_path: Path, blank_range_threshold: int) -> bool:
image = Image.open(image_path).convert("L")
min_value, max_value = image.getextrema()
@@ -406,8 +433,8 @@ def build_report(
invalid_label_count = 0
valid_label_count = 0
low_visual_variance_tile_count = 0
relationship_highlight_count = 0
relationship_highlights = build_relationship_highlights(summary)
highlight_category_counts: dict[str, int] = {}
label_highlights = build_label_highlights(summary)
for tile in selected_tiles:
image_path = resolve_path(tile.get("image_path"), summary_path)
@@ -415,8 +442,11 @@ def build_report(
boxes, tile_invalid_count, missing_label_file = parse_yolo_label_file(
label_path
)
tile_relationship_highlights = relationship_highlights.get(str(label_path), {})
relationship_highlight_count += len(tile_relationship_highlights)
tile_label_highlights = label_highlights.get(str(label_path), {})
for category in tile_label_highlights.values():
highlight_category_counts[category] = (
highlight_category_counts.get(category, 0) + 1
)
invalid_label_count += tile_invalid_count
valid_label_count += len(boxes)
if missing_label_file:
@@ -441,7 +471,7 @@ def build_report(
invalid_label_count=tile_invalid_count,
missing_label_file=missing_label_file,
low_visual_variance=low_visual_variance,
relationship_highlights=tile_relationship_highlights,
relationship_highlights=tile_label_highlights,
)
)
rendered = True
@@ -460,7 +490,7 @@ def build_report(
"invalid_label_count": tile_invalid_count,
"missing_label_file": missing_label_file,
"low_visual_variance": low_visual_variance,
"relationship_highlight_count": len(tile_relationship_highlights),
"highlighted_label_count": len(tile_label_highlights),
"rendered": rendered,
}
)
@@ -478,6 +508,14 @@ def build_report(
}
for start in range(0, len(rendered_cards), args.tiles_per_sheet)
]
relationship_highlight_count = sum(
highlight_category_counts.get(category, 0)
for category in ("exact_duplicate", "near_duplicate", "possible_nested")
)
outlier_highlight_count = sum(
highlight_category_counts.get(category, 0)
for category in ("extreme_aspect_ratio", "small_dimension", "tile_edge")
)
return (
{
@@ -503,7 +541,12 @@ def build_report(
"invalid_label_count": invalid_label_count,
"valid_label_count": valid_label_count,
"low_visual_variance_tile_count": low_visual_variance_tile_count,
"highlighted_label_count": sum(highlight_category_counts.values()),
"relationship_highlight_count": relationship_highlight_count,
"outlier_highlight_count": outlier_highlight_count,
"highlight_category_counts": dict(
sorted(highlight_category_counts.items())
),
"blank_range_threshold": args.blank_range_threshold,
"contact_sheets": contact_sheets,
"selected_tiles": selected_report_tiles,
@@ -525,7 +568,8 @@ def write_markdown(report: dict[str, Any], output_dir: Path) -> None:
f"- invalid label rows: {report['invalid_label_count']}",
f"- low-variance rendered tiles: {report['low_visual_variance_tile_count']}",
f"- valid labels rendered: {report['valid_label_count']}",
f"- relationship-highlighted labels: {report['relationship_highlight_count']}",
f"- highlighted labels: {report['highlighted_label_count']}",
f"- highlight categories: `{report['highlight_category_counts']}`",
"",
"## Contact Sheets",
"",