audit YOLO geometry and overlapping validation rows
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Jens
2026-08-09 19:36:15 +02:00
parent fc18e72c7f
commit 736e773fb8
13 changed files with 789 additions and 16 deletions
@@ -1,7 +1,7 @@
{ {
"schema_version": 1, "schema_version": 1,
"ledger_id": "reviewedexp6-ai-assisted-review-r2-full-corpus", "ledger_id": "reviewedexp6-ai-assisted-review-r2-full-corpus",
"reviewed_at": "2026-08-09T19:10:57+02:00", "reviewed_at": "2026-08-09T19:34:31+02:00",
"reviewer": { "reviewer": {
"id": "openai-codex", "id": "openai-codex",
"type": "ai_assistant", "type": "ai_assistant",
@@ -96,5 +96,36 @@
"share_of_rendered_labels": 0.000543, "share_of_rendered_labels": 0.000543,
"ai_visual_followup": "all 86 implicated label rows were index-highlighted; no cyan exact duplicates or red near duplicates exist, while the 43 magenta nesting pairs are dispersed across dense reference AOIs and predominantly show larger GRB envelopes around smaller building components rather than one systematic duplicate-label pattern", "ai_visual_followup": "all 86 implicated label rows were index-highlighted; no cyan exact duplicates or red near duplicates exist, while the 43 magenta nesting pairs are dispersed across dense reference AOIs and predominantly show larger GRB envelopes around smaller building components rather than one systematic duplicate-label pattern",
"action": "retain for experimental analysis, require human adjudication before release, and perform no automatic rewrite or exclusion" "action": "retain for experimental analysis, require human adjudication before release, and perform no automatic rewrite or exclusion"
},
"geometric_outlier_audit": {
"audit_path": "/app/storage/operator-data/model-review/reviewedexp6-corpus/full-ai-review-r1/label_outlier_audit.json",
"audit_sha256": "cac2c0cefb49aa3785ee9fb173a0187b3dd8abfc9c25e8e91df6c4af91ff5a40",
"reviewed_label_count": 79192,
"unique_flagged_label_count": 6657,
"small_dimension_below_4_px": 1812,
"extreme_aspect_ratio_at_least_8": 26,
"tile_edge_rows": 5079,
"extreme_rows_also_below_4_px": 16,
"extreme_contact_sheet_sha256": "67ad11e7d80678c78157a01a5e7ceb7cff9ac24410b127bdfaafd4350170cdb6",
"small_dimension_contact_sheet_sha256s": [
"5b3ca4d5bfd57b466854a00b4791ac82cf4a826fad8a4694d0227c931b00dfad",
"0e5d2058154f8298fc6d2c064150bcf896e4e51b9bcce5fccbfc049e1ebb9515",
"03060f65843f3c2a291dd009124484f4cf798fe653e658bab007375613fa3010",
"2fe7c748a68b21a673f3d950909b07e3ddba81b736977fbee8d64ea62b422ca2"
],
"tile_edge_sample_contact_sheet_sha256": "5d96c5a0a732077ed16b22d33bed998b37bd33b5536b1491e778999a2a6638dd",
"ai_visual_followup": "extreme rows are mostly plausible elongated structures, while sub-4-pixel rows are often visually marginal; tile-edge rows are expected clipped objects under the 0.35 visibility policy",
"next_corpus_decision": "restore the exporter minimum dimension to at least 4 px for the next immutable experimental corpus; retain elongated structures above that floor and measure any higher edge-visibility threshold in a separately versioned ablation"
},
"cross_tile_repetition_audit": {
"audit_path": "/app/storage/operator-data/model-review/reviewedexp6-corpus/full-ai-review-r1/cross_tile_repetition_audit.json",
"audit_sha256": "3cdf5174fec61e0aa426b4a3f44ed943a738bf038f3b58028ec856ef1a9d000c",
"interior_label_rows": 74113,
"unique_interior_objects": 34082,
"extra_repeated_rows": 40031,
"repeated_object_groups": 24701,
"max_repetition": 4,
"cross_split_repetition_groups": 0,
"interpretation": "the 512 px tiles use stride 256, so tile-level validation rows are not statistically independent even though no reconstructed object crosses train and validation splits"
} }
} }
+43
View File
@@ -12595,3 +12595,46 @@ Open:
- This is explicit AI-assisted triage, not human adjudication. It improves the - This is explicit AI-assisted triage, not human adjudication. It improves the
evidence and review precision but does not unlock governed production evidence and review precision but does not unlock governed production
training or checkpoint promotion. training or checkpoint promotion.
## 2026-08-09 - Row-level geometry and cross-tile independence audit
### Improved
- Added a reproducible, read-only label-row auditor for sub-threshold pixel
dimensions, extreme aspect ratios and tile-edge clipping, with separate
category manifests accepted directly by the visual QA renderer.
- Extended the renderer with orange extreme-aspect, blue small-dimension and
green tile-edge highlighting while preserving the existing duplicate/nesting
colors and source row indices.
- Added an exact cross-tile repetition auditor that reconstructs global pixel
boxes from the exporter's `_r<row>_c<column>` offsets. Edge-clipped rows stay
explicitly unlinked rather than being guessed back to source objects.
- Made checkpoint-matrix output disclose dataset tile overlap and explicitly
state whether validation rows are independent whenever a dataset summary is
available.
### Tower findings
- Reviewed all 79,192 rows: 1,812 have a dimension below 4 px, 26 have aspect
ratio at least 8 and 5,079 touch a tile edge; 6,657 unique rows are involved.
- Inspected the complete 23-tile extreme sheet, all four small-dimension pages
covering 198 tiles, and a 64-tile edge sample. Most extreme shapes are valid
elongated structures, but 16 of 26 also fall below the 4 px floor.
- Reconstructed 34,082 unique interior objects from 74,113 interior label rows.
Tile overlap adds 40,031 exact repeated rows across 24,701 object groups,
with maximum repetition four and zero cross-split groups.
- Froze the next-corpus decision: restore `min_label_px >= 4`; test any stricter
edge-visible ratio as a separate immutable ablation rather than modifying
this corpus in place.
### Verified
- Focused auditor, renderer and checkpoint-evidence suite: 18 tests passed.
- Scoped Ruff check, format check and Git whitespace check passed.
### Remaining limitations
- Existing overlapping-tile mAP is valid for relative non-protected ranking,
not independent object-level accuracy. A future benchmark must aggregate or
deduplicate objects before uncertainty claims.
- No source label, frozen dataset, model weight or live runtime was changed.
+8
View File
@@ -1159,6 +1159,14 @@ This file now starts with the current implementation status. Older preparation/b
contact sheet; AI follow-up found no systematic duplicate-label pattern. contact sheet; AI follow-up found no systematic duplicate-label pattern.
- [x] Bind the relationship audit back to exact label-row indices and render - [x] Bind the relationship audit back to exact label-row indices and render
all 86 implicated rows with type-specific highlighting for focused review. all 86 implicated rows with type-specific highlighting for focused review.
- [x] Audit all 79,192 labels at row level for sub-4-pixel dimensions, extreme
aspect ratio and tile-edge clipping; inspect the complete small/extreme
sheets and retain immutable hashes.
- [x] Reconstruct exact cross-tile interior objects and prove that overlap
repeats 40,031 label rows but causes zero reconstructed train/val crossings.
- [ ] Build the next immutable experimental corpus with `min_label_px >= 4`
and compare the existing 0.35 edge-visibility policy against a separately
versioned stricter ablation before any new governed training.
- [ ] Convert the AI-assisted ledger into no stronger claim than experimental - [ ] Convert the AI-assisted ledger into no stronger claim than experimental
triage; a real human must independently review and sign the frozen artifacts triage; a real human must independently review and sign the frozen artifacts
before the governed training wrapper may unlock. before the governed training wrapper may unlock.
@@ -23,6 +23,15 @@ The checkpoint matrix used only the 36-image non-protected validation split:
measured mAP50 `0.345426` and mAP50-95 `0.140719`. The `reviewedexp6` measured mAP50 `0.345426` and mAP50-95 `0.140719`. The `reviewedexp6`
challenger measured `0.368177` and `0.155438` respectively. challenger measured `0.368177` and `0.155438` respectively.
A later exact cross-tile reconstruction established that this 512 px corpus
uses stride 256. Across the full corpus, 74,113 interior label rows represent
34,082 unique reconstructed objects; 40,031 rows are overlap repetitions and
one object can occur four times. No reconstructed object crosses the
train/validation boundary, but the tile rows are not statistically independent.
The matrix therefore remains useful only for relative non-protected candidate
ranking. Its mAP values are not an independent object-level accuracy estimate
or a basis for narrow confidence claims.
At confidence `0.25` and match IoU `0.25`, the active model measured F1 At confidence `0.25` and match IoU `0.25`, the active model measured F1
`0.633058`; the challenger measured `0.642599`. Both produced zero detections `0.633058`; the challenger measured `0.642599`. Both produced zero detections
on the 18 pure-background validation tiles at this threshold. At confidence on the 18 pure-background validation tiles at this threshold. At confidence
@@ -129,3 +138,26 @@ rewrite candidates. Excluding the 36 complete tiles would also discard 17,167
unflagged labels, so no automatic tile exclusion or label mutation was made. unflagged labels, so no automatic tile exclusion or label mutation was made.
The checksum-bound highlighted sheet and summary are recorded in the AI review The checksum-bound highlighted sheet and summary are recorded in the AI review
ledger. ledger.
## Row-level geometry and overlap follow-up
The complete 79,192-label corpus was additionally audited at label-row level.
The immutable manifest identifies 6,657 unique rows with at least one geometric
training-risk signal: 1,812 rows have a width or height below 4 pixels, 26 have
aspect ratio at least 8 and 5,079 touch a tile edge. The categories were
rendered separately. All 23 extreme-aspect tiles and all 198 small-dimension
tiles were inspected, plus a 64-tile edge sample.
The extreme-aspect group predominantly shows plausible elongated sheds and
building components. Sixteen of its 26 rows are also below 4 pixels, allowing
the resolution floor to address most ambiguous extremes without deleting valid
long structures. The small-dimension sheets contain many visually marginal
miniature targets. The next immutable experimental corpus should therefore use
the exporter's normal minimum dimension of at least 4 pixels instead of this
legacy corpus's 3-pixel override. This finding does not justify mutating the
frozen corpus or retroactively changing its checkpoint.
The edge sample shows expected clipped buildings under the 0.35 minimum-visible
policy. Raising that value may reduce partial-target pressure, but it must be a
separately versioned ablation because removing all 5,079 rows without checking
the original visible fraction would be unsound.
+8
View File
@@ -20,6 +20,14 @@ files. Its output retains the flagged source tiles so it can be passed directly
to the contact-sheet renderer. Possible nesting remains review evidence and is to the contact-sheet renderer. Possible nesting remains review evidence and is
never treated as an automatic label error or rewrite instruction. never treated as an automatic label error or rewrite instruction.
`audit_yolo_label_outliers.py` retains row-level evidence for sub-threshold
pixel dimensions, extreme aspect ratios and tile-edge labels. Categories can
be emitted separately with repeated `--category` arguments and passed directly
to the contact-sheet renderer. `audit_yolo_cross_tile_repetition.py`
reconstructs global pixel boxes from exporter tile offsets to quantify exact
interior-object repetition caused by overlap; edge rows remain explicitly
unlinked and no repetition is automatically classified as an error.
Setup-, import-, demo- en maintenance-scripts voor GeoIntel. Setup-, import-, demo- en maintenance-scripts voor GeoIntel.
## WALOUS source provisioning ## WALOUS source provisioning
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
"""Quantify exact interior-object repetition caused by overlapping YOLO tiles."""
from __future__ import annotations
import argparse
import json
import re
from collections import defaultdict
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
try:
from scripts.audit_yolo_label_relationships import (
Box,
parse_label_file,
resolve_path,
sha256_file,
)
except ModuleNotFoundError: # Standalone operator-tool copy beside the auditor.
from audit_yolo_label_relationships import (
Box,
parse_label_file,
resolve_path,
sha256_file,
)
OFFSET_PATTERN = re.compile(r"_r(?P<row>\d+)_c(?P<column>\d+)$")
def tile_offsets(image_path: str) -> tuple[int, int]:
match = OFFSET_PATTERN.search(Path(image_path).stem)
if match is None:
raise ValueError(f"tile filename has no row/column offsets: {image_path}")
return int(match["row"]), int(match["column"])
def interior_global_key(
box: Box,
*,
row_offset: int,
column_offset: int,
tile_size: int,
edge_tolerance_pixels: float,
) -> tuple[float, float, float, float] | None:
left, top, right, bottom = box.coordinates
pixel_box = (
left * tile_size,
top * tile_size,
right * tile_size,
bottom * tile_size,
)
if (
pixel_box[0] <= edge_tolerance_pixels
or pixel_box[1] <= edge_tolerance_pixels
or tile_size - pixel_box[2] <= edge_tolerance_pixels
or tile_size - pixel_box[3] <= edge_tolerance_pixels
):
return None
return (
round(column_offset + pixel_box[0], 3),
round(row_offset + pixel_box[1], 3),
round(column_offset + pixel_box[2], 3),
round(row_offset + pixel_box[3], 3),
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary-path", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--edge-tolerance-pixels", type=float, default=0.5)
args = parser.parse_args()
if args.output.exists():
parser.error(f"output already exists: {args.output}")
if args.edge_tolerance_pixels < 0:
parser.error("--edge-tolerance-pixels must be non-negative")
summary_path = args.summary_path.expanduser().resolve(strict=True)
summary = json.loads(summary_path.read_text(encoding="utf-8"))
tiles = summary.get("tiles")
tile_size = summary.get("tile_size")
if not isinstance(tiles, list):
raise ValueError("dataset summary must contain a tiles list")
if not isinstance(tile_size, int) or tile_size <= 0:
raise ValueError("dataset summary must contain a positive integer tile_size")
groups: dict[tuple[str, int, float, float, float, float], list[dict[str, Any]]] = (
defaultdict(list)
)
reviewed_label_count = 0
edge_label_count = 0
for tile in tiles:
if not isinstance(tile, dict) or not tile.get("kept", True):
continue
image_path = str(tile.get("image_path") or "")
row_offset, column_offset = tile_offsets(image_path)
label_path = resolve_path(str(tile.get("label_path") or ""), summary_path)
boxes = parse_label_file(label_path)
reviewed_label_count += len(boxes)
for index, box in enumerate(boxes):
global_key = interior_global_key(
box,
row_offset=row_offset,
column_offset=column_offset,
tile_size=tile_size,
edge_tolerance_pixels=args.edge_tolerance_pixels,
)
if global_key is None:
edge_label_count += 1
continue
key = (
str(tile.get("sample_slug") or "unknown"),
box.class_id,
*global_key,
)
groups[key].append(
{
"split": str(tile.get("split") or "unknown"),
"tile_index": int(tile.get("tile_index") or 0),
"label_index": index,
"image_path": image_path,
"label_path": str(label_path),
}
)
repeated_groups = [members for members in groups.values() if len(members) > 1]
split_leakage_groups = [
members
for members in repeated_groups
if len({member["split"] for member in members}) > 1
]
repetition_histogram: dict[str, int] = defaultdict(int)
for members in groups.values():
repetition_histogram[str(len(members))] += 1
sample_stats: dict[str, dict[str, int]] = defaultdict(
lambda: {
"unique_interior_objects": 0,
"interior_label_rows": 0,
"repeated_object_groups": 0,
"extra_repeated_rows": 0,
}
)
for key, members in groups.items():
sample = sample_stats[key[0]]
sample["unique_interior_objects"] += 1
sample["interior_label_rows"] += len(members)
if len(members) > 1:
sample["repeated_object_groups"] += 1
sample["extra_repeated_rows"] += len(members) - 1
interior_label_count = sum(len(members) for members in groups.values())
extra_repeated_rows = sum(len(members) - 1 for members in repeated_groups)
payload = {
"schema_version": 1,
"generated_at": datetime.now(UTC).isoformat(),
"status": "attention" if repeated_groups else "ok",
"claim_boundary": (
"Exact reconstructed repetition among interior boxes only; tile-edge "
"rows remain unlinked and repetition is not automatically an error."
),
"summary_path": str(summary_path),
"summary_sha256": sha256_file(summary_path),
"tile_size": tile_size,
"stride": summary.get("stride"),
"edge_tolerance_pixels": args.edge_tolerance_pixels,
"reviewed_label_count": reviewed_label_count,
"interior_label_count": interior_label_count,
"edge_label_count": edge_label_count,
"unique_interior_object_count": len(groups),
"repeated_object_group_count": len(repeated_groups),
"extra_repeated_label_row_count": extra_repeated_rows,
"max_repetition": max((len(members) for members in groups.values()), default=0),
"repetition_histogram": dict(
sorted(repetition_histogram.items(), key=lambda item: int(item[0]))
),
"cross_split_repetition_group_count": len(split_leakage_groups),
"sample_stats": [
{"sample_slug": slug, **stats}
for slug, stats in sorted(sample_stats.items())
],
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
print(json.dumps(payload, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""Audit individual YOLO rows for geometric training risks without mutation."""
from __future__ import annotations
import argparse
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
try:
from scripts.audit_yolo_label_relationships import (
Box,
parse_label_file,
resolve_path,
sha256_file,
)
except ModuleNotFoundError: # Standalone operator-tool copy beside the auditor.
from audit_yolo_label_relationships import (
Box,
parse_label_file,
resolve_path,
sha256_file,
)
def classify_box(
box: Box,
*,
tile_size: int,
min_dimension_pixels: float,
extreme_aspect_ratio: float,
edge_tolerance_pixels: float,
) -> list[dict[str, Any]]:
width_px = box.width * tile_size
height_px = box.height * tile_size
aspect_ratio = max(width_px / height_px, height_px / width_px)
left, top, right, bottom = box.coordinates
edge_sides = [
side
for side, distance in (
("left", left * tile_size),
("top", top * tile_size),
("right", (1 - right) * tile_size),
("bottom", (1 - bottom) * tile_size),
)
if distance <= edge_tolerance_pixels
]
metrics = {
"width_px": round(width_px, 6),
"height_px": round(height_px, 6),
"area_px2": round(width_px * height_px, 6),
"aspect_ratio": round(aspect_ratio, 6),
}
outliers: list[dict[str, Any]] = []
if min(width_px, height_px) < min_dimension_pixels:
outliers.append({"category": "small_dimension", **metrics})
if aspect_ratio >= extreme_aspect_ratio:
outliers.append({"category": "extreme_aspect_ratio", **metrics})
if edge_sides:
outliers.append({"category": "tile_edge", "edge_sides": edge_sides, **metrics})
return outliers
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary-path", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--tile-size", type=int)
parser.add_argument("--min-dimension-pixels", type=float, default=4.0)
parser.add_argument("--extreme-aspect-ratio", type=float, default=8.0)
parser.add_argument("--edge-tolerance-pixels", type=float, default=0.5)
parser.add_argument(
"--category",
action="append",
choices=("small_dimension", "extreme_aspect_ratio", "tile_edge"),
default=[],
help="Limit output to one risk category; repeat to select multiple.",
)
args = parser.parse_args()
if args.output.exists():
parser.error(f"output already exists: {args.output}")
if args.min_dimension_pixels <= 0:
parser.error("--min-dimension-pixels must be positive")
if args.extreme_aspect_ratio < 1:
parser.error("--extreme-aspect-ratio must be at least one")
if args.edge_tolerance_pixels < 0:
parser.error("--edge-tolerance-pixels must be non-negative")
summary_path = args.summary_path.expanduser().resolve(strict=True)
summary = json.loads(summary_path.read_text(encoding="utf-8"))
tiles = summary.get("tiles")
if not isinstance(tiles, list):
raise ValueError("dataset summary must contain a tiles list")
tile_size = args.tile_size or summary.get("tile_size")
if not isinstance(tile_size, int) or tile_size <= 0:
raise ValueError("a positive integer tile size is required")
selected_categories = set(
args.category or ("small_dimension", "extreme_aspect_ratio", "tile_edge")
)
category_totals = {
"small_dimension": 0,
"extreme_aspect_ratio": 0,
"tile_edge": 0,
}
reviewed_label_count = 0
unique_flagged_rows: set[tuple[str, int]] = set()
flagged_tiles: list[dict[str, Any]] = []
renderable_tiles: list[dict[str, Any]] = []
for tile in tiles:
if not isinstance(tile, dict) or not tile.get("kept", True):
continue
label_path = resolve_path(str(tile.get("label_path") or ""), summary_path)
boxes = parse_label_file(label_path)
reviewed_label_count += len(boxes)
outliers: list[dict[str, Any]] = []
for index, box in enumerate(boxes):
classifications = classify_box(
box,
tile_size=tile_size,
min_dimension_pixels=args.min_dimension_pixels,
extreme_aspect_ratio=args.extreme_aspect_ratio,
edge_tolerance_pixels=args.edge_tolerance_pixels,
)
for classification in classifications:
if classification["category"] not in selected_categories:
continue
category_totals[classification["category"]] += 1
unique_flagged_rows.add((str(label_path), index))
outliers.append(
{
"index": index,
"box": box.as_list(),
**classification,
}
)
if not outliers:
continue
flagged_tiles.append(
{
"sample_slug": str(tile.get("sample_slug") or "unknown"),
"split": str(tile.get("split") or "unknown"),
"tile_index": int(tile.get("tile_index") or 0),
"label_path": str(label_path),
"label_count": len(boxes),
"outlier_count": len(outliers),
"outliers": outliers,
}
)
renderable_tiles.append(tile)
flagged_tiles.sort(
key=lambda item: (
-item["outlier_count"],
item["sample_slug"],
item["split"],
item["tile_index"],
)
)
payload = {
"schema_version": 1,
"generated_at": datetime.now(UTC).isoformat(),
"status": "attention" if flagged_tiles else "ok",
"claim_boundary": (
"Read-only geometric risk triage; a flagged label is not an automatic "
"ground-truth error or rewrite instruction."
),
"summary_path": str(summary_path),
"summary_sha256": sha256_file(summary_path),
"output_dir": summary.get("output_dir"),
"class_names": summary.get("class_names", []),
"tile_size": tile_size,
"thresholds": {
"min_dimension_pixels": args.min_dimension_pixels,
"extreme_aspect_ratio": args.extreme_aspect_ratio,
"edge_tolerance_pixels": args.edge_tolerance_pixels,
},
"selected_categories": sorted(selected_categories),
"reviewed_tile_count": sum(
1 for tile in tiles if isinstance(tile, dict) and tile.get("kept", True)
),
"reviewed_label_count": reviewed_label_count,
"flagged_tile_count": len(flagged_tiles),
"unique_flagged_label_count": len(unique_flagged_rows),
"category_totals": category_totals,
"flagged_tiles": flagged_tiles,
"tiles": renderable_tiles,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
print(
json.dumps(
{
key: payload[key]
for key in (
"status",
"reviewed_tile_count",
"reviewed_label_count",
"flagged_tile_count",
"unique_flagged_label_count",
"category_totals",
)
},
indent=2,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+53 -4
View File
@@ -45,13 +45,50 @@ def validation_images(dataset_yaml: Path) -> list[Path]:
images = sorted( images = sorted(
path path
for path in directory.iterdir() for path in directory.iterdir()
if path.is_file() and path.suffix.casefold() in {".jpg", ".jpeg", ".png", ".tif", ".tiff"} if path.is_file()
and path.suffix.casefold() in {".jpg", ".jpeg", ".png", ".tif", ".tiff"}
) )
if not images: if not images:
raise ValueError("validation image directory is empty") raise ValueError("validation image directory is empty")
return images return images
def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
summary_path = dataset_yaml.parent / "yolo_tile_dataset_summary.json"
if not summary_path.is_file():
return {
"status": "unavailable",
"summary_path": str(summary_path),
"validation_rows_independent": None,
}
payload = json.loads(summary_path.read_text(encoding="utf-8"))
tile_size = payload.get("tile_size")
stride = payload.get("stride")
if not isinstance(tile_size, int) or not isinstance(stride, int) or stride <= 0:
return {
"status": "invalid",
"summary_path": str(summary_path),
"summary_sha256": sha256_file(summary_path),
"validation_rows_independent": None,
}
overlap_pixels = max(tile_size - stride, 0)
return {
"status": "overlapping" if overlap_pixels else "non_overlapping",
"summary_path": str(summary_path),
"summary_sha256": sha256_file(summary_path),
"tile_size": tile_size,
"stride": stride,
"overlap_pixels": overlap_pixels,
"validation_rows_independent": overlap_pixels == 0,
"interpretation": (
"Tile metrics can repeat the same source object and are valid for "
"candidate ranking only, not independent object-level uncertainty."
if overlap_pixels
else "Tile rows do not overlap according to the dataset summary."
),
}
def metric_value(metrics: Any, attribute: str) -> float: def metric_value(metrics: Any, attribute: str) -> float:
value = getattr(metrics.box, attribute) value = getattr(metrics.box, attribute)
return float(value) return float(value)
@@ -68,7 +105,9 @@ def background_detection_count(
) -> tuple[int, int]: ) -> tuple[int, int]:
selected = [path for path in images if path.stem.casefold().startswith(prefixes)] selected = [path for path in images if path.stem.casefold().startswith(prefixes)]
if not selected: if not selected:
raise ValueError("no validation images match the declared pure-background prefixes") raise ValueError(
"no validation images match the declared pure-background prefixes"
)
count = 0 count = 0
for start in range(0, len(selected), 16): for start in range(0, len(selected), 16):
results = model.predict( results = model.predict(
@@ -102,6 +141,7 @@ def main() -> int:
parser.error("--background-confidence must be between zero and one") parser.error("--background-confidence must be between zero and one")
dataset_yaml = args.dataset_yaml.expanduser().resolve(strict=True) dataset_yaml = args.dataset_yaml.expanduser().resolve(strict=True)
images = validation_images(dataset_yaml) images = validation_images(dataset_yaml)
overlap_evidence = dataset_overlap_evidence(dataset_yaml)
prefixes = tuple(value.casefold() for value in args.background_prefix) prefixes = tuple(value.casefold() for value in args.background_prefix)
import torch import torch
@@ -161,7 +201,13 @@ def main() -> int:
} }
) )
except Exception as exc: # preserve the complete attempted matrix except Exception as exc: # preserve the complete attempted matrix
row.update({"status": "error", "error_type": type(exc).__name__, "error": str(exc)[:1000]}) row.update(
{
"status": "error",
"error_type": type(exc).__name__,
"error": str(exc)[:1000],
}
)
finally: finally:
del model del model
if torch.cuda.is_available(): if torch.cuda.is_available():
@@ -187,6 +233,7 @@ def main() -> int:
"claim_boundary": "Non-protected validation ranking only; no test, challenge or promotion claim.", "claim_boundary": "Non-protected validation ranking only; no test, challenge or promotion claim.",
"dataset_yaml": str(dataset_yaml), "dataset_yaml": str(dataset_yaml),
"dataset_yaml_sha256": sha256_file(dataset_yaml), "dataset_yaml_sha256": sha256_file(dataset_yaml),
"dataset_overlap_evidence": overlap_evidence,
"validation_image_count": len(images), "validation_image_count": len(images),
"pure_background_prefixes": list(prefixes), "pure_background_prefixes": list(prefixes),
"pure_background_confidence": args.background_confidence, "pure_background_confidence": args.background_confidence,
@@ -198,7 +245,9 @@ def main() -> int:
"attempts": rows, "attempts": rows,
} }
args.output.parent.mkdir(parents=True, exist_ok=True) args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") args.output.write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
return 0 if successful else 2 return 0 if successful else 2
@@ -287,6 +287,9 @@ def draw_tile_card(
"exact_duplicate": ((0, 220, 255), 5), "exact_duplicate": ((0, 220, 255), 5),
"near_duplicate": ((255, 55, 55), 5), "near_duplicate": ((255, 55, 55), 5),
"possible_nested": ((255, 0, 220), 5), "possible_nested": ((255, 0, 220), 5),
"extreme_aspect_ratio": ((255, 128, 0), 5),
"small_dimension": ((80, 160, 255), 5),
"tile_edge": ((57, 255, 20), 5),
} }
for box_index, box in enumerate(boxes): for box_index, box in enumerate(boxes):
x_center = box["center_x"] * thumb_size x_center = box["center_x"] * thumb_size
@@ -317,14 +320,17 @@ def draw_tile_card(
return card return card
def build_relationship_highlights( def build_label_highlights(
summary: dict[str, Any], summary: dict[str, Any],
) -> dict[str, dict[int, str]]: ) -> dict[str, dict[int, str]]:
"""Map audited label rows to their highest-priority visual warning.""" """Map audited label rows to their highest-priority visual warning."""
priorities = { priorities = {
"possible_nested": 1, "tile_edge": 1,
"near_duplicate": 2, "small_dimension": 2,
"exact_duplicate": 3, "extreme_aspect_ratio": 3,
"possible_nested": 4,
"near_duplicate": 5,
"exact_duplicate": 6,
} }
highlights: dict[str, dict[int, str]] = {} highlights: dict[str, dict[int, str]] = {}
flagged_tiles = summary.get("flagged_tiles") or [] flagged_tiles = summary.get("flagged_tiles") or []
@@ -355,9 +361,30 @@ def build_relationship_highlights(
): ):
tile_highlights[index] = relationship_type tile_highlights[index] = relationship_type
outliers = tile.get("outliers") or []
if not isinstance(outliers, list):
continue
for outlier in outliers:
if not isinstance(outlier, dict):
continue
category = str(outlier.get("category") or "")
index = outlier.get("index")
if category not in priorities or not isinstance(index, int) or index < 0:
continue
existing = tile_highlights.get(index)
if existing is None or priorities[category] > priorities[existing]:
tile_highlights[index] = category
return highlights return highlights
def build_relationship_highlights(
summary: dict[str, Any],
) -> dict[str, dict[int, str]]:
"""Backward-compatible name for callers using relationship manifests."""
return build_label_highlights(summary)
def image_has_low_visual_variance(image_path: Path, blank_range_threshold: int) -> bool: def image_has_low_visual_variance(image_path: Path, blank_range_threshold: int) -> bool:
image = Image.open(image_path).convert("L") image = Image.open(image_path).convert("L")
min_value, max_value = image.getextrema() min_value, max_value = image.getextrema()
@@ -406,8 +433,8 @@ def build_report(
invalid_label_count = 0 invalid_label_count = 0
valid_label_count = 0 valid_label_count = 0
low_visual_variance_tile_count = 0 low_visual_variance_tile_count = 0
relationship_highlight_count = 0 highlight_category_counts: dict[str, int] = {}
relationship_highlights = build_relationship_highlights(summary) label_highlights = build_label_highlights(summary)
for tile in selected_tiles: for tile in selected_tiles:
image_path = resolve_path(tile.get("image_path"), summary_path) image_path = resolve_path(tile.get("image_path"), summary_path)
@@ -415,8 +442,11 @@ def build_report(
boxes, tile_invalid_count, missing_label_file = parse_yolo_label_file( boxes, tile_invalid_count, missing_label_file = parse_yolo_label_file(
label_path label_path
) )
tile_relationship_highlights = relationship_highlights.get(str(label_path), {}) tile_label_highlights = label_highlights.get(str(label_path), {})
relationship_highlight_count += len(tile_relationship_highlights) for category in tile_label_highlights.values():
highlight_category_counts[category] = (
highlight_category_counts.get(category, 0) + 1
)
invalid_label_count += tile_invalid_count invalid_label_count += tile_invalid_count
valid_label_count += len(boxes) valid_label_count += len(boxes)
if missing_label_file: if missing_label_file:
@@ -441,7 +471,7 @@ def build_report(
invalid_label_count=tile_invalid_count, invalid_label_count=tile_invalid_count,
missing_label_file=missing_label_file, missing_label_file=missing_label_file,
low_visual_variance=low_visual_variance, low_visual_variance=low_visual_variance,
relationship_highlights=tile_relationship_highlights, relationship_highlights=tile_label_highlights,
) )
) )
rendered = True rendered = True
@@ -460,7 +490,7 @@ def build_report(
"invalid_label_count": tile_invalid_count, "invalid_label_count": tile_invalid_count,
"missing_label_file": missing_label_file, "missing_label_file": missing_label_file,
"low_visual_variance": low_visual_variance, "low_visual_variance": low_visual_variance,
"relationship_highlight_count": len(tile_relationship_highlights), "highlighted_label_count": len(tile_label_highlights),
"rendered": rendered, "rendered": rendered,
} }
) )
@@ -478,6 +508,14 @@ def build_report(
} }
for start in range(0, len(rendered_cards), args.tiles_per_sheet) for start in range(0, len(rendered_cards), args.tiles_per_sheet)
] ]
relationship_highlight_count = sum(
highlight_category_counts.get(category, 0)
for category in ("exact_duplicate", "near_duplicate", "possible_nested")
)
outlier_highlight_count = sum(
highlight_category_counts.get(category, 0)
for category in ("extreme_aspect_ratio", "small_dimension", "tile_edge")
)
return ( return (
{ {
@@ -503,7 +541,12 @@ def build_report(
"invalid_label_count": invalid_label_count, "invalid_label_count": invalid_label_count,
"valid_label_count": valid_label_count, "valid_label_count": valid_label_count,
"low_visual_variance_tile_count": low_visual_variance_tile_count, "low_visual_variance_tile_count": low_visual_variance_tile_count,
"highlighted_label_count": sum(highlight_category_counts.values()),
"relationship_highlight_count": relationship_highlight_count, "relationship_highlight_count": relationship_highlight_count,
"outlier_highlight_count": outlier_highlight_count,
"highlight_category_counts": dict(
sorted(highlight_category_counts.items())
),
"blank_range_threshold": args.blank_range_threshold, "blank_range_threshold": args.blank_range_threshold,
"contact_sheets": contact_sheets, "contact_sheets": contact_sheets,
"selected_tiles": selected_report_tiles, "selected_tiles": selected_report_tiles,
@@ -525,7 +568,8 @@ def write_markdown(report: dict[str, Any], output_dir: Path) -> None:
f"- invalid label rows: {report['invalid_label_count']}", f"- invalid label rows: {report['invalid_label_count']}",
f"- low-variance rendered tiles: {report['low_visual_variance_tile_count']}", f"- low-variance rendered tiles: {report['low_visual_variance_tile_count']}",
f"- valid labels rendered: {report['valid_label_count']}", f"- valid labels rendered: {report['valid_label_count']}",
f"- relationship-highlighted labels: {report['relationship_highlight_count']}", f"- highlighted labels: {report['highlighted_label_count']}",
f"- highlight categories: `{report['highlight_category_counts']}`",
"", "",
"## Contact Sheets", "## Contact Sheets",
"", "",
@@ -0,0 +1,50 @@
import pytest
from scripts.audit_yolo_cross_tile_repetition import (
interior_global_key,
tile_offsets,
)
from scripts.audit_yolo_label_relationships import Box
def test_tile_offsets_parse_exporter_filename() -> None:
assert tile_offsets("/data/geel_0005_r256_c512.png") == (256, 512)
with pytest.raises(ValueError, match="no row/column offsets"):
tile_offsets("/data/not-a-tile.png")
def test_interior_global_key_reconstructs_same_object_across_tiles() -> None:
first = Box(0, 0.75, 0.5, 0.1, 0.1)
second = Box(0, 0.25, 0.5, 0.1, 0.1)
first_key = interior_global_key(
first,
row_offset=0,
column_offset=0,
tile_size=512,
edge_tolerance_pixels=0.5,
)
second_key = interior_global_key(
second,
row_offset=0,
column_offset=256,
tile_size=512,
edge_tolerance_pixels=0.5,
)
assert first_key == second_key == (358.4, 230.4, 409.6, 281.6)
def test_interior_global_key_rejects_edge_clipped_box() -> None:
edge_box = Box(0, 0.05, 0.5, 0.1, 0.1)
assert (
interior_global_key(
edge_box,
row_offset=0,
column_offset=256,
tile_size=512,
edge_tolerance_pixels=0.5,
)
is None
)
+45
View File
@@ -0,0 +1,45 @@
from scripts.audit_yolo_label_outliers import classify_box
from scripts.audit_yolo_label_relationships import Box
DEFAULTS = {
"tile_size": 640,
"min_dimension_pixels": 4.0,
"extreme_aspect_ratio": 8.0,
"edge_tolerance_pixels": 0.5,
}
def test_classify_box_reports_independent_geometric_risks() -> None:
box = Box(0, 0.5, 0.003, 0.2, 0.006)
assert classify_box(box, **DEFAULTS) == [
{
"category": "small_dimension",
"width_px": 128.0,
"height_px": 3.84,
"area_px2": 491.52,
"aspect_ratio": 33.333333,
},
{
"category": "extreme_aspect_ratio",
"width_px": 128.0,
"height_px": 3.84,
"area_px2": 491.52,
"aspect_ratio": 33.333333,
},
{
"category": "tile_edge",
"edge_sides": ["top"],
"width_px": 128.0,
"height_px": 3.84,
"area_px2": 491.52,
"aspect_ratio": 33.333333,
},
]
def test_classify_box_ignores_ordinary_interior_box() -> None:
box = Box(0, 0.5, 0.5, 0.1, 0.08)
assert classify_box(box, **DEFAULTS) == []
@@ -0,0 +1,29 @@
import json
from pathlib import Path
from scripts.evaluate_yolo_checkpoint_matrix import dataset_overlap_evidence
def test_dataset_overlap_evidence_marks_repeated_validation_rows(
tmp_path: Path,
) -> None:
dataset_yaml = tmp_path / "dataset.yaml"
dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8")
(tmp_path / "yolo_tile_dataset_summary.json").write_text(
json.dumps({"tile_size": 512, "stride": 256}), encoding="utf-8"
)
evidence = dataset_overlap_evidence(dataset_yaml)
assert evidence["status"] == "overlapping"
assert evidence["overlap_pixels"] == 256
assert evidence["validation_rows_independent"] is False
def test_dataset_overlap_evidence_is_explicit_when_summary_missing(
tmp_path: Path,
) -> None:
evidence = dataset_overlap_evidence(tmp_path / "dataset.yaml")
assert evidence["status"] == "unavailable"
assert evidence["validation_rows_independent"] is None
@@ -2,6 +2,7 @@ import pytest
from scripts.render_operator_yolo_label_qa_contact_sheets import ( from scripts.render_operator_yolo_label_qa_contact_sheets import (
CONTACT_SHEET_NAME_TEMPLATE, CONTACT_SHEET_NAME_TEMPLATE,
build_label_highlights,
build_relationship_highlights, build_relationship_highlights,
filter_tiles_by_samples, filter_tiles_by_samples,
) )
@@ -72,3 +73,25 @@ def test_relationship_highlights_bind_indices_and_preserve_highest_priority() ->
7: "near_duplicate", 7: "near_duplicate",
} }
} }
def test_label_highlights_support_outliers_with_stable_priority() -> None:
summary = {
"flagged_tiles": [
{
"label_path": "/data/tile.txt",
"outliers": [
{"category": "tile_edge", "index": 3},
{"category": "small_dimension", "index": 3},
{"category": "extreme_aspect_ratio", "index": 4},
],
}
]
}
assert build_label_highlights(summary) == {
"/data/tile.txt": {
3: "small_dimension",
4: "extreme_aspect_ratio",
}
}