audit YOLO geometry and overlapping validation rows
This commit is contained in:
+32
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"ledger_id": "reviewedexp6-ai-assisted-review-r2-full-corpus",
|
||||
"reviewed_at": "2026-08-09T19:10:57+02:00",
|
||||
"reviewed_at": "2026-08-09T19:34:31+02:00",
|
||||
"reviewer": {
|
||||
"id": "openai-codex",
|
||||
"type": "ai_assistant",
|
||||
@@ -96,5 +96,36 @@
|
||||
"share_of_rendered_labels": 0.000543,
|
||||
"ai_visual_followup": "all 86 implicated label rows were index-highlighted; no cyan exact duplicates or red near duplicates exist, while the 43 magenta nesting pairs are dispersed across dense reference AOIs and predominantly show larger GRB envelopes around smaller building components rather than one systematic duplicate-label pattern",
|
||||
"action": "retain for experimental analysis, require human adjudication before release, and perform no automatic rewrite or exclusion"
|
||||
},
|
||||
"geometric_outlier_audit": {
|
||||
"audit_path": "/app/storage/operator-data/model-review/reviewedexp6-corpus/full-ai-review-r1/label_outlier_audit.json",
|
||||
"audit_sha256": "cac2c0cefb49aa3785ee9fb173a0187b3dd8abfc9c25e8e91df6c4af91ff5a40",
|
||||
"reviewed_label_count": 79192,
|
||||
"unique_flagged_label_count": 6657,
|
||||
"small_dimension_below_4_px": 1812,
|
||||
"extreme_aspect_ratio_at_least_8": 26,
|
||||
"tile_edge_rows": 5079,
|
||||
"extreme_rows_also_below_4_px": 16,
|
||||
"extreme_contact_sheet_sha256": "67ad11e7d80678c78157a01a5e7ceb7cff9ac24410b127bdfaafd4350170cdb6",
|
||||
"small_dimension_contact_sheet_sha256s": [
|
||||
"5b3ca4d5bfd57b466854a00b4791ac82cf4a826fad8a4694d0227c931b00dfad",
|
||||
"0e5d2058154f8298fc6d2c064150bcf896e4e51b9bcce5fccbfc049e1ebb9515",
|
||||
"03060f65843f3c2a291dd009124484f4cf798fe653e658bab007375613fa3010",
|
||||
"2fe7c748a68b21a673f3d950909b07e3ddba81b736977fbee8d64ea62b422ca2"
|
||||
],
|
||||
"tile_edge_sample_contact_sheet_sha256": "5d96c5a0a732077ed16b22d33bed998b37bd33b5536b1491e778999a2a6638dd",
|
||||
"ai_visual_followup": "extreme rows are mostly plausible elongated structures, while sub-4-pixel rows are often visually marginal; tile-edge rows are expected clipped objects under the 0.35 visibility policy",
|
||||
"next_corpus_decision": "restore the exporter minimum dimension to at least 4 px for the next immutable experimental corpus; retain elongated structures above that floor and measure any higher edge-visibility threshold in a separately versioned ablation"
|
||||
},
|
||||
"cross_tile_repetition_audit": {
|
||||
"audit_path": "/app/storage/operator-data/model-review/reviewedexp6-corpus/full-ai-review-r1/cross_tile_repetition_audit.json",
|
||||
"audit_sha256": "3cdf5174fec61e0aa426b4a3f44ed943a738bf038f3b58028ec856ef1a9d000c",
|
||||
"interior_label_rows": 74113,
|
||||
"unique_interior_objects": 34082,
|
||||
"extra_repeated_rows": 40031,
|
||||
"repeated_object_groups": 24701,
|
||||
"max_repetition": 4,
|
||||
"cross_split_repetition_groups": 0,
|
||||
"interpretation": "the 512 px tiles use stride 256, so tile-level validation rows are not statistically independent even though no reconstructed object crosses train and validation splits"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12595,3 +12595,46 @@ Open:
|
||||
- This is explicit AI-assisted triage, not human adjudication. It improves the
|
||||
evidence and review precision but does not unlock governed production
|
||||
training or checkpoint promotion.
|
||||
|
||||
## 2026-08-09 - Row-level geometry and cross-tile independence audit
|
||||
|
||||
### Improved
|
||||
|
||||
- Added a reproducible, read-only label-row auditor for sub-threshold pixel
|
||||
dimensions, extreme aspect ratios and tile-edge clipping, with separate
|
||||
category manifests accepted directly by the visual QA renderer.
|
||||
- Extended the renderer with orange extreme-aspect, blue small-dimension and
|
||||
green tile-edge highlighting while preserving the existing duplicate/nesting
|
||||
colors and source row indices.
|
||||
- Added an exact cross-tile repetition auditor that reconstructs global pixel
|
||||
boxes from the exporter's `_r<row>_c<column>` offsets. Edge-clipped rows stay
|
||||
explicitly unlinked rather than being guessed back to source objects.
|
||||
- Made checkpoint-matrix output disclose dataset tile overlap and explicitly
|
||||
state whether validation rows are independent whenever a dataset summary is
|
||||
available.
|
||||
|
||||
### Tower findings
|
||||
|
||||
- Reviewed all 79,192 rows: 1,812 have a dimension below 4 px, 26 have aspect
|
||||
ratio at least 8 and 5,079 touch a tile edge; 6,657 unique rows are involved.
|
||||
- Inspected the complete 23-tile extreme sheet, all four small-dimension pages
|
||||
covering 198 tiles, and a 64-tile edge sample. Most extreme shapes are valid
|
||||
elongated structures, but 16 of 26 also fall below the 4 px floor.
|
||||
- Reconstructed 34,082 unique interior objects from 74,113 interior label rows.
|
||||
Tile overlap adds 40,031 exact repeated rows across 24,701 object groups,
|
||||
with maximum repetition four and zero cross-split groups.
|
||||
- Froze the next-corpus decision: restore `min_label_px >= 4`; test any stricter
|
||||
edge-visible ratio as a separate immutable ablation rather than modifying
|
||||
this corpus in place.
|
||||
|
||||
### Verified
|
||||
|
||||
- Focused auditor, renderer and checkpoint-evidence suite: 18 tests passed.
|
||||
- Scoped Ruff check, format check and Git whitespace check passed.
|
||||
|
||||
### Remaining limitations
|
||||
|
||||
- Existing overlapping-tile mAP is valid for relative non-protected ranking,
|
||||
not independent object-level accuracy. A future benchmark must aggregate or
|
||||
deduplicate objects before uncertainty claims.
|
||||
- No source label, frozen dataset, model weight or live runtime was changed.
|
||||
|
||||
@@ -1159,6 +1159,14 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
contact sheet; AI follow-up found no systematic duplicate-label pattern.
|
||||
- [x] Bind the relationship audit back to exact label-row indices and render
|
||||
all 86 implicated rows with type-specific highlighting for focused review.
|
||||
- [x] Audit all 79,192 labels at row level for sub-4-pixel dimensions, extreme
|
||||
aspect ratio and tile-edge clipping; inspect the complete small/extreme
|
||||
sheets and retain immutable hashes.
|
||||
- [x] Reconstruct exact cross-tile interior objects and prove that overlap
|
||||
repeats 40,031 label rows but causes zero reconstructed train/val crossings.
|
||||
- [ ] Build the next immutable experimental corpus with `min_label_px >= 4`
|
||||
and compare the existing 0.35 edge-visibility policy against a separately
|
||||
versioned stricter ablation before any new governed training.
|
||||
- [ ] Convert the AI-assisted ledger into no stronger claim than experimental
|
||||
triage; a real human must independently review and sign the frozen artifacts
|
||||
before the governed training wrapper may unlock.
|
||||
|
||||
@@ -23,6 +23,15 @@ The checkpoint matrix used only the 36-image non-protected validation split:
|
||||
measured mAP50 `0.345426` and mAP50-95 `0.140719`. The `reviewedexp6`
|
||||
challenger measured `0.368177` and `0.155438` respectively.
|
||||
|
||||
A later exact cross-tile reconstruction established that this 512 px corpus
|
||||
uses stride 256. Across the full corpus, 74,113 interior label rows represent
|
||||
34,082 unique reconstructed objects; 40,031 rows are overlap repetitions and
|
||||
one object can occur four times. No reconstructed object crosses the
|
||||
train/validation boundary, but the tile rows are not statistically independent.
|
||||
The matrix therefore remains useful only for relative non-protected candidate
|
||||
ranking. Its mAP values are not an independent object-level accuracy estimate
|
||||
or a basis for narrow confidence claims.
|
||||
|
||||
At confidence `0.25` and match IoU `0.25`, the active model measured F1
|
||||
`0.633058`; the challenger measured `0.642599`. Both produced zero detections
|
||||
on the 18 pure-background validation tiles at this threshold. At confidence
|
||||
@@ -129,3 +138,26 @@ rewrite candidates. Excluding the 36 complete tiles would also discard 17,167
|
||||
unflagged labels, so no automatic tile exclusion or label mutation was made.
|
||||
The checksum-bound highlighted sheet and summary are recorded in the AI review
|
||||
ledger.
|
||||
|
||||
## Row-level geometry and overlap follow-up
|
||||
|
||||
The complete 79,192-label corpus was additionally audited at label-row level.
|
||||
The immutable manifest identifies 6,657 unique rows with at least one geometric
|
||||
training-risk signal: 1,812 rows have a width or height below 4 pixels, 26 have
|
||||
aspect ratio at least 8 and 5,079 touch a tile edge. The categories were
|
||||
rendered separately. All 23 extreme-aspect tiles and all 198 small-dimension
|
||||
tiles were inspected, plus a 64-tile edge sample.
|
||||
|
||||
The extreme-aspect group predominantly shows plausible elongated sheds and
|
||||
building components. Sixteen of its 26 rows are also below 4 pixels, allowing
|
||||
the resolution floor to address most ambiguous extremes without deleting valid
|
||||
long structures. The small-dimension sheets contain many visually marginal
|
||||
miniature targets. The next immutable experimental corpus should therefore use
|
||||
the exporter's normal minimum dimension of at least 4 pixels instead of this
|
||||
legacy corpus's 3-pixel override. This finding does not justify mutating the
|
||||
frozen corpus or retroactively changing its checkpoint.
|
||||
|
||||
The edge sample shows expected clipped buildings under the 0.35 minimum-visible
|
||||
policy. Raising that value may reduce partial-target pressure, but it must be a
|
||||
separately versioned ablation because removing all 5,079 rows without checking
|
||||
the original visible fraction would be unsound.
|
||||
|
||||
@@ -20,6 +20,14 @@ files. Its output retains the flagged source tiles so it can be passed directly
|
||||
to the contact-sheet renderer. Possible nesting remains review evidence and is
|
||||
never treated as an automatic label error or rewrite instruction.
|
||||
|
||||
`audit_yolo_label_outliers.py` retains row-level evidence for sub-threshold
|
||||
pixel dimensions, extreme aspect ratios and tile-edge labels. Categories can
|
||||
be emitted separately with repeated `--category` arguments and passed directly
|
||||
to the contact-sheet renderer. `audit_yolo_cross_tile_repetition.py`
|
||||
reconstructs global pixel boxes from exporter tile offsets to quantify exact
|
||||
interior-object repetition caused by overlap; edge rows remain explicitly
|
||||
unlinked and no repetition is automatically classified as an error.
|
||||
|
||||
Setup-, import-, demo- en maintenance-scripts voor GeoIntel.
|
||||
|
||||
## WALOUS source provisioning
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Quantify exact interior-object repetition caused by overlapping YOLO tiles."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from scripts.audit_yolo_label_relationships import (
|
||||
Box,
|
||||
parse_label_file,
|
||||
resolve_path,
|
||||
sha256_file,
|
||||
)
|
||||
except ModuleNotFoundError: # Standalone operator-tool copy beside the auditor.
|
||||
from audit_yolo_label_relationships import (
|
||||
Box,
|
||||
parse_label_file,
|
||||
resolve_path,
|
||||
sha256_file,
|
||||
)
|
||||
|
||||
|
||||
OFFSET_PATTERN = re.compile(r"_r(?P<row>\d+)_c(?P<column>\d+)$")
|
||||
|
||||
|
||||
def tile_offsets(image_path: str) -> tuple[int, int]:
|
||||
match = OFFSET_PATTERN.search(Path(image_path).stem)
|
||||
if match is None:
|
||||
raise ValueError(f"tile filename has no row/column offsets: {image_path}")
|
||||
return int(match["row"]), int(match["column"])
|
||||
|
||||
|
||||
def interior_global_key(
|
||||
box: Box,
|
||||
*,
|
||||
row_offset: int,
|
||||
column_offset: int,
|
||||
tile_size: int,
|
||||
edge_tolerance_pixels: float,
|
||||
) -> tuple[float, float, float, float] | None:
|
||||
left, top, right, bottom = box.coordinates
|
||||
pixel_box = (
|
||||
left * tile_size,
|
||||
top * tile_size,
|
||||
right * tile_size,
|
||||
bottom * tile_size,
|
||||
)
|
||||
if (
|
||||
pixel_box[0] <= edge_tolerance_pixels
|
||||
or pixel_box[1] <= edge_tolerance_pixels
|
||||
or tile_size - pixel_box[2] <= edge_tolerance_pixels
|
||||
or tile_size - pixel_box[3] <= edge_tolerance_pixels
|
||||
):
|
||||
return None
|
||||
return (
|
||||
round(column_offset + pixel_box[0], 3),
|
||||
round(row_offset + pixel_box[1], 3),
|
||||
round(column_offset + pixel_box[2], 3),
|
||||
round(row_offset + pixel_box[3], 3),
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--summary-path", required=True, type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--edge-tolerance-pixels", type=float, default=0.5)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
parser.error(f"output already exists: {args.output}")
|
||||
if args.edge_tolerance_pixels < 0:
|
||||
parser.error("--edge-tolerance-pixels must be non-negative")
|
||||
|
||||
summary_path = args.summary_path.expanduser().resolve(strict=True)
|
||||
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||
tiles = summary.get("tiles")
|
||||
tile_size = summary.get("tile_size")
|
||||
if not isinstance(tiles, list):
|
||||
raise ValueError("dataset summary must contain a tiles list")
|
||||
if not isinstance(tile_size, int) or tile_size <= 0:
|
||||
raise ValueError("dataset summary must contain a positive integer tile_size")
|
||||
|
||||
groups: dict[tuple[str, int, float, float, float, float], list[dict[str, Any]]] = (
|
||||
defaultdict(list)
|
||||
)
|
||||
reviewed_label_count = 0
|
||||
edge_label_count = 0
|
||||
for tile in tiles:
|
||||
if not isinstance(tile, dict) or not tile.get("kept", True):
|
||||
continue
|
||||
image_path = str(tile.get("image_path") or "")
|
||||
row_offset, column_offset = tile_offsets(image_path)
|
||||
label_path = resolve_path(str(tile.get("label_path") or ""), summary_path)
|
||||
boxes = parse_label_file(label_path)
|
||||
reviewed_label_count += len(boxes)
|
||||
for index, box in enumerate(boxes):
|
||||
global_key = interior_global_key(
|
||||
box,
|
||||
row_offset=row_offset,
|
||||
column_offset=column_offset,
|
||||
tile_size=tile_size,
|
||||
edge_tolerance_pixels=args.edge_tolerance_pixels,
|
||||
)
|
||||
if global_key is None:
|
||||
edge_label_count += 1
|
||||
continue
|
||||
key = (
|
||||
str(tile.get("sample_slug") or "unknown"),
|
||||
box.class_id,
|
||||
*global_key,
|
||||
)
|
||||
groups[key].append(
|
||||
{
|
||||
"split": str(tile.get("split") or "unknown"),
|
||||
"tile_index": int(tile.get("tile_index") or 0),
|
||||
"label_index": index,
|
||||
"image_path": image_path,
|
||||
"label_path": str(label_path),
|
||||
}
|
||||
)
|
||||
|
||||
repeated_groups = [members for members in groups.values() if len(members) > 1]
|
||||
split_leakage_groups = [
|
||||
members
|
||||
for members in repeated_groups
|
||||
if len({member["split"] for member in members}) > 1
|
||||
]
|
||||
repetition_histogram: dict[str, int] = defaultdict(int)
|
||||
for members in groups.values():
|
||||
repetition_histogram[str(len(members))] += 1
|
||||
sample_stats: dict[str, dict[str, int]] = defaultdict(
|
||||
lambda: {
|
||||
"unique_interior_objects": 0,
|
||||
"interior_label_rows": 0,
|
||||
"repeated_object_groups": 0,
|
||||
"extra_repeated_rows": 0,
|
||||
}
|
||||
)
|
||||
for key, members in groups.items():
|
||||
sample = sample_stats[key[0]]
|
||||
sample["unique_interior_objects"] += 1
|
||||
sample["interior_label_rows"] += len(members)
|
||||
if len(members) > 1:
|
||||
sample["repeated_object_groups"] += 1
|
||||
sample["extra_repeated_rows"] += len(members) - 1
|
||||
|
||||
interior_label_count = sum(len(members) for members in groups.values())
|
||||
extra_repeated_rows = sum(len(members) - 1 for members in repeated_groups)
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(UTC).isoformat(),
|
||||
"status": "attention" if repeated_groups else "ok",
|
||||
"claim_boundary": (
|
||||
"Exact reconstructed repetition among interior boxes only; tile-edge "
|
||||
"rows remain unlinked and repetition is not automatically an error."
|
||||
),
|
||||
"summary_path": str(summary_path),
|
||||
"summary_sha256": sha256_file(summary_path),
|
||||
"tile_size": tile_size,
|
||||
"stride": summary.get("stride"),
|
||||
"edge_tolerance_pixels": args.edge_tolerance_pixels,
|
||||
"reviewed_label_count": reviewed_label_count,
|
||||
"interior_label_count": interior_label_count,
|
||||
"edge_label_count": edge_label_count,
|
||||
"unique_interior_object_count": len(groups),
|
||||
"repeated_object_group_count": len(repeated_groups),
|
||||
"extra_repeated_label_row_count": extra_repeated_rows,
|
||||
"max_repetition": max((len(members) for members in groups.values()), default=0),
|
||||
"repetition_histogram": dict(
|
||||
sorted(repetition_histogram.items(), key=lambda item: int(item[0]))
|
||||
),
|
||||
"cross_split_repetition_group_count": len(split_leakage_groups),
|
||||
"sample_stats": [
|
||||
{"sample_slug": slug, **stats}
|
||||
for slug, stats in sorted(sample_stats.items())
|
||||
],
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(
|
||||
json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
print(json.dumps(payload, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit individual YOLO rows for geometric training risks without mutation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from scripts.audit_yolo_label_relationships import (
|
||||
Box,
|
||||
parse_label_file,
|
||||
resolve_path,
|
||||
sha256_file,
|
||||
)
|
||||
except ModuleNotFoundError: # Standalone operator-tool copy beside the auditor.
|
||||
from audit_yolo_label_relationships import (
|
||||
Box,
|
||||
parse_label_file,
|
||||
resolve_path,
|
||||
sha256_file,
|
||||
)
|
||||
|
||||
|
||||
def classify_box(
|
||||
box: Box,
|
||||
*,
|
||||
tile_size: int,
|
||||
min_dimension_pixels: float,
|
||||
extreme_aspect_ratio: float,
|
||||
edge_tolerance_pixels: float,
|
||||
) -> list[dict[str, Any]]:
|
||||
width_px = box.width * tile_size
|
||||
height_px = box.height * tile_size
|
||||
aspect_ratio = max(width_px / height_px, height_px / width_px)
|
||||
left, top, right, bottom = box.coordinates
|
||||
edge_sides = [
|
||||
side
|
||||
for side, distance in (
|
||||
("left", left * tile_size),
|
||||
("top", top * tile_size),
|
||||
("right", (1 - right) * tile_size),
|
||||
("bottom", (1 - bottom) * tile_size),
|
||||
)
|
||||
if distance <= edge_tolerance_pixels
|
||||
]
|
||||
metrics = {
|
||||
"width_px": round(width_px, 6),
|
||||
"height_px": round(height_px, 6),
|
||||
"area_px2": round(width_px * height_px, 6),
|
||||
"aspect_ratio": round(aspect_ratio, 6),
|
||||
}
|
||||
outliers: list[dict[str, Any]] = []
|
||||
if min(width_px, height_px) < min_dimension_pixels:
|
||||
outliers.append({"category": "small_dimension", **metrics})
|
||||
if aspect_ratio >= extreme_aspect_ratio:
|
||||
outliers.append({"category": "extreme_aspect_ratio", **metrics})
|
||||
if edge_sides:
|
||||
outliers.append({"category": "tile_edge", "edge_sides": edge_sides, **metrics})
|
||||
return outliers
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--summary-path", required=True, type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--tile-size", type=int)
|
||||
parser.add_argument("--min-dimension-pixels", type=float, default=4.0)
|
||||
parser.add_argument("--extreme-aspect-ratio", type=float, default=8.0)
|
||||
parser.add_argument("--edge-tolerance-pixels", type=float, default=0.5)
|
||||
parser.add_argument(
|
||||
"--category",
|
||||
action="append",
|
||||
choices=("small_dimension", "extreme_aspect_ratio", "tile_edge"),
|
||||
default=[],
|
||||
help="Limit output to one risk category; repeat to select multiple.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.output.exists():
|
||||
parser.error(f"output already exists: {args.output}")
|
||||
if args.min_dimension_pixels <= 0:
|
||||
parser.error("--min-dimension-pixels must be positive")
|
||||
if args.extreme_aspect_ratio < 1:
|
||||
parser.error("--extreme-aspect-ratio must be at least one")
|
||||
if args.edge_tolerance_pixels < 0:
|
||||
parser.error("--edge-tolerance-pixels must be non-negative")
|
||||
|
||||
summary_path = args.summary_path.expanduser().resolve(strict=True)
|
||||
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||
tiles = summary.get("tiles")
|
||||
if not isinstance(tiles, list):
|
||||
raise ValueError("dataset summary must contain a tiles list")
|
||||
tile_size = args.tile_size or summary.get("tile_size")
|
||||
if not isinstance(tile_size, int) or tile_size <= 0:
|
||||
raise ValueError("a positive integer tile size is required")
|
||||
|
||||
selected_categories = set(
|
||||
args.category or ("small_dimension", "extreme_aspect_ratio", "tile_edge")
|
||||
)
|
||||
category_totals = {
|
||||
"small_dimension": 0,
|
||||
"extreme_aspect_ratio": 0,
|
||||
"tile_edge": 0,
|
||||
}
|
||||
reviewed_label_count = 0
|
||||
unique_flagged_rows: set[tuple[str, int]] = set()
|
||||
flagged_tiles: list[dict[str, Any]] = []
|
||||
renderable_tiles: list[dict[str, Any]] = []
|
||||
|
||||
for tile in tiles:
|
||||
if not isinstance(tile, dict) or not tile.get("kept", True):
|
||||
continue
|
||||
label_path = resolve_path(str(tile.get("label_path") or ""), summary_path)
|
||||
boxes = parse_label_file(label_path)
|
||||
reviewed_label_count += len(boxes)
|
||||
outliers: list[dict[str, Any]] = []
|
||||
for index, box in enumerate(boxes):
|
||||
classifications = classify_box(
|
||||
box,
|
||||
tile_size=tile_size,
|
||||
min_dimension_pixels=args.min_dimension_pixels,
|
||||
extreme_aspect_ratio=args.extreme_aspect_ratio,
|
||||
edge_tolerance_pixels=args.edge_tolerance_pixels,
|
||||
)
|
||||
for classification in classifications:
|
||||
if classification["category"] not in selected_categories:
|
||||
continue
|
||||
category_totals[classification["category"]] += 1
|
||||
unique_flagged_rows.add((str(label_path), index))
|
||||
outliers.append(
|
||||
{
|
||||
"index": index,
|
||||
"box": box.as_list(),
|
||||
**classification,
|
||||
}
|
||||
)
|
||||
if not outliers:
|
||||
continue
|
||||
flagged_tiles.append(
|
||||
{
|
||||
"sample_slug": str(tile.get("sample_slug") or "unknown"),
|
||||
"split": str(tile.get("split") or "unknown"),
|
||||
"tile_index": int(tile.get("tile_index") or 0),
|
||||
"label_path": str(label_path),
|
||||
"label_count": len(boxes),
|
||||
"outlier_count": len(outliers),
|
||||
"outliers": outliers,
|
||||
}
|
||||
)
|
||||
renderable_tiles.append(tile)
|
||||
|
||||
flagged_tiles.sort(
|
||||
key=lambda item: (
|
||||
-item["outlier_count"],
|
||||
item["sample_slug"],
|
||||
item["split"],
|
||||
item["tile_index"],
|
||||
)
|
||||
)
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(UTC).isoformat(),
|
||||
"status": "attention" if flagged_tiles else "ok",
|
||||
"claim_boundary": (
|
||||
"Read-only geometric risk triage; a flagged label is not an automatic "
|
||||
"ground-truth error or rewrite instruction."
|
||||
),
|
||||
"summary_path": str(summary_path),
|
||||
"summary_sha256": sha256_file(summary_path),
|
||||
"output_dir": summary.get("output_dir"),
|
||||
"class_names": summary.get("class_names", []),
|
||||
"tile_size": tile_size,
|
||||
"thresholds": {
|
||||
"min_dimension_pixels": args.min_dimension_pixels,
|
||||
"extreme_aspect_ratio": args.extreme_aspect_ratio,
|
||||
"edge_tolerance_pixels": args.edge_tolerance_pixels,
|
||||
},
|
||||
"selected_categories": sorted(selected_categories),
|
||||
"reviewed_tile_count": sum(
|
||||
1 for tile in tiles if isinstance(tile, dict) and tile.get("kept", True)
|
||||
),
|
||||
"reviewed_label_count": reviewed_label_count,
|
||||
"flagged_tile_count": len(flagged_tiles),
|
||||
"unique_flagged_label_count": len(unique_flagged_rows),
|
||||
"category_totals": category_totals,
|
||||
"flagged_tiles": flagged_tiles,
|
||||
"tiles": renderable_tiles,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(
|
||||
json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
key: payload[key]
|
||||
for key in (
|
||||
"status",
|
||||
"reviewed_tile_count",
|
||||
"reviewed_label_count",
|
||||
"flagged_tile_count",
|
||||
"unique_flagged_label_count",
|
||||
"category_totals",
|
||||
)
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -45,13 +45,50 @@ def validation_images(dataset_yaml: Path) -> list[Path]:
|
||||
images = sorted(
|
||||
path
|
||||
for path in directory.iterdir()
|
||||
if path.is_file() and path.suffix.casefold() in {".jpg", ".jpeg", ".png", ".tif", ".tiff"}
|
||||
if path.is_file()
|
||||
and path.suffix.casefold() in {".jpg", ".jpeg", ".png", ".tif", ".tiff"}
|
||||
)
|
||||
if not images:
|
||||
raise ValueError("validation image directory is empty")
|
||||
return images
|
||||
|
||||
|
||||
def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
|
||||
summary_path = dataset_yaml.parent / "yolo_tile_dataset_summary.json"
|
||||
if not summary_path.is_file():
|
||||
return {
|
||||
"status": "unavailable",
|
||||
"summary_path": str(summary_path),
|
||||
"validation_rows_independent": None,
|
||||
}
|
||||
payload = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||
tile_size = payload.get("tile_size")
|
||||
stride = payload.get("stride")
|
||||
if not isinstance(tile_size, int) or not isinstance(stride, int) or stride <= 0:
|
||||
return {
|
||||
"status": "invalid",
|
||||
"summary_path": str(summary_path),
|
||||
"summary_sha256": sha256_file(summary_path),
|
||||
"validation_rows_independent": None,
|
||||
}
|
||||
overlap_pixels = max(tile_size - stride, 0)
|
||||
return {
|
||||
"status": "overlapping" if overlap_pixels else "non_overlapping",
|
||||
"summary_path": str(summary_path),
|
||||
"summary_sha256": sha256_file(summary_path),
|
||||
"tile_size": tile_size,
|
||||
"stride": stride,
|
||||
"overlap_pixels": overlap_pixels,
|
||||
"validation_rows_independent": overlap_pixels == 0,
|
||||
"interpretation": (
|
||||
"Tile metrics can repeat the same source object and are valid for "
|
||||
"candidate ranking only, not independent object-level uncertainty."
|
||||
if overlap_pixels
|
||||
else "Tile rows do not overlap according to the dataset summary."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def metric_value(metrics: Any, attribute: str) -> float:
|
||||
value = getattr(metrics.box, attribute)
|
||||
return float(value)
|
||||
@@ -68,7 +105,9 @@ def background_detection_count(
|
||||
) -> tuple[int, int]:
|
||||
selected = [path for path in images if path.stem.casefold().startswith(prefixes)]
|
||||
if not selected:
|
||||
raise ValueError("no validation images match the declared pure-background prefixes")
|
||||
raise ValueError(
|
||||
"no validation images match the declared pure-background prefixes"
|
||||
)
|
||||
count = 0
|
||||
for start in range(0, len(selected), 16):
|
||||
results = model.predict(
|
||||
@@ -102,6 +141,7 @@ def main() -> int:
|
||||
parser.error("--background-confidence must be between zero and one")
|
||||
dataset_yaml = args.dataset_yaml.expanduser().resolve(strict=True)
|
||||
images = validation_images(dataset_yaml)
|
||||
overlap_evidence = dataset_overlap_evidence(dataset_yaml)
|
||||
prefixes = tuple(value.casefold() for value in args.background_prefix)
|
||||
|
||||
import torch
|
||||
@@ -161,7 +201,13 @@ def main() -> int:
|
||||
}
|
||||
)
|
||||
except Exception as exc: # preserve the complete attempted matrix
|
||||
row.update({"status": "error", "error_type": type(exc).__name__, "error": str(exc)[:1000]})
|
||||
row.update(
|
||||
{
|
||||
"status": "error",
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc)[:1000],
|
||||
}
|
||||
)
|
||||
finally:
|
||||
del model
|
||||
if torch.cuda.is_available():
|
||||
@@ -187,6 +233,7 @@ def main() -> int:
|
||||
"claim_boundary": "Non-protected validation ranking only; no test, challenge or promotion claim.",
|
||||
"dataset_yaml": str(dataset_yaml),
|
||||
"dataset_yaml_sha256": sha256_file(dataset_yaml),
|
||||
"dataset_overlap_evidence": overlap_evidence,
|
||||
"validation_image_count": len(images),
|
||||
"pure_background_prefixes": list(prefixes),
|
||||
"pure_background_confidence": args.background_confidence,
|
||||
@@ -198,7 +245,9 @@ def main() -> int:
|
||||
"attempts": rows,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
args.output.write_text(
|
||||
json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
return 0 if successful else 2
|
||||
|
||||
|
||||
|
||||
@@ -287,6 +287,9 @@ def draw_tile_card(
|
||||
"exact_duplicate": ((0, 220, 255), 5),
|
||||
"near_duplicate": ((255, 55, 55), 5),
|
||||
"possible_nested": ((255, 0, 220), 5),
|
||||
"extreme_aspect_ratio": ((255, 128, 0), 5),
|
||||
"small_dimension": ((80, 160, 255), 5),
|
||||
"tile_edge": ((57, 255, 20), 5),
|
||||
}
|
||||
for box_index, box in enumerate(boxes):
|
||||
x_center = box["center_x"] * thumb_size
|
||||
@@ -317,14 +320,17 @@ def draw_tile_card(
|
||||
return card
|
||||
|
||||
|
||||
def build_relationship_highlights(
|
||||
def build_label_highlights(
|
||||
summary: dict[str, Any],
|
||||
) -> dict[str, dict[int, str]]:
|
||||
"""Map audited label rows to their highest-priority visual warning."""
|
||||
priorities = {
|
||||
"possible_nested": 1,
|
||||
"near_duplicate": 2,
|
||||
"exact_duplicate": 3,
|
||||
"tile_edge": 1,
|
||||
"small_dimension": 2,
|
||||
"extreme_aspect_ratio": 3,
|
||||
"possible_nested": 4,
|
||||
"near_duplicate": 5,
|
||||
"exact_duplicate": 6,
|
||||
}
|
||||
highlights: dict[str, dict[int, str]] = {}
|
||||
flagged_tiles = summary.get("flagged_tiles") or []
|
||||
@@ -355,9 +361,30 @@ def build_relationship_highlights(
|
||||
):
|
||||
tile_highlights[index] = relationship_type
|
||||
|
||||
outliers = tile.get("outliers") or []
|
||||
if not isinstance(outliers, list):
|
||||
continue
|
||||
for outlier in outliers:
|
||||
if not isinstance(outlier, dict):
|
||||
continue
|
||||
category = str(outlier.get("category") or "")
|
||||
index = outlier.get("index")
|
||||
if category not in priorities or not isinstance(index, int) or index < 0:
|
||||
continue
|
||||
existing = tile_highlights.get(index)
|
||||
if existing is None or priorities[category] > priorities[existing]:
|
||||
tile_highlights[index] = category
|
||||
|
||||
return highlights
|
||||
|
||||
|
||||
def build_relationship_highlights(
|
||||
summary: dict[str, Any],
|
||||
) -> dict[str, dict[int, str]]:
|
||||
"""Backward-compatible name for callers using relationship manifests."""
|
||||
return build_label_highlights(summary)
|
||||
|
||||
|
||||
def image_has_low_visual_variance(image_path: Path, blank_range_threshold: int) -> bool:
|
||||
image = Image.open(image_path).convert("L")
|
||||
min_value, max_value = image.getextrema()
|
||||
@@ -406,8 +433,8 @@ def build_report(
|
||||
invalid_label_count = 0
|
||||
valid_label_count = 0
|
||||
low_visual_variance_tile_count = 0
|
||||
relationship_highlight_count = 0
|
||||
relationship_highlights = build_relationship_highlights(summary)
|
||||
highlight_category_counts: dict[str, int] = {}
|
||||
label_highlights = build_label_highlights(summary)
|
||||
|
||||
for tile in selected_tiles:
|
||||
image_path = resolve_path(tile.get("image_path"), summary_path)
|
||||
@@ -415,8 +442,11 @@ def build_report(
|
||||
boxes, tile_invalid_count, missing_label_file = parse_yolo_label_file(
|
||||
label_path
|
||||
)
|
||||
tile_relationship_highlights = relationship_highlights.get(str(label_path), {})
|
||||
relationship_highlight_count += len(tile_relationship_highlights)
|
||||
tile_label_highlights = label_highlights.get(str(label_path), {})
|
||||
for category in tile_label_highlights.values():
|
||||
highlight_category_counts[category] = (
|
||||
highlight_category_counts.get(category, 0) + 1
|
||||
)
|
||||
invalid_label_count += tile_invalid_count
|
||||
valid_label_count += len(boxes)
|
||||
if missing_label_file:
|
||||
@@ -441,7 +471,7 @@ def build_report(
|
||||
invalid_label_count=tile_invalid_count,
|
||||
missing_label_file=missing_label_file,
|
||||
low_visual_variance=low_visual_variance,
|
||||
relationship_highlights=tile_relationship_highlights,
|
||||
relationship_highlights=tile_label_highlights,
|
||||
)
|
||||
)
|
||||
rendered = True
|
||||
@@ -460,7 +490,7 @@ def build_report(
|
||||
"invalid_label_count": tile_invalid_count,
|
||||
"missing_label_file": missing_label_file,
|
||||
"low_visual_variance": low_visual_variance,
|
||||
"relationship_highlight_count": len(tile_relationship_highlights),
|
||||
"highlighted_label_count": len(tile_label_highlights),
|
||||
"rendered": rendered,
|
||||
}
|
||||
)
|
||||
@@ -478,6 +508,14 @@ def build_report(
|
||||
}
|
||||
for start in range(0, len(rendered_cards), args.tiles_per_sheet)
|
||||
]
|
||||
relationship_highlight_count = sum(
|
||||
highlight_category_counts.get(category, 0)
|
||||
for category in ("exact_duplicate", "near_duplicate", "possible_nested")
|
||||
)
|
||||
outlier_highlight_count = sum(
|
||||
highlight_category_counts.get(category, 0)
|
||||
for category in ("extreme_aspect_ratio", "small_dimension", "tile_edge")
|
||||
)
|
||||
|
||||
return (
|
||||
{
|
||||
@@ -503,7 +541,12 @@ def build_report(
|
||||
"invalid_label_count": invalid_label_count,
|
||||
"valid_label_count": valid_label_count,
|
||||
"low_visual_variance_tile_count": low_visual_variance_tile_count,
|
||||
"highlighted_label_count": sum(highlight_category_counts.values()),
|
||||
"relationship_highlight_count": relationship_highlight_count,
|
||||
"outlier_highlight_count": outlier_highlight_count,
|
||||
"highlight_category_counts": dict(
|
||||
sorted(highlight_category_counts.items())
|
||||
),
|
||||
"blank_range_threshold": args.blank_range_threshold,
|
||||
"contact_sheets": contact_sheets,
|
||||
"selected_tiles": selected_report_tiles,
|
||||
@@ -525,7 +568,8 @@ def write_markdown(report: dict[str, Any], output_dir: Path) -> None:
|
||||
f"- invalid label rows: {report['invalid_label_count']}",
|
||||
f"- low-variance rendered tiles: {report['low_visual_variance_tile_count']}",
|
||||
f"- valid labels rendered: {report['valid_label_count']}",
|
||||
f"- relationship-highlighted labels: {report['relationship_highlight_count']}",
|
||||
f"- highlighted labels: {report['highlighted_label_count']}",
|
||||
f"- highlight categories: `{report['highlight_category_counts']}`",
|
||||
"",
|
||||
"## Contact Sheets",
|
||||
"",
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import pytest
|
||||
|
||||
from scripts.audit_yolo_cross_tile_repetition import (
|
||||
interior_global_key,
|
||||
tile_offsets,
|
||||
)
|
||||
from scripts.audit_yolo_label_relationships import Box
|
||||
|
||||
|
||||
def test_tile_offsets_parse_exporter_filename() -> None:
|
||||
assert tile_offsets("/data/geel_0005_r256_c512.png") == (256, 512)
|
||||
with pytest.raises(ValueError, match="no row/column offsets"):
|
||||
tile_offsets("/data/not-a-tile.png")
|
||||
|
||||
|
||||
def test_interior_global_key_reconstructs_same_object_across_tiles() -> None:
|
||||
first = Box(0, 0.75, 0.5, 0.1, 0.1)
|
||||
second = Box(0, 0.25, 0.5, 0.1, 0.1)
|
||||
|
||||
first_key = interior_global_key(
|
||||
first,
|
||||
row_offset=0,
|
||||
column_offset=0,
|
||||
tile_size=512,
|
||||
edge_tolerance_pixels=0.5,
|
||||
)
|
||||
second_key = interior_global_key(
|
||||
second,
|
||||
row_offset=0,
|
||||
column_offset=256,
|
||||
tile_size=512,
|
||||
edge_tolerance_pixels=0.5,
|
||||
)
|
||||
|
||||
assert first_key == second_key == (358.4, 230.4, 409.6, 281.6)
|
||||
|
||||
|
||||
def test_interior_global_key_rejects_edge_clipped_box() -> None:
|
||||
edge_box = Box(0, 0.05, 0.5, 0.1, 0.1)
|
||||
|
||||
assert (
|
||||
interior_global_key(
|
||||
edge_box,
|
||||
row_offset=0,
|
||||
column_offset=256,
|
||||
tile_size=512,
|
||||
edge_tolerance_pixels=0.5,
|
||||
)
|
||||
is None
|
||||
)
|
||||
@@ -0,0 +1,45 @@
|
||||
from scripts.audit_yolo_label_outliers import classify_box
|
||||
from scripts.audit_yolo_label_relationships import Box
|
||||
|
||||
|
||||
DEFAULTS = {
|
||||
"tile_size": 640,
|
||||
"min_dimension_pixels": 4.0,
|
||||
"extreme_aspect_ratio": 8.0,
|
||||
"edge_tolerance_pixels": 0.5,
|
||||
}
|
||||
|
||||
|
||||
def test_classify_box_reports_independent_geometric_risks() -> None:
|
||||
box = Box(0, 0.5, 0.003, 0.2, 0.006)
|
||||
|
||||
assert classify_box(box, **DEFAULTS) == [
|
||||
{
|
||||
"category": "small_dimension",
|
||||
"width_px": 128.0,
|
||||
"height_px": 3.84,
|
||||
"area_px2": 491.52,
|
||||
"aspect_ratio": 33.333333,
|
||||
},
|
||||
{
|
||||
"category": "extreme_aspect_ratio",
|
||||
"width_px": 128.0,
|
||||
"height_px": 3.84,
|
||||
"area_px2": 491.52,
|
||||
"aspect_ratio": 33.333333,
|
||||
},
|
||||
{
|
||||
"category": "tile_edge",
|
||||
"edge_sides": ["top"],
|
||||
"width_px": 128.0,
|
||||
"height_px": 3.84,
|
||||
"area_px2": 491.52,
|
||||
"aspect_ratio": 33.333333,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_classify_box_ignores_ordinary_interior_box() -> None:
|
||||
box = Box(0, 0.5, 0.5, 0.1, 0.08)
|
||||
|
||||
assert classify_box(box, **DEFAULTS) == []
|
||||
@@ -0,0 +1,29 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.evaluate_yolo_checkpoint_matrix import dataset_overlap_evidence
|
||||
|
||||
|
||||
def test_dataset_overlap_evidence_marks_repeated_validation_rows(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
dataset_yaml = tmp_path / "dataset.yaml"
|
||||
dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8")
|
||||
(tmp_path / "yolo_tile_dataset_summary.json").write_text(
|
||||
json.dumps({"tile_size": 512, "stride": 256}), encoding="utf-8"
|
||||
)
|
||||
|
||||
evidence = dataset_overlap_evidence(dataset_yaml)
|
||||
|
||||
assert evidence["status"] == "overlapping"
|
||||
assert evidence["overlap_pixels"] == 256
|
||||
assert evidence["validation_rows_independent"] is False
|
||||
|
||||
|
||||
def test_dataset_overlap_evidence_is_explicit_when_summary_missing(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
evidence = dataset_overlap_evidence(tmp_path / "dataset.yaml")
|
||||
|
||||
assert evidence["status"] == "unavailable"
|
||||
assert evidence["validation_rows_independent"] is None
|
||||
@@ -2,6 +2,7 @@ import pytest
|
||||
|
||||
from scripts.render_operator_yolo_label_qa_contact_sheets import (
|
||||
CONTACT_SHEET_NAME_TEMPLATE,
|
||||
build_label_highlights,
|
||||
build_relationship_highlights,
|
||||
filter_tiles_by_samples,
|
||||
)
|
||||
@@ -72,3 +73,25 @@ def test_relationship_highlights_bind_indices_and_preserve_highest_priority() ->
|
||||
7: "near_duplicate",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_label_highlights_support_outliers_with_stable_priority() -> None:
|
||||
summary = {
|
||||
"flagged_tiles": [
|
||||
{
|
||||
"label_path": "/data/tile.txt",
|
||||
"outliers": [
|
||||
{"category": "tile_edge", "index": 3},
|
||||
{"category": "small_dimension", "index": 3},
|
||||
{"category": "extreme_aspect_ratio", "index": 4},
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
assert build_label_highlights(summary) == {
|
||||
"/data/tile.txt": {
|
||||
3: "small_dimension",
|
||||
4: "extreme_aspect_ratio",
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user