derive cleaner min-4px YOLO corpus
This commit is contained in:
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_id": "reviewedexp6-minpx4-derived-corpus-r1",
|
||||
"created_at": "2026-08-09T19:45:41+02:00",
|
||||
"status": "complete_experimental_only",
|
||||
"claim_boundary": "Deterministic derived-corpus improvement and AI-assisted visual triage only; no human-review, governed training, evaluation, promotion or production claim.",
|
||||
"source": {
|
||||
"dataset_dir": "/app/storage/operator-data/yolo-building-aoi1024-reviewedexp6-minpx3vis035",
|
||||
"summary_sha256": "7c917e31216d1df2174c0f9c736f88a81f3835aa991971f8fb8665e17ddf5c9c",
|
||||
"tile_size": 512,
|
||||
"stride": 256,
|
||||
"min_label_px": 3.0,
|
||||
"min_visible_ratio": 0.35,
|
||||
"tile_count": 252,
|
||||
"label_count": 79192
|
||||
},
|
||||
"derived": {
|
||||
"dataset_dir": "/app/storage/operator-data/yolo-building-aoi1024-reviewedexp6-minpx4-derived-r1",
|
||||
"manifest_sha256": "8a80d0e9d5ac2d32556e2f1ea0c6009045d48a3e56d931b3161622dca4173a67",
|
||||
"summary_sha256": "391b0302cc8a46d15fef426d8971a4a8e594fb6d63ed1c7ce16feeee9f1406",
|
||||
"quality_audit_sha256": "0fc9391ce09a399f51f1e0ed30c677f482c7c1b49b187926da01ecdfd9f1067e",
|
||||
"outlier_audit_sha256": "c2453abb83aaff8aebd84f63efb14da131225192cf0434d5d1df7cf06be240ca",
|
||||
"relationship_audit_sha256": "61660d0c448ff579272bc585de8140e82d1577c0b017e1c078b66dc093f14c2e",
|
||||
"cross_tile_repetition_audit_sha256": "9ca22d83271707713f7f96d5409af598f834787ad5a48be14f8752a704de0da0",
|
||||
"min_label_px": 4.0,
|
||||
"tile_count": 252,
|
||||
"positive_tile_count": 234,
|
||||
"negative_tile_count": 18,
|
||||
"label_count": 77380,
|
||||
"removed_label_count": 1812,
|
||||
"excluded_tile_count": 0,
|
||||
"invalid_label_count": 0,
|
||||
"missing_label_file_count": 0,
|
||||
"sub_4_pixel_label_count": 0,
|
||||
"extreme_aspect_label_count": 10,
|
||||
"tile_edge_label_count": 4840,
|
||||
"possible_nested_pair_count": 41,
|
||||
"exact_duplicate_pair_count": 0,
|
||||
"near_duplicate_pair_count": 0,
|
||||
"cross_split_repetition_group_count": 0,
|
||||
"training_release_eligible": false
|
||||
},
|
||||
"visual_review": {
|
||||
"reviewer_type": "ai_assistant",
|
||||
"human": false,
|
||||
"rendered_tile_count": 252,
|
||||
"rendered_label_count": 77380,
|
||||
"contact_sheet_sha256s": [
|
||||
"f197769ca68da08745277ff93331f3ea93738f6b63eb383a3b6168a9e438ee12",
|
||||
"e9ecde3f38b4f580adb7523f647460af5d5933e2e1efe1f11f98c1656ca1cd13",
|
||||
"497bf5878560113fcc59f21aeba09b1de167cb3eff4d0327c7584ba8caeebf70",
|
||||
"4a8262291ed29699ae59517e1a542e513a50081a225ea1e84268afa37f99d6df"
|
||||
],
|
||||
"extreme_aspect_contact_sheet_sha256": "6c08c42eb6478d1fa06745135dd588ce6522b5cec4adf0bdcd177d77cd6c78a7",
|
||||
"conclusion": "All 252 tiles remain visually coherent; pure-background tiles remain empty, sparse contexts remain distinct, and the ten remaining extreme-aspect labels predominantly correspond to plausible elongated structures."
|
||||
},
|
||||
"decision": {
|
||||
"preferred_experimental_corpus": true,
|
||||
"replaces_source_in_place": false,
|
||||
"reason": "It deterministically removes all 1,812 visually marginal sub-4-pixel labels while preserving every tile and all larger labels with exact lineage.",
|
||||
"remaining_gate": "Requires independent human review and the governed release-contract inputs before training eligibility."
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,9 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Path) -> None:
|
||||
def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
script_path = ROOT / "scripts" / "audit_operator_yolo_dataset_quality.py"
|
||||
assert script_path.exists()
|
||||
|
||||
@@ -135,10 +137,12 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert "Operator YOLO dataset quality audit passed" in result.stdout
|
||||
assert "Operator YOLO dataset quality audit completed" in result.stdout
|
||||
|
||||
report = json.loads(
|
||||
(output_dir / "operator_yolo_dataset_quality_audit.json").read_text(encoding="utf-8")
|
||||
(output_dir / "operator_yolo_dataset_quality_audit.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
assert report["status"] == "needs_attention"
|
||||
assert report["sample_count"] == 3
|
||||
@@ -150,7 +154,9 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat
|
||||
assert report["label_stats"]["parsed_label_count"] == 3
|
||||
assert report["label_stats"]["invalid_label_count"] == 0
|
||||
assert report["min_label_visible_ratio"] == 0.25
|
||||
sample_by_slug = {sample["sample_slug"]: sample for sample in report["sample_summaries"]}
|
||||
sample_by_slug = {
|
||||
sample["sample_slug"]: sample for sample in report["sample_summaries"]
|
||||
}
|
||||
assert sample_by_slug["geel"]["parsed_label_count"] == 2
|
||||
assert sample_by_slug["geel"]["invalid_label_count"] == 0
|
||||
assert sample_by_slug["geel"]["small_box_share"] == 0.5
|
||||
@@ -161,7 +167,9 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat
|
||||
]
|
||||
assert sample_by_slug["turnhout"]["parsed_label_count"] == 1
|
||||
assert sample_by_slug["turnhout"]["small_box_share"] == 0.0
|
||||
assert sample_by_slug["turnhout"]["quality_warnings"] == ["median_box_area_below_gate"]
|
||||
assert sample_by_slug["turnhout"]["quality_warnings"] == [
|
||||
"median_box_area_below_gate"
|
||||
]
|
||||
assert sample_by_slug["postel_bos"]["parsed_label_count"] == 0
|
||||
assert sample_by_slug["postel_bos"]["quality_warnings"] == []
|
||||
|
||||
@@ -173,7 +181,9 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat
|
||||
assert "small_box_share_above_gate" in warning_codes
|
||||
assert "positive_tiles_have_low_visual_variance" in warning_codes
|
||||
|
||||
markdown = (output_dir / "operator_yolo_dataset_quality_audit.md").read_text(encoding="utf-8")
|
||||
markdown = (output_dir / "operator_yolo_dataset_quality_audit.md").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert "Operator YOLO Dataset Quality Audit" in markdown
|
||||
assert "Label Quality" in markdown
|
||||
assert "Minimum visible label ratio" in markdown
|
||||
|
||||
@@ -12638,3 +12638,36 @@ Open:
|
||||
not independent object-level accuracy. A future benchmark must aggregate or
|
||||
deduplicate objects before uncertainty claims.
|
||||
- No source label, frozen dataset, model weight or live runtime was changed.
|
||||
|
||||
## 2026-08-09 - Immutable min-4px derived corpus
|
||||
|
||||
### Improved
|
||||
|
||||
- Added a deterministic derived-corpus builder that preserves parent hashes,
|
||||
copies imagery, records every removed source row and refuses to turn a
|
||||
positive tile into an unlabeled negative. It creates no release manifest or
|
||||
human-review claim.
|
||||
- Derived `yolo-building-aoi1024-reviewedexp6-minpx4-derived-r1` from the
|
||||
frozen min-3px corpus. Exactly 1,812 marginal labels were removed; all 252
|
||||
tiles and 77,380 labels remain, with zero excluded tiles.
|
||||
- Corrected the quality-audit console message from `passed` to `completed`, so
|
||||
a machine-readable `needs_attention` status is no longer contradicted by
|
||||
human-facing output.
|
||||
|
||||
### Verified on Tower
|
||||
|
||||
- Structural audit: zero missing/invalid labels and zero sub-4-pixel rows.
|
||||
- Relationship audit: zero exact/near duplicates and 41 possible nesting
|
||||
pairs, down from 43.
|
||||
- Geometry audit: 10 plausible extreme-aspect rows, down from 26; tile-edge
|
||||
rows fell from 5,079 to 4,840 without a blanket edge deletion.
|
||||
- Cross-tile audit: zero reconstructed train/validation crossings.
|
||||
- Inspected four full contact-sheet pages covering all 252 tiles and 77,380
|
||||
labels plus the focused nine-tile extreme-aspect sheet. Pure backgrounds
|
||||
remain empty and sparse contexts remain distinct.
|
||||
|
||||
### Decision and limitation
|
||||
|
||||
- The min-4px corpus is preferred for future experimental work, but remains
|
||||
explicitly training/release-ineligible until real human review and governed
|
||||
release evidence exist. The active GPU model was not changed.
|
||||
|
||||
+6
-3
@@ -1164,9 +1164,12 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
sheets and retain immutable hashes.
|
||||
- [x] Reconstruct exact cross-tile interior objects and prove that overlap
|
||||
repeats 40,031 label rows but causes zero reconstructed train/val crossings.
|
||||
- [ ] Build the next immutable experimental corpus with `min_label_px >= 4`
|
||||
and compare the existing 0.35 edge-visibility policy against a separately
|
||||
versioned stricter ablation before any new governed training.
|
||||
- [x] Build and fully audit an immutable experimental corpus with
|
||||
`min_label_px=4`; retain 252 tiles and 77,380 labels with zero sub-4-pixel
|
||||
rows, missing files, invalid labels or cross-split reconstructed objects.
|
||||
- [ ] Compare the existing 0.35 edge-visibility policy against a separately
|
||||
versioned stricter ablation from original source geometry before any new
|
||||
governed training; do not approximate this by dropping every edge label.
|
||||
- [ ] Convert the AI-assisted ledger into no stronger claim than experimental
|
||||
triage; a real human must independently review and sign the frozen artifacts
|
||||
before the governed training wrapper may unlock.
|
||||
|
||||
@@ -161,3 +161,29 @@ The edge sample shows expected clipped buildings under the 0.35 minimum-visible
|
||||
policy. Raising that value may reduce partial-target pressure, but it must be a
|
||||
separately versioned ablation because removing all 5,079 rows without checking
|
||||
the original visible fraction would be unsound.
|
||||
|
||||
## Derived min-4px experimental corpus
|
||||
|
||||
The row-level finding was converted into a new immutable derived corpus rather
|
||||
than changing the historical dataset. The derivation is checksum-bound to the
|
||||
79,192-label source summary and removes only labels whose smallest 512-tile
|
||||
dimension is below 4 pixels. It retains all 252 tiles and 77,380 labels; no
|
||||
positive tile became empty, and no source file was overwritten.
|
||||
|
||||
Fresh audits report zero sub-4-pixel labels, zero invalid or missing label
|
||||
files, zero exact/high-IoU duplicates and zero reconstructed cross-split
|
||||
object groups. Extreme-aspect labels fall from 26 to 10, possible nesting from
|
||||
43 to 41 and tile-edge rows from 5,079 to 4,840. The complete four-page visual
|
||||
render was inspected again. Pure-background tiles remain visibly empty, sparse
|
||||
contexts remain distinct and the ten remaining elongated labels predominantly
|
||||
match plausible long structures.
|
||||
|
||||
The generic quality auditor still reports `needs_attention` because normalized
|
||||
median box area `0.000641` is below its conservative `0.001` warning threshold.
|
||||
That corresponds to median dimensions near 12 by 13 pixels at source tile
|
||||
resolution and is not a failed minimum-dimension gate. The warning remains
|
||||
visible; it was not suppressed or relabelled as success.
|
||||
|
||||
This min-4px version is the preferred experimental successor to the legacy
|
||||
min-3px corpus. It explicitly remains ineligible for governed training until
|
||||
real human review and all release-contract evidence exist.
|
||||
|
||||
@@ -28,6 +28,13 @@ reconstructs global pixel boxes from exporter tile offsets to quantify exact
|
||||
interior-object repetition caused by overlap; edge rows remain explicitly
|
||||
unlinked and no repetition is automatically classified as an error.
|
||||
|
||||
`derive_yolo_min_dimension_corpus.py` creates a new experimental-only corpus
|
||||
from an existing checksum-bound YOLO tile dataset. It copies imagery, filters
|
||||
only labels whose smallest dimension is below the declared pixel floor,
|
||||
excludes rather than relabels any positive tile that would become empty, and
|
||||
writes complete source/output hashes plus a row-level removal manifest. It
|
||||
never creates a training-release manifest or human-review claim.
|
||||
|
||||
Setup-, import-, demo- en maintenance-scripts voor GeoIntel.
|
||||
|
||||
## WALOUS source provisioning
|
||||
|
||||
@@ -20,8 +20,12 @@ def parse_args() -> argparse.Namespace:
|
||||
"negative balance and label-quality risks."
|
||||
)
|
||||
)
|
||||
parser.add_argument("--summary-path", required=True, help="Path to yolo_tile_dataset_summary.json")
|
||||
parser.add_argument("--output-dir", required=True, help="Directory for JSON and Markdown reports")
|
||||
parser.add_argument(
|
||||
"--summary-path", required=True, help="Path to yolo_tile_dataset_summary.json"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir", required=True, help="Directory for JSON and Markdown reports"
|
||||
)
|
||||
parser.add_argument("--min-positive-samples", type=int, default=6)
|
||||
parser.add_argument("--min-val-positive-samples", type=int, default=2)
|
||||
parser.add_argument("--max-repeated-negative-share", type=float, default=0.65)
|
||||
@@ -86,7 +90,12 @@ def parse_yolo_label_file(path: Path | None) -> tuple[list[dict[str, float]], in
|
||||
except ValueError:
|
||||
invalid_count += 1
|
||||
continue
|
||||
if not (0 <= center_x <= 1 and 0 <= center_y <= 1 and 0 < width <= 1 and 0 < height <= 1):
|
||||
if not (
|
||||
0 <= center_x <= 1
|
||||
and 0 <= center_y <= 1
|
||||
and 0 < width <= 1
|
||||
and 0 < height <= 1
|
||||
):
|
||||
invalid_count += 1
|
||||
continue
|
||||
boxes.append(
|
||||
@@ -138,7 +147,9 @@ def summarize_label_files(
|
||||
areas = [box["area"] for box in boxes]
|
||||
widths = [box["width"] for box in boxes]
|
||||
heights = [box["height"] for box in boxes]
|
||||
aspect_ratios = [max(box["width"] / box["height"], box["height"] / box["width"]) for box in boxes]
|
||||
aspect_ratios = [
|
||||
max(box["width"] / box["height"], box["height"] / box["width"]) for box in boxes
|
||||
]
|
||||
small_box_count = sum(1 for area in areas if area < small_box_area_threshold)
|
||||
|
||||
return {
|
||||
@@ -175,7 +186,10 @@ def build_label_quality_warning_codes(
|
||||
median_box_area = label_stats["median_box_area"]
|
||||
if median_box_area is not None and median_box_area < args.min_median_box_area:
|
||||
warning_codes.append("median_box_area_below_gate")
|
||||
if label_stats["parsed_label_count"] and label_stats["small_box_share"] > args.max_small_box_share:
|
||||
if (
|
||||
label_stats["parsed_label_count"]
|
||||
and label_stats["small_box_share"] > args.max_small_box_share
|
||||
):
|
||||
warning_codes.append("small_box_share_above_gate")
|
||||
return warning_codes
|
||||
|
||||
@@ -233,7 +247,9 @@ def build_sample_summaries(
|
||||
|
||||
result: list[dict[str, Any]] = []
|
||||
for sample in samples.values():
|
||||
label_stats = summarize_label_files(sample.pop("_label_file_paths"), args.small_box_area_threshold)
|
||||
label_stats = summarize_label_files(
|
||||
sample.pop("_label_file_paths"), args.small_box_area_threshold
|
||||
)
|
||||
quality_warnings = build_label_quality_warning_codes(label_stats, args)
|
||||
result.append(
|
||||
{
|
||||
@@ -246,8 +262,14 @@ def build_sample_summaries(
|
||||
return sorted(result, key=lambda item: str(item["sample_slug"]))
|
||||
|
||||
|
||||
def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Namespace) -> dict[str, Any]:
|
||||
tiles = [tile for tile in summary.get("tiles", []) if isinstance(tile, dict) and tile.get("kept", True)]
|
||||
def build_audit(
|
||||
summary: dict[str, Any], summary_path: Path, args: argparse.Namespace
|
||||
) -> dict[str, Any]:
|
||||
tiles = [
|
||||
tile
|
||||
for tile in summary.get("tiles", [])
|
||||
if isinstance(tile, dict) and tile.get("kept", True)
|
||||
]
|
||||
sample_summaries = build_sample_summaries(tiles, summary_path, args)
|
||||
|
||||
split_counts = Counter(str(tile.get("split") or "unknown") for tile in tiles)
|
||||
@@ -258,8 +280,12 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
|
||||
if bool(tile.get("is_negative")) or int(tile.get("label_count") or 0) == 0
|
||||
]
|
||||
positive_tiles = [tile for tile in tiles if tile not in negative_tiles]
|
||||
train_negative_tiles = [tile for tile in negative_tiles if tile.get("split") == "train"]
|
||||
low_variance_positive_tiles = [tile for tile in positive_tiles if tile.get("low_visual_variance")]
|
||||
train_negative_tiles = [
|
||||
tile for tile in negative_tiles if tile.get("split") == "train"
|
||||
]
|
||||
low_variance_positive_tiles = [
|
||||
tile for tile in positive_tiles if tile.get("low_visual_variance")
|
||||
]
|
||||
val_positive_samples = {
|
||||
str(tile.get("sample_slug"))
|
||||
for tile in positive_tiles
|
||||
@@ -275,8 +301,12 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
|
||||
for sample in sample_summaries
|
||||
if sample["sample_role"] == "background_candidate"
|
||||
}
|
||||
repeated_negative_count = sum(1 for tile in negative_tiles if tile.get("is_repeated_background_negative"))
|
||||
repeated_negative_share = repeated_negative_count / len(negative_tiles) if negative_tiles else 0.0
|
||||
repeated_negative_count = sum(
|
||||
1 for tile in negative_tiles if tile.get("is_repeated_background_negative")
|
||||
)
|
||||
repeated_negative_share = (
|
||||
repeated_negative_count / len(negative_tiles) if negative_tiles else 0.0
|
||||
)
|
||||
|
||||
label_stats = summarize_labels(tiles, summary_path, args.small_box_area_threshold)
|
||||
warnings: list[dict[str, str]] = []
|
||||
@@ -381,7 +411,10 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
|
||||
def build_recommendations(warnings: list[dict[str, str]]) -> list[str]:
|
||||
codes = {warning["code"] for warning in warnings}
|
||||
recommendations: list[str] = []
|
||||
if "positive_sample_count_below_gate" in codes or "val_positive_sample_count_below_gate" in codes:
|
||||
if (
|
||||
"positive_sample_count_below_gate" in codes
|
||||
or "val_positive_sample_count_below_gate" in codes
|
||||
):
|
||||
recommendations.append(
|
||||
"Add more labeled positive AOIs before extending training duration or increasing model size."
|
||||
)
|
||||
@@ -394,13 +427,17 @@ def build_recommendations(warnings: list[dict[str, str]]) -> list[str]:
|
||||
"Inspect clipped building labels visually; very small boxes may indicate tile size or label clipping issues."
|
||||
)
|
||||
if "label_files_missing" in codes or "invalid_label_rows" in codes:
|
||||
recommendations.append("Regenerate the YOLO tile dataset and review exporter path/label integrity.")
|
||||
recommendations.append(
|
||||
"Regenerate the YOLO tile dataset and review exporter path/label integrity."
|
||||
)
|
||||
if "positive_tiles_have_low_visual_variance" in codes:
|
||||
recommendations.append(
|
||||
"Reject the raster product for affected AOIs or replace it with an officially complete imagery edition before training."
|
||||
)
|
||||
if not recommendations:
|
||||
recommendations.append("Dataset audit passed the configured gates; continue with benchmarked training.")
|
||||
recommendations.append(
|
||||
"Dataset audit passed the configured gates; continue with benchmarked training."
|
||||
)
|
||||
return recommendations
|
||||
|
||||
|
||||
@@ -475,7 +512,7 @@ def main() -> int:
|
||||
json_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
|
||||
write_markdown(report, markdown_path)
|
||||
|
||||
print("Operator YOLO dataset quality audit passed")
|
||||
print("Operator YOLO dataset quality audit completed")
|
||||
print(f"Status: {report['status']}")
|
||||
print(f"JSON: {json_path}")
|
||||
print(f"Markdown: {markdown_path}")
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create an immutable experimental YOLO corpus with a stricter pixel floor."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def filter_label_lines(
|
||||
lines: list[str], *, tile_size: int, min_dimension_pixels: float
|
||||
) -> tuple[list[str], list[int]]:
|
||||
kept: list[str] = []
|
||||
removed_indices: list[int] = []
|
||||
for index, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
parts = stripped.split()
|
||||
if len(parts) != 5:
|
||||
raise ValueError(f"invalid YOLO row at zero-based index {index}")
|
||||
try:
|
||||
float(parts[0])
|
||||
width = float(parts[3])
|
||||
height = float(parts[4])
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"invalid numeric YOLO row at zero-based index {index}"
|
||||
) from exc
|
||||
if width <= 0 or height <= 0 or width > 1 or height > 1:
|
||||
raise ValueError(f"out-of-range YOLO row at zero-based index {index}")
|
||||
if min(width, height) * tile_size < min_dimension_pixels:
|
||||
removed_indices.append(index)
|
||||
else:
|
||||
kept.append(stripped)
|
||||
return kept, removed_indices
|
||||
|
||||
|
||||
def aggregate_hash(paths: list[Path], root: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
for path in sorted(paths):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
digest.update(relative.encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(sha256_file(path).encode("ascii"))
|
||||
digest.update(b"\n")
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-dir", required=True, type=Path)
|
||||
parser.add_argument("--output-dir", required=True, type=Path)
|
||||
parser.add_argument("--min-dimension-pixels", type=float, default=4.0)
|
||||
args = parser.parse_args()
|
||||
if args.min_dimension_pixels <= 0:
|
||||
parser.error("--min-dimension-pixels must be positive")
|
||||
|
||||
source_dir = args.source_dir.expanduser().resolve(strict=True)
|
||||
output_dir = args.output_dir.expanduser().resolve(strict=False)
|
||||
if output_dir.exists():
|
||||
parser.error(f"output directory already exists: {output_dir}")
|
||||
source_summary_path = source_dir / "yolo_tile_dataset_summary.json"
|
||||
source_summary = json.loads(source_summary_path.read_text(encoding="utf-8"))
|
||||
tiles = source_summary.get("tiles")
|
||||
tile_size = source_summary.get("tile_size")
|
||||
if not isinstance(tiles, list):
|
||||
raise ValueError("source summary must contain a tiles list")
|
||||
if not isinstance(tile_size, int) or tile_size <= 0:
|
||||
raise ValueError("source summary must contain a positive integer tile_size")
|
||||
source_minimum = source_summary.get("min_label_px")
|
||||
if isinstance(source_minimum, (int, float)) and args.min_dimension_pixels < float(
|
||||
source_minimum
|
||||
):
|
||||
raise ValueError("derived minimum may not weaken the source pixel floor")
|
||||
|
||||
output_dir.mkdir(parents=True)
|
||||
kept_tiles: list[dict[str, Any]] = []
|
||||
excluded_tiles: list[dict[str, Any]] = []
|
||||
removed_labels: list[dict[str, Any]] = []
|
||||
output_images: list[Path] = []
|
||||
output_labels: list[Path] = []
|
||||
for tile in tiles:
|
||||
if not isinstance(tile, dict) or not tile.get("kept", True):
|
||||
continue
|
||||
source_image = Path(str(tile.get("image_path") or "")).resolve(strict=True)
|
||||
source_label = Path(str(tile.get("label_path") or "")).resolve(strict=True)
|
||||
image_relative = source_image.relative_to(source_dir)
|
||||
label_relative = source_label.relative_to(source_dir)
|
||||
lines = source_label.read_text(encoding="utf-8").splitlines()
|
||||
kept_lines, removed_indices = filter_label_lines(
|
||||
lines,
|
||||
tile_size=tile_size,
|
||||
min_dimension_pixels=args.min_dimension_pixels,
|
||||
)
|
||||
for index in removed_indices:
|
||||
removed_labels.append(
|
||||
{
|
||||
"sample_slug": tile.get("sample_slug"),
|
||||
"split": tile.get("split"),
|
||||
"tile_index": tile.get("tile_index"),
|
||||
"source_label_path": str(source_label),
|
||||
"source_label_index": index,
|
||||
"source_row": lines[index].strip(),
|
||||
"reason": "dimension_below_pixel_floor",
|
||||
}
|
||||
)
|
||||
source_had_labels = any(line.strip() for line in lines)
|
||||
if source_had_labels and not kept_lines:
|
||||
excluded_tiles.append(
|
||||
{
|
||||
"sample_slug": tile.get("sample_slug"),
|
||||
"split": tile.get("split"),
|
||||
"tile_index": tile.get("tile_index"),
|
||||
"reason": "positive_tile_became_empty_after_filter",
|
||||
"removed_label_count": len(removed_indices),
|
||||
"source_image_path": str(source_image),
|
||||
"source_label_path": str(source_label),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
target_image = output_dir / image_relative
|
||||
target_label = output_dir / label_relative
|
||||
target_image.parent.mkdir(parents=True, exist_ok=True)
|
||||
target_label.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source_image, target_image)
|
||||
target_label.write_text(
|
||||
"".join(f"{line}\n" for line in kept_lines), encoding="utf-8"
|
||||
)
|
||||
output_images.append(target_image)
|
||||
output_labels.append(target_label)
|
||||
derived_tile = dict(tile)
|
||||
derived_tile.update(
|
||||
{
|
||||
"image_path": str(target_image),
|
||||
"label_path": str(target_label),
|
||||
"label_count": len(kept_lines),
|
||||
"is_negative": not kept_lines,
|
||||
"derived_from_image_path": str(source_image),
|
||||
"derived_from_label_path": str(source_label),
|
||||
"removed_label_count": len(removed_indices),
|
||||
}
|
||||
)
|
||||
kept_tiles.append(derived_tile)
|
||||
|
||||
dataset_yaml = output_dir / "dataset.yaml"
|
||||
dataset_yaml.write_text(
|
||||
"\n".join(
|
||||
(
|
||||
f"path: {output_dir}",
|
||||
"train: images/train",
|
||||
"val: images/val",
|
||||
"names:",
|
||||
" 0: building",
|
||||
"",
|
||||
)
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
|
||||
negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
|
||||
summary = {
|
||||
**{key: value for key, value in source_summary.items() if key != "tiles"},
|
||||
"schema_version": 1,
|
||||
"status": "experimental_derived_not_release_eligible",
|
||||
"claim_boundary": (
|
||||
"Deterministic min-dimension ablation only; no human-review, training, "
|
||||
"evaluation or production-release claim."
|
||||
),
|
||||
"output_dir": str(output_dir),
|
||||
"dataset_yaml": str(dataset_yaml),
|
||||
"source_dataset_dir": str(source_dir),
|
||||
"source_summary_path": str(source_summary_path),
|
||||
"source_summary_sha256": sha256_file(source_summary_path),
|
||||
"transformation": "drop_label_if_min_dimension_pixels_below_threshold",
|
||||
"min_label_px": args.min_dimension_pixels,
|
||||
"source_min_label_px": source_minimum,
|
||||
"tile_count": len(kept_tiles),
|
||||
"positive_tile_count": len(positive_tiles),
|
||||
"negative_tile_count": len(negative_tiles),
|
||||
"train_tile_count": sum(tile.get("split") == "train" for tile in kept_tiles),
|
||||
"val_tile_count": sum(tile.get("split") == "val" for tile in kept_tiles),
|
||||
"label_count": sum(int(tile["label_count"]) for tile in kept_tiles),
|
||||
"removed_label_count": len(removed_labels),
|
||||
"excluded_tile_count": len(excluded_tiles),
|
||||
"training_release_eligible": False,
|
||||
"tiles": kept_tiles,
|
||||
}
|
||||
summary_path = output_dir / "yolo_tile_dataset_summary.json"
|
||||
summary_path.write_text(
|
||||
json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(UTC).isoformat(),
|
||||
"status": "complete_experimental_only",
|
||||
"source_summary_path": str(source_summary_path),
|
||||
"source_summary_sha256": sha256_file(source_summary_path),
|
||||
"output_summary_path": str(summary_path),
|
||||
"output_summary_sha256": sha256_file(summary_path),
|
||||
"dataset_yaml_sha256": sha256_file(dataset_yaml),
|
||||
"image_set_aggregate_sha256": aggregate_hash(output_images, output_dir),
|
||||
"label_set_aggregate_sha256": aggregate_hash(output_labels, output_dir),
|
||||
"min_dimension_pixels": args.min_dimension_pixels,
|
||||
"source_label_count": int(source_summary.get("label_count") or 0),
|
||||
"derived_label_count": summary["label_count"],
|
||||
"removed_label_count": len(removed_labels),
|
||||
"excluded_tiles": excluded_tiles,
|
||||
"removed_labels": removed_labels,
|
||||
"training_release_eligible": False,
|
||||
}
|
||||
manifest_path = output_dir / "derived_corpus_manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": manifest["status"],
|
||||
"derived_label_count": manifest["derived_label_count"],
|
||||
"removed_label_count": manifest["removed_label_count"],
|
||||
"excluded_tile_count": len(excluded_tiles),
|
||||
"manifest_path": str(manifest_path),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,21 @@
|
||||
import pytest
|
||||
|
||||
from scripts.derive_yolo_min_dimension_corpus import filter_label_lines
|
||||
|
||||
|
||||
def test_filter_label_lines_removes_only_rows_below_pixel_floor() -> None:
|
||||
lines = [
|
||||
"0 0.5 0.5 0.005 0.25",
|
||||
"0 0.5 0.5 0.0078125 0.25",
|
||||
"0 0.5 0.5 0.1 0.1",
|
||||
]
|
||||
|
||||
kept, removed = filter_label_lines(lines, tile_size=512, min_dimension_pixels=4)
|
||||
|
||||
assert kept == lines[1:]
|
||||
assert removed == [0]
|
||||
|
||||
|
||||
def test_filter_label_lines_rejects_invalid_source_rows() -> None:
|
||||
with pytest.raises(ValueError, match="invalid YOLO row"):
|
||||
filter_label_lines(["0 0.5 0.5"], tile_size=512, min_dimension_pixels=4)
|
||||
Reference in New Issue
Block a user