derive cleaner min-4px YOLO corpus
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Jens
2026-08-09 19:51:17 +02:00
parent 736e773fb8
commit 2de438b9cc
9 changed files with 471 additions and 25 deletions
@@ -0,0 +1,63 @@
{
"schema_version": 1,
"evidence_id": "reviewedexp6-minpx4-derived-corpus-r1",
"created_at": "2026-08-09T19:45:41+02:00",
"status": "complete_experimental_only",
"claim_boundary": "Deterministic derived-corpus improvement and AI-assisted visual triage only; no human-review, governed training, evaluation, promotion or production claim.",
"source": {
"dataset_dir": "/app/storage/operator-data/yolo-building-aoi1024-reviewedexp6-minpx3vis035",
"summary_sha256": "7c917e31216d1df2174c0f9c736f88a81f3835aa991971f8fb8665e17ddf5c9c",
"tile_size": 512,
"stride": 256,
"min_label_px": 3.0,
"min_visible_ratio": 0.35,
"tile_count": 252,
"label_count": 79192
},
"derived": {
"dataset_dir": "/app/storage/operator-data/yolo-building-aoi1024-reviewedexp6-minpx4-derived-r1",
"manifest_sha256": "8a80d0e9d5ac2d32556e2f1ea0c6009045d48a3e56d931b3161622dca4173a67",
"summary_sha256": "391b0302cc8a46d15fef426d8971a4a8e594fb6d63ed1c7ce16feeee9f1406",
"quality_audit_sha256": "0fc9391ce09a399f51f1e0ed30c677f482c7c1b49b187926da01ecdfd9f1067e",
"outlier_audit_sha256": "c2453abb83aaff8aebd84f63efb14da131225192cf0434d5d1df7cf06be240ca",
"relationship_audit_sha256": "61660d0c448ff579272bc585de8140e82d1577c0b017e1c078b66dc093f14c2e",
"cross_tile_repetition_audit_sha256": "9ca22d83271707713f7f96d5409af598f834787ad5a48be14f8752a704de0da0",
"min_label_px": 4.0,
"tile_count": 252,
"positive_tile_count": 234,
"negative_tile_count": 18,
"label_count": 77380,
"removed_label_count": 1812,
"excluded_tile_count": 0,
"invalid_label_count": 0,
"missing_label_file_count": 0,
"sub_4_pixel_label_count": 0,
"extreme_aspect_label_count": 10,
"tile_edge_label_count": 4840,
"possible_nested_pair_count": 41,
"exact_duplicate_pair_count": 0,
"near_duplicate_pair_count": 0,
"cross_split_repetition_group_count": 0,
"training_release_eligible": false
},
"visual_review": {
"reviewer_type": "ai_assistant",
"human": false,
"rendered_tile_count": 252,
"rendered_label_count": 77380,
"contact_sheet_sha256s": [
"f197769ca68da08745277ff93331f3ea93738f6b63eb383a3b6168a9e438ee12",
"e9ecde3f38b4f580adb7523f647460af5d5933e2e1efe1f11f98c1656ca1cd13",
"497bf5878560113fcc59f21aeba09b1de167cb3eff4d0327c7584ba8caeebf70",
"4a8262291ed29699ae59517e1a542e513a50081a225ea1e84268afa37f99d6df"
],
"extreme_aspect_contact_sheet_sha256": "6c08c42eb6478d1fa06745135dd588ce6522b5cec4adf0bdcd177d77cd6c78a7",
"conclusion": "All 252 tiles remain visually coherent; pure-background tiles remain empty, sparse contexts remain distinct, and the ten remaining extreme-aspect labels predominantly correspond to plausible elongated structures."
},
"decision": {
"preferred_experimental_corpus": true,
"replaces_source_in_place": false,
"reason": "It deterministically removes all 1,812 visually marginal sub-4-pixel labels while preserving every tile and all larger labels with exact lineage.",
"remaining_gate": "Requires independent human review and the governed release-contract inputs before training eligibility."
}
}
@@ -9,7 +9,9 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Path) -> None:
def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(
tmp_path: Path,
) -> None:
script_path = ROOT / "scripts" / "audit_operator_yolo_dataset_quality.py"
assert script_path.exists()
@@ -135,10 +137,12 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat
text=True,
)
assert "Operator YOLO dataset quality audit passed" in result.stdout
assert "Operator YOLO dataset quality audit completed" in result.stdout
report = json.loads(
(output_dir / "operator_yolo_dataset_quality_audit.json").read_text(encoding="utf-8")
(output_dir / "operator_yolo_dataset_quality_audit.json").read_text(
encoding="utf-8"
)
)
assert report["status"] == "needs_attention"
assert report["sample_count"] == 3
@@ -150,7 +154,9 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat
assert report["label_stats"]["parsed_label_count"] == 3
assert report["label_stats"]["invalid_label_count"] == 0
assert report["min_label_visible_ratio"] == 0.25
sample_by_slug = {sample["sample_slug"]: sample for sample in report["sample_summaries"]}
sample_by_slug = {
sample["sample_slug"]: sample for sample in report["sample_summaries"]
}
assert sample_by_slug["geel"]["parsed_label_count"] == 2
assert sample_by_slug["geel"]["invalid_label_count"] == 0
assert sample_by_slug["geel"]["small_box_share"] == 0.5
@@ -161,7 +167,9 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat
]
assert sample_by_slug["turnhout"]["parsed_label_count"] == 1
assert sample_by_slug["turnhout"]["small_box_share"] == 0.0
assert sample_by_slug["turnhout"]["quality_warnings"] == ["median_box_area_below_gate"]
assert sample_by_slug["turnhout"]["quality_warnings"] == [
"median_box_area_below_gate"
]
assert sample_by_slug["postel_bos"]["parsed_label_count"] == 0
assert sample_by_slug["postel_bos"]["quality_warnings"] == []
@@ -173,7 +181,9 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat
assert "small_box_share_above_gate" in warning_codes
assert "positive_tiles_have_low_visual_variance" in warning_codes
markdown = (output_dir / "operator_yolo_dataset_quality_audit.md").read_text(encoding="utf-8")
markdown = (output_dir / "operator_yolo_dataset_quality_audit.md").read_text(
encoding="utf-8"
)
assert "Operator YOLO Dataset Quality Audit" in markdown
assert "Label Quality" in markdown
assert "Minimum visible label ratio" in markdown
+33
View File
@@ -12638,3 +12638,36 @@ Open:
not independent object-level accuracy. A future benchmark must aggregate or
deduplicate objects before uncertainty claims.
- No source label, frozen dataset, model weight or live runtime was changed.
## 2026-08-09 - Immutable min-4px derived corpus
### Improved
- Added a deterministic derived-corpus builder that preserves parent hashes,
copies imagery, records every removed source row and refuses to turn a
positive tile into an unlabeled negative. It creates no release manifest or
human-review claim.
- Derived `yolo-building-aoi1024-reviewedexp6-minpx4-derived-r1` from the
frozen min-3px corpus. Exactly 1,812 marginal labels were removed; all 252
tiles and 77,380 labels remain, with zero excluded tiles.
- Corrected the quality-audit console message from `passed` to `completed`, so
a machine-readable `needs_attention` status is no longer contradicted by
human-facing output.
### Verified on Tower
- Structural audit: zero missing/invalid labels and zero sub-4-pixel rows.
- Relationship audit: zero exact/near duplicates and 41 possible nesting
pairs, down from 43.
- Geometry audit: 10 plausible extreme-aspect rows, down from 26; tile-edge
rows fell from 5,079 to 4,840 without a blanket edge deletion.
- Cross-tile audit: zero reconstructed train/validation crossings.
- Inspected four full contact-sheet pages covering all 252 tiles and 77,380
labels plus the focused nine-tile extreme-aspect sheet. Pure backgrounds
remain empty and sparse contexts remain distinct.
### Decision and limitation
- The min-4px corpus is preferred for future experimental work, but remains
explicitly training/release-ineligible until real human review and governed
release evidence exist. The active GPU model was not changed.
+6 -3
View File
@@ -1164,9 +1164,12 @@ This file now starts with the current implementation status. Older preparation/b
sheets and retain immutable hashes.
- [x] Reconstruct exact cross-tile interior objects and prove that overlap
repeats 40,031 label rows but causes zero reconstructed train/val crossings.
- [ ] Build the next immutable experimental corpus with `min_label_px >= 4`
and compare the existing 0.35 edge-visibility policy against a separately
versioned stricter ablation before any new governed training.
- [x] Build and fully audit an immutable experimental corpus with
`min_label_px=4`; retain 252 tiles and 77,380 labels with zero sub-4-pixel
rows, missing files, invalid labels or cross-split reconstructed objects.
- [ ] Compare the existing 0.35 edge-visibility policy against a separately
versioned stricter ablation from original source geometry before any new
governed training; do not approximate this by dropping every edge label.
- [ ] Convert the AI-assisted ledger into no stronger claim than experimental
triage; a real human must independently review and sign the frozen artifacts
before the governed training wrapper may unlock.
@@ -161,3 +161,29 @@ The edge sample shows expected clipped buildings under the 0.35 minimum-visible
policy. Raising that value may reduce partial-target pressure, but it must be a
separately versioned ablation because removing all 5,079 rows without checking
the original visible fraction would be unsound.
## Derived min-4px experimental corpus
The row-level finding was converted into a new immutable derived corpus rather
than changing the historical dataset. The derivation is checksum-bound to the
79,192-label source summary and removes only labels whose smallest 512-tile
dimension is below 4 pixels. It retains all 252 tiles and 77,380 labels; no
positive tile became empty, and no source file was overwritten.
Fresh audits report zero sub-4-pixel labels, zero invalid or missing label
files, zero exact/high-IoU duplicates and zero reconstructed cross-split
object groups. Extreme-aspect labels fall from 26 to 10, possible nesting from
43 to 41 and tile-edge rows from 5,079 to 4,840. The complete four-page visual
render was inspected again. Pure-background tiles remain visibly empty, sparse
contexts remain distinct and the ten remaining elongated labels predominantly
match plausible long structures.
The generic quality auditor still reports `needs_attention` because normalized
median box area `0.000641` is below its conservative `0.001` warning threshold.
That corresponds to median dimensions near 12 by 13 pixels at source tile
resolution and is not a failed minimum-dimension gate. The warning remains
visible; it was not suppressed or relabelled as success.
This min-4px version is the preferred experimental successor to the legacy
min-3px corpus. It explicitly remains ineligible for governed training until
real human review and all release-contract evidence exist.
+7
View File
@@ -28,6 +28,13 @@ reconstructs global pixel boxes from exporter tile offsets to quantify exact
interior-object repetition caused by overlap; edge rows remain explicitly
unlinked and no repetition is automatically classified as an error.
`derive_yolo_min_dimension_corpus.py` creates a new experimental-only corpus
from an existing checksum-bound YOLO tile dataset. It copies imagery, filters
only labels whose smallest dimension is below the declared pixel floor,
excludes rather than relabels any positive tile that would become empty, and
writes complete source/output hashes plus a row-level removal manifest. It
never creates a training-release manifest or human-review claim.
Setup-, import-, demo- en maintenance-scripts voor GeoIntel.
## WALOUS source provisioning
+53 -16
View File
@@ -20,8 +20,12 @@ def parse_args() -> argparse.Namespace:
"negative balance and label-quality risks."
)
)
parser.add_argument("--summary-path", required=True, help="Path to yolo_tile_dataset_summary.json")
parser.add_argument("--output-dir", required=True, help="Directory for JSON and Markdown reports")
parser.add_argument(
"--summary-path", required=True, help="Path to yolo_tile_dataset_summary.json"
)
parser.add_argument(
"--output-dir", required=True, help="Directory for JSON and Markdown reports"
)
parser.add_argument("--min-positive-samples", type=int, default=6)
parser.add_argument("--min-val-positive-samples", type=int, default=2)
parser.add_argument("--max-repeated-negative-share", type=float, default=0.65)
@@ -86,7 +90,12 @@ def parse_yolo_label_file(path: Path | None) -> tuple[list[dict[str, float]], in
except ValueError:
invalid_count += 1
continue
if not (0 <= center_x <= 1 and 0 <= center_y <= 1 and 0 < width <= 1 and 0 < height <= 1):
if not (
0 <= center_x <= 1
and 0 <= center_y <= 1
and 0 < width <= 1
and 0 < height <= 1
):
invalid_count += 1
continue
boxes.append(
@@ -138,7 +147,9 @@ def summarize_label_files(
areas = [box["area"] for box in boxes]
widths = [box["width"] for box in boxes]
heights = [box["height"] for box in boxes]
aspect_ratios = [max(box["width"] / box["height"], box["height"] / box["width"]) for box in boxes]
aspect_ratios = [
max(box["width"] / box["height"], box["height"] / box["width"]) for box in boxes
]
small_box_count = sum(1 for area in areas if area < small_box_area_threshold)
return {
@@ -175,7 +186,10 @@ def build_label_quality_warning_codes(
median_box_area = label_stats["median_box_area"]
if median_box_area is not None and median_box_area < args.min_median_box_area:
warning_codes.append("median_box_area_below_gate")
if label_stats["parsed_label_count"] and label_stats["small_box_share"] > args.max_small_box_share:
if (
label_stats["parsed_label_count"]
and label_stats["small_box_share"] > args.max_small_box_share
):
warning_codes.append("small_box_share_above_gate")
return warning_codes
@@ -233,7 +247,9 @@ def build_sample_summaries(
result: list[dict[str, Any]] = []
for sample in samples.values():
label_stats = summarize_label_files(sample.pop("_label_file_paths"), args.small_box_area_threshold)
label_stats = summarize_label_files(
sample.pop("_label_file_paths"), args.small_box_area_threshold
)
quality_warnings = build_label_quality_warning_codes(label_stats, args)
result.append(
{
@@ -246,8 +262,14 @@ def build_sample_summaries(
return sorted(result, key=lambda item: str(item["sample_slug"]))
def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Namespace) -> dict[str, Any]:
tiles = [tile for tile in summary.get("tiles", []) if isinstance(tile, dict) and tile.get("kept", True)]
def build_audit(
summary: dict[str, Any], summary_path: Path, args: argparse.Namespace
) -> dict[str, Any]:
tiles = [
tile
for tile in summary.get("tiles", [])
if isinstance(tile, dict) and tile.get("kept", True)
]
sample_summaries = build_sample_summaries(tiles, summary_path, args)
split_counts = Counter(str(tile.get("split") or "unknown") for tile in tiles)
@@ -258,8 +280,12 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
if bool(tile.get("is_negative")) or int(tile.get("label_count") or 0) == 0
]
positive_tiles = [tile for tile in tiles if tile not in negative_tiles]
train_negative_tiles = [tile for tile in negative_tiles if tile.get("split") == "train"]
low_variance_positive_tiles = [tile for tile in positive_tiles if tile.get("low_visual_variance")]
train_negative_tiles = [
tile for tile in negative_tiles if tile.get("split") == "train"
]
low_variance_positive_tiles = [
tile for tile in positive_tiles if tile.get("low_visual_variance")
]
val_positive_samples = {
str(tile.get("sample_slug"))
for tile in positive_tiles
@@ -275,8 +301,12 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
for sample in sample_summaries
if sample["sample_role"] == "background_candidate"
}
repeated_negative_count = sum(1 for tile in negative_tiles if tile.get("is_repeated_background_negative"))
repeated_negative_share = repeated_negative_count / len(negative_tiles) if negative_tiles else 0.0
repeated_negative_count = sum(
1 for tile in negative_tiles if tile.get("is_repeated_background_negative")
)
repeated_negative_share = (
repeated_negative_count / len(negative_tiles) if negative_tiles else 0.0
)
label_stats = summarize_labels(tiles, summary_path, args.small_box_area_threshold)
warnings: list[dict[str, str]] = []
@@ -381,7 +411,10 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
def build_recommendations(warnings: list[dict[str, str]]) -> list[str]:
codes = {warning["code"] for warning in warnings}
recommendations: list[str] = []
if "positive_sample_count_below_gate" in codes or "val_positive_sample_count_below_gate" in codes:
if (
"positive_sample_count_below_gate" in codes
or "val_positive_sample_count_below_gate" in codes
):
recommendations.append(
"Add more labeled positive AOIs before extending training duration or increasing model size."
)
@@ -394,13 +427,17 @@ def build_recommendations(warnings: list[dict[str, str]]) -> list[str]:
"Inspect clipped building labels visually; very small boxes may indicate tile size or label clipping issues."
)
if "label_files_missing" in codes or "invalid_label_rows" in codes:
recommendations.append("Regenerate the YOLO tile dataset and review exporter path/label integrity.")
recommendations.append(
"Regenerate the YOLO tile dataset and review exporter path/label integrity."
)
if "positive_tiles_have_low_visual_variance" in codes:
recommendations.append(
"Reject the raster product for affected AOIs or replace it with an officially complete imagery edition before training."
)
if not recommendations:
recommendations.append("Dataset audit passed the configured gates; continue with benchmarked training.")
recommendations.append(
"Dataset audit passed the configured gates; continue with benchmarked training."
)
return recommendations
@@ -475,7 +512,7 @@ def main() -> int:
json_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
write_markdown(report, markdown_path)
print("Operator YOLO dataset quality audit passed")
print("Operator YOLO dataset quality audit completed")
print(f"Status: {report['status']}")
print(f"JSON: {json_path}")
print(f"Markdown: {markdown_path}")
+246
View File
@@ -0,0 +1,246 @@
#!/usr/bin/env python3
"""Create an immutable experimental YOLO corpus with a stricter pixel floor."""
from __future__ import annotations
import argparse
import hashlib
import json
import shutil
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def filter_label_lines(
lines: list[str], *, tile_size: int, min_dimension_pixels: float
) -> tuple[list[str], list[int]]:
kept: list[str] = []
removed_indices: list[int] = []
for index, line in enumerate(lines):
stripped = line.strip()
if not stripped:
continue
parts = stripped.split()
if len(parts) != 5:
raise ValueError(f"invalid YOLO row at zero-based index {index}")
try:
float(parts[0])
width = float(parts[3])
height = float(parts[4])
except ValueError as exc:
raise ValueError(
f"invalid numeric YOLO row at zero-based index {index}"
) from exc
if width <= 0 or height <= 0 or width > 1 or height > 1:
raise ValueError(f"out-of-range YOLO row at zero-based index {index}")
if min(width, height) * tile_size < min_dimension_pixels:
removed_indices.append(index)
else:
kept.append(stripped)
return kept, removed_indices
def aggregate_hash(paths: list[Path], root: Path) -> str:
digest = hashlib.sha256()
for path in sorted(paths):
relative = path.relative_to(root).as_posix()
digest.update(relative.encode("utf-8"))
digest.update(b"\0")
digest.update(sha256_file(path).encode("ascii"))
digest.update(b"\n")
return digest.hexdigest()
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--source-dir", required=True, type=Path)
parser.add_argument("--output-dir", required=True, type=Path)
parser.add_argument("--min-dimension-pixels", type=float, default=4.0)
args = parser.parse_args()
if args.min_dimension_pixels <= 0:
parser.error("--min-dimension-pixels must be positive")
source_dir = args.source_dir.expanduser().resolve(strict=True)
output_dir = args.output_dir.expanduser().resolve(strict=False)
if output_dir.exists():
parser.error(f"output directory already exists: {output_dir}")
source_summary_path = source_dir / "yolo_tile_dataset_summary.json"
source_summary = json.loads(source_summary_path.read_text(encoding="utf-8"))
tiles = source_summary.get("tiles")
tile_size = source_summary.get("tile_size")
if not isinstance(tiles, list):
raise ValueError("source summary must contain a tiles list")
if not isinstance(tile_size, int) or tile_size <= 0:
raise ValueError("source summary must contain a positive integer tile_size")
source_minimum = source_summary.get("min_label_px")
if isinstance(source_minimum, (int, float)) and args.min_dimension_pixels < float(
source_minimum
):
raise ValueError("derived minimum may not weaken the source pixel floor")
output_dir.mkdir(parents=True)
kept_tiles: list[dict[str, Any]] = []
excluded_tiles: list[dict[str, Any]] = []
removed_labels: list[dict[str, Any]] = []
output_images: list[Path] = []
output_labels: list[Path] = []
for tile in tiles:
if not isinstance(tile, dict) or not tile.get("kept", True):
continue
source_image = Path(str(tile.get("image_path") or "")).resolve(strict=True)
source_label = Path(str(tile.get("label_path") or "")).resolve(strict=True)
image_relative = source_image.relative_to(source_dir)
label_relative = source_label.relative_to(source_dir)
lines = source_label.read_text(encoding="utf-8").splitlines()
kept_lines, removed_indices = filter_label_lines(
lines,
tile_size=tile_size,
min_dimension_pixels=args.min_dimension_pixels,
)
for index in removed_indices:
removed_labels.append(
{
"sample_slug": tile.get("sample_slug"),
"split": tile.get("split"),
"tile_index": tile.get("tile_index"),
"source_label_path": str(source_label),
"source_label_index": index,
"source_row": lines[index].strip(),
"reason": "dimension_below_pixel_floor",
}
)
source_had_labels = any(line.strip() for line in lines)
if source_had_labels and not kept_lines:
excluded_tiles.append(
{
"sample_slug": tile.get("sample_slug"),
"split": tile.get("split"),
"tile_index": tile.get("tile_index"),
"reason": "positive_tile_became_empty_after_filter",
"removed_label_count": len(removed_indices),
"source_image_path": str(source_image),
"source_label_path": str(source_label),
}
)
continue
target_image = output_dir / image_relative
target_label = output_dir / label_relative
target_image.parent.mkdir(parents=True, exist_ok=True)
target_label.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source_image, target_image)
target_label.write_text(
"".join(f"{line}\n" for line in kept_lines), encoding="utf-8"
)
output_images.append(target_image)
output_labels.append(target_label)
derived_tile = dict(tile)
derived_tile.update(
{
"image_path": str(target_image),
"label_path": str(target_label),
"label_count": len(kept_lines),
"is_negative": not kept_lines,
"derived_from_image_path": str(source_image),
"derived_from_label_path": str(source_label),
"removed_label_count": len(removed_indices),
}
)
kept_tiles.append(derived_tile)
dataset_yaml = output_dir / "dataset.yaml"
dataset_yaml.write_text(
"\n".join(
(
f"path: {output_dir}",
"train: images/train",
"val: images/val",
"names:",
" 0: building",
"",
)
),
encoding="utf-8",
)
positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
summary = {
**{key: value for key, value in source_summary.items() if key != "tiles"},
"schema_version": 1,
"status": "experimental_derived_not_release_eligible",
"claim_boundary": (
"Deterministic min-dimension ablation only; no human-review, training, "
"evaluation or production-release claim."
),
"output_dir": str(output_dir),
"dataset_yaml": str(dataset_yaml),
"source_dataset_dir": str(source_dir),
"source_summary_path": str(source_summary_path),
"source_summary_sha256": sha256_file(source_summary_path),
"transformation": "drop_label_if_min_dimension_pixels_below_threshold",
"min_label_px": args.min_dimension_pixels,
"source_min_label_px": source_minimum,
"tile_count": len(kept_tiles),
"positive_tile_count": len(positive_tiles),
"negative_tile_count": len(negative_tiles),
"train_tile_count": sum(tile.get("split") == "train" for tile in kept_tiles),
"val_tile_count": sum(tile.get("split") == "val" for tile in kept_tiles),
"label_count": sum(int(tile["label_count"]) for tile in kept_tiles),
"removed_label_count": len(removed_labels),
"excluded_tile_count": len(excluded_tiles),
"training_release_eligible": False,
"tiles": kept_tiles,
}
summary_path = output_dir / "yolo_tile_dataset_summary.json"
summary_path.write_text(
json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
manifest = {
"schema_version": 1,
"generated_at": datetime.now(UTC).isoformat(),
"status": "complete_experimental_only",
"source_summary_path": str(source_summary_path),
"source_summary_sha256": sha256_file(source_summary_path),
"output_summary_path": str(summary_path),
"output_summary_sha256": sha256_file(summary_path),
"dataset_yaml_sha256": sha256_file(dataset_yaml),
"image_set_aggregate_sha256": aggregate_hash(output_images, output_dir),
"label_set_aggregate_sha256": aggregate_hash(output_labels, output_dir),
"min_dimension_pixels": args.min_dimension_pixels,
"source_label_count": int(source_summary.get("label_count") or 0),
"derived_label_count": summary["label_count"],
"removed_label_count": len(removed_labels),
"excluded_tiles": excluded_tiles,
"removed_labels": removed_labels,
"training_release_eligible": False,
}
manifest_path = output_dir / "derived_corpus_manifest.json"
manifest_path.write_text(
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
print(
json.dumps(
{
"status": manifest["status"],
"derived_label_count": manifest["derived_label_count"],
"removed_label_count": manifest["removed_label_count"],
"excluded_tile_count": len(excluded_tiles),
"manifest_path": str(manifest_path),
},
indent=2,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,21 @@
import pytest
from scripts.derive_yolo_min_dimension_corpus import filter_label_lines
def test_filter_label_lines_removes_only_rows_below_pixel_floor() -> None:
lines = [
"0 0.5 0.5 0.005 0.25",
"0 0.5 0.5 0.0078125 0.25",
"0 0.5 0.5 0.1 0.1",
]
kept, removed = filter_label_lines(lines, tile_size=512, min_dimension_pixels=4)
assert kept == lines[1:]
assert removed == [0]
def test_filter_label_lines_rejects_invalid_source_rows() -> None:
with pytest.raises(ValueError, match="invalid YOLO row"):
filter_label_lines(["0 0.5 0.5"], tile_size=512, min_dimension_pixels=4)