diff --git a/artifacts/evidence/accuracy/model-training/20260809-reviewedexp6-minpx4-derived-corpus.json b/artifacts/evidence/accuracy/model-training/20260809-reviewedexp6-minpx4-derived-corpus.json new file mode 100644 index 00000000..bdc1d39c --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-reviewedexp6-minpx4-derived-corpus.json @@ -0,0 +1,63 @@ +{ + "schema_version": 1, + "evidence_id": "reviewedexp6-minpx4-derived-corpus-r1", + "created_at": "2026-08-09T19:45:41+02:00", + "status": "complete_experimental_only", + "claim_boundary": "Deterministic derived-corpus improvement and AI-assisted visual triage only; no human-review, governed training, evaluation, promotion or production claim.", + "source": { + "dataset_dir": "/app/storage/operator-data/yolo-building-aoi1024-reviewedexp6-minpx3vis035", + "summary_sha256": "7c917e31216d1df2174c0f9c736f88a81f3835aa991971f8fb8665e17ddf5c9c", + "tile_size": 512, + "stride": 256, + "min_label_px": 3.0, + "min_visible_ratio": 0.35, + "tile_count": 252, + "label_count": 79192 + }, + "derived": { + "dataset_dir": "/app/storage/operator-data/yolo-building-aoi1024-reviewedexp6-minpx4-derived-r1", + "manifest_sha256": "8a80d0e9d5ac2d32556e2f1ea0c6009045d48a3e56d931b3161622dca4173a67", + "summary_sha256": "391b0302cc8a46d15fef426d8971a4a8e594fb6d63ed1c7ce16feeee9f1406", + "quality_audit_sha256": "0fc9391ce09a399f51f1e0ed30c677f482c7c1b49b187926da01ecdfd9f1067e", + "outlier_audit_sha256": "c2453abb83aaff8aebd84f63efb14da131225192cf0434d5d1df7cf06be240ca", + "relationship_audit_sha256": "61660d0c448ff579272bc585de8140e82d1577c0b017e1c078b66dc093f14c2e", + "cross_tile_repetition_audit_sha256": "9ca22d83271707713f7f96d5409af598f834787ad5a48be14f8752a704de0da0", + "min_label_px": 4.0, + "tile_count": 252, + "positive_tile_count": 234, + "negative_tile_count": 18, + "label_count": 77380, + "removed_label_count": 1812, + "excluded_tile_count": 0, + "invalid_label_count": 0, + "missing_label_file_count": 0, + "sub_4_pixel_label_count": 0, + "extreme_aspect_label_count": 10, + "tile_edge_label_count": 4840, + "possible_nested_pair_count": 41, + "exact_duplicate_pair_count": 0, + "near_duplicate_pair_count": 0, + "cross_split_repetition_group_count": 0, + "training_release_eligible": false + }, + "visual_review": { + "reviewer_type": "ai_assistant", + "human": false, + "rendered_tile_count": 252, + "rendered_label_count": 77380, + "contact_sheet_sha256s": [ + "f197769ca68da08745277ff93331f3ea93738f6b63eb383a3b6168a9e438ee12", + "e9ecde3f38b4f580adb7523f647460af5d5933e2e1efe1f11f98c1656ca1cd13", + "497bf5878560113fcc59f21aeba09b1de167cb3eff4d0327c7584ba8caeebf70", + "4a8262291ed29699ae59517e1a542e513a50081a225ea1e84268afa37f99d6df" + ], + "extreme_aspect_contact_sheet_sha256": "6c08c42eb6478d1fa06745135dd588ce6522b5cec4adf0bdcd177d77cd6c78a7", + "conclusion": "All 252 tiles remain visually coherent; pure-background tiles remain empty, sparse contexts remain distinct, and the ten remaining extreme-aspect labels predominantly correspond to plausible elongated structures." + }, + "decision": { + "preferred_experimental_corpus": true, + "replaces_source_in_place": false, + "reason": "It deterministically removes all 1,812 visually marginal sub-4-pixel labels while preserving every tile and all larger labels with exact lineage.", + "remaining_gate": "Requires independent human review and the governed release-contract inputs before training eligibility." + } +} diff --git a/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py b/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py index d21357ce..d8ceac27 100644 --- a/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py +++ b/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py @@ -9,7 +9,9 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Path) -> None: +def test_operator_yolo_dataset_quality_audit_reports_dataset_risks( + tmp_path: Path, +) -> None: script_path = ROOT / "scripts" / "audit_operator_yolo_dataset_quality.py" assert script_path.exists() @@ -135,10 +137,12 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat text=True, ) - assert "Operator YOLO dataset quality audit passed" in result.stdout + assert "Operator YOLO dataset quality audit completed" in result.stdout report = json.loads( - (output_dir / "operator_yolo_dataset_quality_audit.json").read_text(encoding="utf-8") + (output_dir / "operator_yolo_dataset_quality_audit.json").read_text( + encoding="utf-8" + ) ) assert report["status"] == "needs_attention" assert report["sample_count"] == 3 @@ -150,7 +154,9 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat assert report["label_stats"]["parsed_label_count"] == 3 assert report["label_stats"]["invalid_label_count"] == 0 assert report["min_label_visible_ratio"] == 0.25 - sample_by_slug = {sample["sample_slug"]: sample for sample in report["sample_summaries"]} + sample_by_slug = { + sample["sample_slug"]: sample for sample in report["sample_summaries"] + } assert sample_by_slug["geel"]["parsed_label_count"] == 2 assert sample_by_slug["geel"]["invalid_label_count"] == 0 assert sample_by_slug["geel"]["small_box_share"] == 0.5 @@ -161,7 +167,9 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat ] assert sample_by_slug["turnhout"]["parsed_label_count"] == 1 assert sample_by_slug["turnhout"]["small_box_share"] == 0.0 - assert sample_by_slug["turnhout"]["quality_warnings"] == ["median_box_area_below_gate"] + assert sample_by_slug["turnhout"]["quality_warnings"] == [ + "median_box_area_below_gate" + ] assert sample_by_slug["postel_bos"]["parsed_label_count"] == 0 assert sample_by_slug["postel_bos"]["quality_warnings"] == [] @@ -173,7 +181,9 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat assert "small_box_share_above_gate" in warning_codes assert "positive_tiles_have_low_visual_variance" in warning_codes - markdown = (output_dir / "operator_yolo_dataset_quality_audit.md").read_text(encoding="utf-8") + markdown = (output_dir / "operator_yolo_dataset_quality_audit.md").read_text( + encoding="utf-8" + ) assert "Operator YOLO Dataset Quality Audit" in markdown assert "Label Quality" in markdown assert "Minimum visible label ratio" in markdown diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index ce54b437..1818d85f 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -12638,3 +12638,36 @@ Open: not independent object-level accuracy. A future benchmark must aggregate or deduplicate objects before uncertainty claims. - No source label, frozen dataset, model weight or live runtime was changed. + +## 2026-08-09 - Immutable min-4px derived corpus + +### Improved + +- Added a deterministic derived-corpus builder that preserves parent hashes, + copies imagery, records every removed source row and refuses to turn a + positive tile into an unlabeled negative. It creates no release manifest or + human-review claim. +- Derived `yolo-building-aoi1024-reviewedexp6-minpx4-derived-r1` from the + frozen min-3px corpus. Exactly 1,812 marginal labels were removed; all 252 + tiles and 77,380 labels remain, with zero excluded tiles. +- Corrected the quality-audit console message from `passed` to `completed`, so + a machine-readable `needs_attention` status is no longer contradicted by + human-facing output. + +### Verified on Tower + +- Structural audit: zero missing/invalid labels and zero sub-4-pixel rows. +- Relationship audit: zero exact/near duplicates and 41 possible nesting + pairs, down from 43. +- Geometry audit: 10 plausible extreme-aspect rows, down from 26; tile-edge + rows fell from 5,079 to 4,840 without a blanket edge deletion. +- Cross-tile audit: zero reconstructed train/validation crossings. +- Inspected four full contact-sheet pages covering all 252 tiles and 77,380 + labels plus the focused nine-tile extreme-aspect sheet. Pure backgrounds + remain empty and sparse contexts remain distinct. + +### Decision and limitation + +- The min-4px corpus is preferred for future experimental work, but remains + explicitly training/release-ineligible until real human review and governed + release evidence exist. The active GPU model was not changed. diff --git a/docs/TODO.md b/docs/TODO.md index a81403ea..c7e18e08 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1164,9 +1164,12 @@ This file now starts with the current implementation status. Older preparation/b sheets and retain immutable hashes. - [x] Reconstruct exact cross-tile interior objects and prove that overlap repeats 40,031 label rows but causes zero reconstructed train/val crossings. -- [ ] Build the next immutable experimental corpus with `min_label_px >= 4` - and compare the existing 0.35 edge-visibility policy against a separately - versioned stricter ablation before any new governed training. +- [x] Build and fully audit an immutable experimental corpus with + `min_label_px=4`; retain 252 tiles and 77,380 labels with zero sub-4-pixel + rows, missing files, invalid labels or cross-split reconstructed objects. +- [ ] Compare the existing 0.35 edge-visibility policy against a separately + versioned stricter ablation from original source geometry before any new + governed training; do not approximate this by dropping every edge label. - [ ] Convert the AI-assisted ledger into no stronger claim than experimental triage; a real human must independently review and sign the frozen artifacts before the governed training wrapper may unlock. diff --git a/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md b/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md index 6424fe1a..9f6f7eab 100644 --- a/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md +++ b/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md @@ -161,3 +161,29 @@ The edge sample shows expected clipped buildings under the 0.35 minimum-visible policy. Raising that value may reduce partial-target pressure, but it must be a separately versioned ablation because removing all 5,079 rows without checking the original visible fraction would be unsound. + +## Derived min-4px experimental corpus + +The row-level finding was converted into a new immutable derived corpus rather +than changing the historical dataset. The derivation is checksum-bound to the +79,192-label source summary and removes only labels whose smallest 512-tile +dimension is below 4 pixels. It retains all 252 tiles and 77,380 labels; no +positive tile became empty, and no source file was overwritten. + +Fresh audits report zero sub-4-pixel labels, zero invalid or missing label +files, zero exact/high-IoU duplicates and zero reconstructed cross-split +object groups. Extreme-aspect labels fall from 26 to 10, possible nesting from +43 to 41 and tile-edge rows from 5,079 to 4,840. The complete four-page visual +render was inspected again. Pure-background tiles remain visibly empty, sparse +contexts remain distinct and the ten remaining elongated labels predominantly +match plausible long structures. + +The generic quality auditor still reports `needs_attention` because normalized +median box area `0.000641` is below its conservative `0.001` warning threshold. +That corresponds to median dimensions near 12 by 13 pixels at source tile +resolution and is not a failed minimum-dimension gate. The warning remains +visible; it was not suppressed or relabelled as success. + +This min-4px version is the preferred experimental successor to the legacy +min-3px corpus. It explicitly remains ineligible for governed training until +real human review and all release-contract evidence exist. diff --git a/scripts/README.md b/scripts/README.md index 50dd2cb3..79a01937 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -28,6 +28,13 @@ reconstructs global pixel boxes from exporter tile offsets to quantify exact interior-object repetition caused by overlap; edge rows remain explicitly unlinked and no repetition is automatically classified as an error. +`derive_yolo_min_dimension_corpus.py` creates a new experimental-only corpus +from an existing checksum-bound YOLO tile dataset. It copies imagery, filters +only labels whose smallest dimension is below the declared pixel floor, +excludes rather than relabels any positive tile that would become empty, and +writes complete source/output hashes plus a row-level removal manifest. It +never creates a training-release manifest or human-review claim. + Setup-, import-, demo- en maintenance-scripts voor GeoIntel. ## WALOUS source provisioning diff --git a/scripts/audit_operator_yolo_dataset_quality.py b/scripts/audit_operator_yolo_dataset_quality.py index 6142d679..b05533ac 100644 --- a/scripts/audit_operator_yolo_dataset_quality.py +++ b/scripts/audit_operator_yolo_dataset_quality.py @@ -20,8 +20,12 @@ def parse_args() -> argparse.Namespace: "negative balance and label-quality risks." ) ) - parser.add_argument("--summary-path", required=True, help="Path to yolo_tile_dataset_summary.json") - parser.add_argument("--output-dir", required=True, help="Directory for JSON and Markdown reports") + parser.add_argument( + "--summary-path", required=True, help="Path to yolo_tile_dataset_summary.json" + ) + parser.add_argument( + "--output-dir", required=True, help="Directory for JSON and Markdown reports" + ) parser.add_argument("--min-positive-samples", type=int, default=6) parser.add_argument("--min-val-positive-samples", type=int, default=2) parser.add_argument("--max-repeated-negative-share", type=float, default=0.65) @@ -86,7 +90,12 @@ def parse_yolo_label_file(path: Path | None) -> tuple[list[dict[str, float]], in except ValueError: invalid_count += 1 continue - if not (0 <= center_x <= 1 and 0 <= center_y <= 1 and 0 < width <= 1 and 0 < height <= 1): + if not ( + 0 <= center_x <= 1 + and 0 <= center_y <= 1 + and 0 < width <= 1 + and 0 < height <= 1 + ): invalid_count += 1 continue boxes.append( @@ -138,7 +147,9 @@ def summarize_label_files( areas = [box["area"] for box in boxes] widths = [box["width"] for box in boxes] heights = [box["height"] for box in boxes] - aspect_ratios = [max(box["width"] / box["height"], box["height"] / box["width"]) for box in boxes] + aspect_ratios = [ + max(box["width"] / box["height"], box["height"] / box["width"]) for box in boxes + ] small_box_count = sum(1 for area in areas if area < small_box_area_threshold) return { @@ -175,7 +186,10 @@ def build_label_quality_warning_codes( median_box_area = label_stats["median_box_area"] if median_box_area is not None and median_box_area < args.min_median_box_area: warning_codes.append("median_box_area_below_gate") - if label_stats["parsed_label_count"] and label_stats["small_box_share"] > args.max_small_box_share: + if ( + label_stats["parsed_label_count"] + and label_stats["small_box_share"] > args.max_small_box_share + ): warning_codes.append("small_box_share_above_gate") return warning_codes @@ -233,7 +247,9 @@ def build_sample_summaries( result: list[dict[str, Any]] = [] for sample in samples.values(): - label_stats = summarize_label_files(sample.pop("_label_file_paths"), args.small_box_area_threshold) + label_stats = summarize_label_files( + sample.pop("_label_file_paths"), args.small_box_area_threshold + ) quality_warnings = build_label_quality_warning_codes(label_stats, args) result.append( { @@ -246,8 +262,14 @@ def build_sample_summaries( return sorted(result, key=lambda item: str(item["sample_slug"])) -def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Namespace) -> dict[str, Any]: - tiles = [tile for tile in summary.get("tiles", []) if isinstance(tile, dict) and tile.get("kept", True)] +def build_audit( + summary: dict[str, Any], summary_path: Path, args: argparse.Namespace +) -> dict[str, Any]: + tiles = [ + tile + for tile in summary.get("tiles", []) + if isinstance(tile, dict) and tile.get("kept", True) + ] sample_summaries = build_sample_summaries(tiles, summary_path, args) split_counts = Counter(str(tile.get("split") or "unknown") for tile in tiles) @@ -258,8 +280,12 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name if bool(tile.get("is_negative")) or int(tile.get("label_count") or 0) == 0 ] positive_tiles = [tile for tile in tiles if tile not in negative_tiles] - train_negative_tiles = [tile for tile in negative_tiles if tile.get("split") == "train"] - low_variance_positive_tiles = [tile for tile in positive_tiles if tile.get("low_visual_variance")] + train_negative_tiles = [ + tile for tile in negative_tiles if tile.get("split") == "train" + ] + low_variance_positive_tiles = [ + tile for tile in positive_tiles if tile.get("low_visual_variance") + ] val_positive_samples = { str(tile.get("sample_slug")) for tile in positive_tiles @@ -275,8 +301,12 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name for sample in sample_summaries if sample["sample_role"] == "background_candidate" } - repeated_negative_count = sum(1 for tile in negative_tiles if tile.get("is_repeated_background_negative")) - repeated_negative_share = repeated_negative_count / len(negative_tiles) if negative_tiles else 0.0 + repeated_negative_count = sum( + 1 for tile in negative_tiles if tile.get("is_repeated_background_negative") + ) + repeated_negative_share = ( + repeated_negative_count / len(negative_tiles) if negative_tiles else 0.0 + ) label_stats = summarize_labels(tiles, summary_path, args.small_box_area_threshold) warnings: list[dict[str, str]] = [] @@ -381,7 +411,10 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name def build_recommendations(warnings: list[dict[str, str]]) -> list[str]: codes = {warning["code"] for warning in warnings} recommendations: list[str] = [] - if "positive_sample_count_below_gate" in codes or "val_positive_sample_count_below_gate" in codes: + if ( + "positive_sample_count_below_gate" in codes + or "val_positive_sample_count_below_gate" in codes + ): recommendations.append( "Add more labeled positive AOIs before extending training duration or increasing model size." ) @@ -394,13 +427,17 @@ def build_recommendations(warnings: list[dict[str, str]]) -> list[str]: "Inspect clipped building labels visually; very small boxes may indicate tile size or label clipping issues." ) if "label_files_missing" in codes or "invalid_label_rows" in codes: - recommendations.append("Regenerate the YOLO tile dataset and review exporter path/label integrity.") + recommendations.append( + "Regenerate the YOLO tile dataset and review exporter path/label integrity." + ) if "positive_tiles_have_low_visual_variance" in codes: recommendations.append( "Reject the raster product for affected AOIs or replace it with an officially complete imagery edition before training." ) if not recommendations: - recommendations.append("Dataset audit passed the configured gates; continue with benchmarked training.") + recommendations.append( + "Dataset audit passed the configured gates; continue with benchmarked training." + ) return recommendations @@ -475,7 +512,7 @@ def main() -> int: json_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") write_markdown(report, markdown_path) - print("Operator YOLO dataset quality audit passed") + print("Operator YOLO dataset quality audit completed") print(f"Status: {report['status']}") print(f"JSON: {json_path}") print(f"Markdown: {markdown_path}") diff --git a/scripts/derive_yolo_min_dimension_corpus.py b/scripts/derive_yolo_min_dimension_corpus.py new file mode 100644 index 00000000..2199f159 --- /dev/null +++ b/scripts/derive_yolo_min_dimension_corpus.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Create an immutable experimental YOLO corpus with a stricter pixel floor.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def filter_label_lines( + lines: list[str], *, tile_size: int, min_dimension_pixels: float +) -> tuple[list[str], list[int]]: + kept: list[str] = [] + removed_indices: list[int] = [] + for index, line in enumerate(lines): + stripped = line.strip() + if not stripped: + continue + parts = stripped.split() + if len(parts) != 5: + raise ValueError(f"invalid YOLO row at zero-based index {index}") + try: + float(parts[0]) + width = float(parts[3]) + height = float(parts[4]) + except ValueError as exc: + raise ValueError( + f"invalid numeric YOLO row at zero-based index {index}" + ) from exc + if width <= 0 or height <= 0 or width > 1 or height > 1: + raise ValueError(f"out-of-range YOLO row at zero-based index {index}") + if min(width, height) * tile_size < min_dimension_pixels: + removed_indices.append(index) + else: + kept.append(stripped) + return kept, removed_indices + + +def aggregate_hash(paths: list[Path], root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(paths): + relative = path.relative_to(root).as_posix() + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(sha256_file(path).encode("ascii")) + digest.update(b"\n") + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source-dir", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--min-dimension-pixels", type=float, default=4.0) + args = parser.parse_args() + if args.min_dimension_pixels <= 0: + parser.error("--min-dimension-pixels must be positive") + + source_dir = args.source_dir.expanduser().resolve(strict=True) + output_dir = args.output_dir.expanduser().resolve(strict=False) + if output_dir.exists(): + parser.error(f"output directory already exists: {output_dir}") + source_summary_path = source_dir / "yolo_tile_dataset_summary.json" + source_summary = json.loads(source_summary_path.read_text(encoding="utf-8")) + tiles = source_summary.get("tiles") + tile_size = source_summary.get("tile_size") + if not isinstance(tiles, list): + raise ValueError("source summary must contain a tiles list") + if not isinstance(tile_size, int) or tile_size <= 0: + raise ValueError("source summary must contain a positive integer tile_size") + source_minimum = source_summary.get("min_label_px") + if isinstance(source_minimum, (int, float)) and args.min_dimension_pixels < float( + source_minimum + ): + raise ValueError("derived minimum may not weaken the source pixel floor") + + output_dir.mkdir(parents=True) + kept_tiles: list[dict[str, Any]] = [] + excluded_tiles: list[dict[str, Any]] = [] + removed_labels: list[dict[str, Any]] = [] + output_images: list[Path] = [] + output_labels: list[Path] = [] + for tile in tiles: + if not isinstance(tile, dict) or not tile.get("kept", True): + continue + source_image = Path(str(tile.get("image_path") or "")).resolve(strict=True) + source_label = Path(str(tile.get("label_path") or "")).resolve(strict=True) + image_relative = source_image.relative_to(source_dir) + label_relative = source_label.relative_to(source_dir) + lines = source_label.read_text(encoding="utf-8").splitlines() + kept_lines, removed_indices = filter_label_lines( + lines, + tile_size=tile_size, + min_dimension_pixels=args.min_dimension_pixels, + ) + for index in removed_indices: + removed_labels.append( + { + "sample_slug": tile.get("sample_slug"), + "split": tile.get("split"), + "tile_index": tile.get("tile_index"), + "source_label_path": str(source_label), + "source_label_index": index, + "source_row": lines[index].strip(), + "reason": "dimension_below_pixel_floor", + } + ) + source_had_labels = any(line.strip() for line in lines) + if source_had_labels and not kept_lines: + excluded_tiles.append( + { + "sample_slug": tile.get("sample_slug"), + "split": tile.get("split"), + "tile_index": tile.get("tile_index"), + "reason": "positive_tile_became_empty_after_filter", + "removed_label_count": len(removed_indices), + "source_image_path": str(source_image), + "source_label_path": str(source_label), + } + ) + continue + + target_image = output_dir / image_relative + target_label = output_dir / label_relative + target_image.parent.mkdir(parents=True, exist_ok=True) + target_label.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_image, target_image) + target_label.write_text( + "".join(f"{line}\n" for line in kept_lines), encoding="utf-8" + ) + output_images.append(target_image) + output_labels.append(target_label) + derived_tile = dict(tile) + derived_tile.update( + { + "image_path": str(target_image), + "label_path": str(target_label), + "label_count": len(kept_lines), + "is_negative": not kept_lines, + "derived_from_image_path": str(source_image), + "derived_from_label_path": str(source_label), + "removed_label_count": len(removed_indices), + } + ) + kept_tiles.append(derived_tile) + + dataset_yaml = output_dir / "dataset.yaml" + dataset_yaml.write_text( + "\n".join( + ( + f"path: {output_dir}", + "train: images/train", + "val: images/val", + "names:", + " 0: building", + "", + ) + ), + encoding="utf-8", + ) + positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]] + negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]] + summary = { + **{key: value for key, value in source_summary.items() if key != "tiles"}, + "schema_version": 1, + "status": "experimental_derived_not_release_eligible", + "claim_boundary": ( + "Deterministic min-dimension ablation only; no human-review, training, " + "evaluation or production-release claim." + ), + "output_dir": str(output_dir), + "dataset_yaml": str(dataset_yaml), + "source_dataset_dir": str(source_dir), + "source_summary_path": str(source_summary_path), + "source_summary_sha256": sha256_file(source_summary_path), + "transformation": "drop_label_if_min_dimension_pixels_below_threshold", + "min_label_px": args.min_dimension_pixels, + "source_min_label_px": source_minimum, + "tile_count": len(kept_tiles), + "positive_tile_count": len(positive_tiles), + "negative_tile_count": len(negative_tiles), + "train_tile_count": sum(tile.get("split") == "train" for tile in kept_tiles), + "val_tile_count": sum(tile.get("split") == "val" for tile in kept_tiles), + "label_count": sum(int(tile["label_count"]) for tile in kept_tiles), + "removed_label_count": len(removed_labels), + "excluded_tile_count": len(excluded_tiles), + "training_release_eligible": False, + "tiles": kept_tiles, + } + summary_path = output_dir / "yolo_tile_dataset_summary.json" + summary_path.write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + manifest = { + "schema_version": 1, + "generated_at": datetime.now(UTC).isoformat(), + "status": "complete_experimental_only", + "source_summary_path": str(source_summary_path), + "source_summary_sha256": sha256_file(source_summary_path), + "output_summary_path": str(summary_path), + "output_summary_sha256": sha256_file(summary_path), + "dataset_yaml_sha256": sha256_file(dataset_yaml), + "image_set_aggregate_sha256": aggregate_hash(output_images, output_dir), + "label_set_aggregate_sha256": aggregate_hash(output_labels, output_dir), + "min_dimension_pixels": args.min_dimension_pixels, + "source_label_count": int(source_summary.get("label_count") or 0), + "derived_label_count": summary["label_count"], + "removed_label_count": len(removed_labels), + "excluded_tiles": excluded_tiles, + "removed_labels": removed_labels, + "training_release_eligible": False, + } + manifest_path = output_dir / "derived_corpus_manifest.json" + manifest_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print( + json.dumps( + { + "status": manifest["status"], + "derived_label_count": manifest["derived_label_count"], + "removed_label_count": manifest["removed_label_count"], + "excluded_tile_count": len(excluded_tiles), + "manifest_path": str(manifest_path), + }, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_derive_yolo_min_dimension_corpus.py b/tests/test_derive_yolo_min_dimension_corpus.py new file mode 100644 index 00000000..7c63c452 --- /dev/null +++ b/tests/test_derive_yolo_min_dimension_corpus.py @@ -0,0 +1,21 @@ +import pytest + +from scripts.derive_yolo_min_dimension_corpus import filter_label_lines + + +def test_filter_label_lines_removes_only_rows_below_pixel_floor() -> None: + lines = [ + "0 0.5 0.5 0.005 0.25", + "0 0.5 0.5 0.0078125 0.25", + "0 0.5 0.5 0.1 0.1", + ] + + kept, removed = filter_label_lines(lines, tile_size=512, min_dimension_pixels=4) + + assert kept == lines[1:] + assert removed == [0] + + +def test_filter_label_lines_rejects_invalid_source_rows() -> None: + with pytest.raises(ValueError, match="invalid YOLO row"): + filter_label_lines(["0 0.5 0.5"], tile_size=512, min_dimension_pixels=4)