From 48084799c9987e41e397b61a6d7f5ab055794a6e Mon Sep 17 00:00:00 2001 From: Jens Date: Sun, 9 Aug 2026 20:41:54 +0200 Subject: [PATCH] correct overlapping checkpoint evaluation --- ...0809-v69-nonoverlap-checkpoint-review.json | 72 +++++ ...print129_operator_yolo_training_dataset.py | 51 +++- docs/CODEX_EXECUTION_LOG.md | 33 +++ docs/TODO.md | 7 + ...026-08-09-ai-assisted-checkpoint-review.md | 29 ++ scripts/README.md | 8 + .../derive_yolo_nonoverlap_evaluation_view.py | 259 ++++++++++++++++++ scripts/evaluate_yolo_checkpoint_matrix.py | 13 +- scripts/train_operator_yolo_detector.sh | 6 + ..._derive_yolo_nonoverlap_evaluation_view.py | 49 ++++ tests/test_evaluate_yolo_checkpoint_matrix.py | 22 +- 11 files changed, 539 insertions(+), 10 deletions(-) create mode 100644 artifacts/evidence/accuracy/model-training/20260809-v69-nonoverlap-checkpoint-review.json create mode 100644 scripts/derive_yolo_nonoverlap_evaluation_view.py create mode 100644 tests/test_derive_yolo_nonoverlap_evaluation_view.py diff --git a/artifacts/evidence/accuracy/model-training/20260809-v69-nonoverlap-checkpoint-review.json b/artifacts/evidence/accuracy/model-training/20260809-v69-nonoverlap-checkpoint-review.json new file mode 100644 index 00000000..f762bd24 --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v69-nonoverlap-checkpoint-review.json @@ -0,0 +1,72 @@ +{ + "schema_version": 1, + "run_id": "building-kempen-v69-nonoverlap-checkpoint-review-r2", + "generated_at": "2026-08-09T20:35:51+02:00", + "status": "completed_no_production_change", + "claim_boundary": "Non-protected two-positive-AOI and one-background-AOI validation refinement only; tiles do not overlap, but statistical independence, protected-test performance, national accuracy and promotion eligibility are not established.", + "correction_to_prior_evidence": { + "prior_validation_tile_count": 36, + "prior_tile_overlap_pixels": 256, + "prior_background_tile_count": 18, + "invalid_background_sample": "arendonk_heide", + "invalid_background_reason": "all nine source tiles are visually blank/low-variance no-data; four non-overlap representatives were excluded with explicit reason codes", + "effect": "The prior matrix remains a historical overlapping-tile ranking, but its 18-image background count must not be interpreted as 18 real background observations." + }, + "evaluation_view": { + "path": "/app/storage/operator-data/yolo-building-aoi1024-bgaware512r8-nonoverlap-eval-r2", + "manifest_sha256": "69f555e57fa52d8b7cdfc5632167bde96b64030ee37e9bf49ff2d9891d66574a", + "summary_sha256": "c17a8c47eaecc2dfb38778ff5a612f9e6c4d67c9e90e31d3a6a66d8857adec4c", + "cross_tile_audit_sha256": "b0a49d7f5983ccaa07754a1e3d6d8f7b10345d3c57b9f613bb856de9d6f70b74", + "contact_sheet_sha256": "f5954a2f6ad3dc59642bf3a51d98ca94343491f522968ab38d60c999ed21c797", + "tile_size": 512, + "stride": 512, + "tile_count": 12, + "positive_tile_count": 8, + "real_background_tile_count": 4, + "positive_aoi_count": 2, + "background_aoi_count": 1, + "label_count": 2551, + "unique_interior_object_count": 2384, + "edge_label_count": 167, + "repeated_interior_object_count": 0, + "excluded_low_variance_tile_count": 4, + "excluded_sample_slug": "arendonk_heide", + "training_prohibited": true, + "statistical_independence_established": false + }, + "runtime": { + "device": "cuda:0", + "gpu": "NVIDIA GeForce RTX 4080 SUPER", + "torch": "2.11.0+cu128", + "ultralytics": "8.4.99" + }, + "checkpoint_metrics": { + "active": { + "model_sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1", + "precision": 0.5426324241859346, + "recall": 0.4353171786761234, + "map50": 0.342034346792601, + "map50_95": 0.1413160008545276, + "background_detections_confidence_0_15": 0, + "background_detections_confidence_0_25": 0 + }, + "challenger": { + "model_sha256": "038f1f97a6afd534f29e1f392a730a58207b928ca01e31ab8d8fed6106705820", + "precision": 0.5817150200182469, + "recall": 0.462299518515736, + "map50": 0.3683635371499511, + "map50_95": 0.15531818619443444, + "background_detections_confidence_0_15": 1, + "background_detections_confidence_0_25": 0 + }, + "confidence_0_15_matrix_sha256": "915c8523fe71a2d88b44bcb75d9070d25940f3969213692cc121f1239b31489b", + "confidence_0_25_matrix_sha256": "173d8af94d5d3d805b67a1a99d36e196255166e2e32ecabbef40ebf065c92138" + }, + "decision": { + "relative_metric_ranking": "challenger", + "strict_background_ranking_at_0_15": "active", + "background_tie_at_0_25": true, + "production_model_changed": false, + "reason": "The challenger remains stronger on this refined non-overlapping ranking and clears Postel at 0.25, but one real background AOI and two adjacent positive AOIs are insufficient for release; historical provenance and human-review gates also remain closed." + } +} diff --git a/backend/tests/test_sprint129_operator_yolo_training_dataset.py b/backend/tests/test_sprint129_operator_yolo_training_dataset.py index 9d8f5bc0..d2a76b45 100644 --- a/backend/tests/test_sprint129_operator_yolo_training_dataset.py +++ b/backend/tests/test_sprint129_operator_yolo_training_dataset.py @@ -1,4 +1,5 @@ from pathlib import Path +import shlex import subprocess import sys @@ -8,7 +9,9 @@ ROOT = Path(__file__).resolve().parents[2] def test_operator_yolo_dataset_export_script_contract() -> None: script_path = ROOT / "scripts" / "export_operator_yolo_dataset.py" - readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text( + encoding="utf-8" + ) assert script_path.exists() script = script_path.read_text(encoding="utf-8") @@ -42,14 +45,18 @@ def test_operator_yolo_dataset_export_help_does_not_require_gis_dependencies() - ) assert result.returncode == 0 - assert "Export operator real-data samples to a YOLO detection dataset" in result.stdout + assert ( + "Export operator real-data samples to a YOLO detection dataset" in result.stdout + ) assert "--manifest-path" in result.stdout assert "--val-samples" in result.stdout def test_operator_yolo_train_smoke_script_contract() -> None: script_path = ROOT / "scripts" / "train_operator_yolo_detector.sh" - readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text( + encoding="utf-8" + ) assert script_path.exists() script = script_path.read_text(encoding="utf-8") @@ -62,7 +69,7 @@ def test_operator_yolo_train_smoke_script_contract() -> None: assert "TRAIN_EPOCHS" in script assert "TRAIN_IMGSZ" in script assert "/opt/geointel/venv/bin/python" in script - assert "PYTHON_BIN=\"python3\"" in script + assert 'PYTHON_BIN="python3"' in script assert "dataset.yaml" in script assert "from ultralytics import YOLO" in script assert "model.train" in script @@ -76,6 +83,42 @@ def test_operator_yolo_train_smoke_script_contract() -> None: assert '"base_model_sha256"' in script assert '"trained_model_sha256"' in script assert "training_release_manifest.py" in script + assert "NO_TRAINING.json" in script + assert "Training is prohibited for this evaluation-only dataset" in script assert "verify" in script assert "download" not in script.lower() assert "fixture_mode" not in script + + +def test_operator_yolo_train_rejects_evaluation_only_marker(tmp_path: Path) -> None: + dataset_dir = tmp_path / "evaluation-only" + dataset_dir.mkdir() + (dataset_dir / "dataset.yaml").write_text("val: images/val\n", encoding="utf-8") + (dataset_dir / "NO_TRAINING.json").write_text( + '{"training_prohibited": true}\n', encoding="utf-8" + ) + + def shell_path(path: Path) -> str: + if path.drive: + return f"/mnt/{path.drive[0].lower()}/{path.as_posix()[3:]}" + return str(path) + + dataset_shell_path = shell_path(dataset_dir) + model_shell_path = shell_path(tmp_path / "missing.pt") + script_shell_path = shell_path(ROOT / "scripts" / "train_operator_yolo_detector.sh") + command = ( + f"OPERATOR_YOLO_DATASET_DIR={shlex.quote(dataset_shell_path)} " + f"YOLO_BASE_MODEL_PATH={shlex.quote(model_shell_path)} " + f"bash {shlex.quote(script_shell_path)}" + ) + + result = subprocess.run( + ["bash", "-lc", command], + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 1 + assert "Training is prohibited for this evaluation-only dataset" in result.stderr + assert "Base model file not found" not in result.stderr diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 1818d85f..09e0efc4 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -12671,3 +12671,36 @@ Open: - The min-4px corpus is preferred for future experimental work, but remains explicitly training/release-ineligible until real human review and governed release evidence exist. The active GPU model was not changed. + +## 2026-08-09 - Non-overlapping checkpoint correction + +### Corrected + +- Traced the prior 36-image checkpoint matrix to four AOIs tiled at 512/256. + The same interiors were repeated and nine Arendonk-heide tiles were visually + blank no-data, despite being counted as background observations. +- Added a deterministic non-overlap evaluation-view builder. It selects four + complete 512 px grid tiles per 1024 px AOI, excludes low-variance no-data + with reason codes, creates empty train directories and writes a hashed + `NO_TRAINING.json` marker. +- Added a fail-fast marker check to the operator YOLO training wrapper before + release verification, model loading or CUDA allocation. +- Corrected checkpoint overlap semantics: zero pixel overlap is reported, but + statistical independence is explicitly not established for adjacent tiles. + +### GPU evidence + +- The clean r2 view contains 12 images: eight positive Turnhout/Westerlo tiles + and four real Postel backgrounds. It has 2,551 labels, 2,384 unique interior + objects, 167 edge labels and zero repeated interior objects. +- Active: precision 0.542632, recall 0.435317, mAP50 0.342034, mAP50-95 + 0.141316. Challenger: 0.581715, 0.462300, 0.368364 and 0.155318. +- At confidence 0.15 the active model has zero Postel detections and the + challenger one; at 0.25 both have zero. The threshold-sensitive relative + challenger advantage is confirmed without changing production. + +### Remaining limitations + +- Two adjacent positive AOIs and one real background AOI are not representative + release evidence. Human review, broader independent geography and exact + challenger provenance remain blocking. diff --git a/docs/TODO.md b/docs/TODO.md index c7e18e08..430f8ae3 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1170,6 +1170,13 @@ This file now starts with the current implementation status. Older preparation/b - [ ] Compare the existing 0.35 edge-visibility policy against a separately versioned stricter ablation from original source geometry before any new governed training; do not approximate this by dropping every edge label. +- [x] Replace the overlapping checkpoint ranking with a checksum-bound, + training-disabled non-overlap view: 12 real tiles, 2,384 unique interior + objects, zero repeated interiors and explicit removal of blank Arendonk + no-data imagery. +- [ ] Add multiple independent real-background AOIs and more geographically + separated positive validation AOIs before interpreting non-overlap ranking + as release evidence; Postel alone is not representative. - [ ] Convert the AI-assisted ledger into no stronger claim than experimental triage; a real human must independently review and sign the frozen artifacts before the governed training wrapper may unlock. diff --git a/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md b/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md index 9f6f7eab..34256554 100644 --- a/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md +++ b/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md @@ -187,3 +187,32 @@ visible; it was not suppressed or relabelled as success. This min-4px version is the preferred experimental successor to the legacy min-3px corpus. It explicitly remains ineligible for governed training until real human review and all release-contract evidence exist. + +## Non-overlapping checkpoint re-evaluation + +The 36-tile checkpoint matrix was traced to a 512 px validation corpus with +stride 256. It contained nine views each of Turnhout, Westerlo, Postel-bos and +Arendonk-heide. Visual inspection then exposed that all Arendonk-heide views +are blank/no-data imagery, not meaningful pure-background observations. The +historical claim of 18 background images is therefore corrected: nine were +blank no-data and nine represented Postel under overlap. + +A training-disabled validation view now covers each 1024 px AOI with four +non-overlapping 512 px tiles. Four blank Arendonk representatives are excluded +with explicit reason codes. The resulting set contains eight positive tiles +from Turnhout/Westerlo, four real Postel background tiles and 2,551 labels. +Exact reconstruction finds 2,384 unique interior objects, 167 edge labels and +zero repeated interior objects. The view has empty train directories, a +checksum-bound `NO_TRAINING.json`, and the training wrapper rejects that marker. + +On the Tower RTX 4080 SUPER, the active checkpoint measures precision +`0.542632`, recall `0.435317`, mAP50 `0.342034` and mAP50-95 `0.141316`. +The reviewedexp6 challenger measures `0.581715`, `0.462300`, `0.368364` and +`0.155318` respectively. At background confidence 0.15 the active checkpoint +has zero Postel detections and the challenger has one. At 0.25 both have zero. +Thus the refined evidence confirms the challenger's relative metric advantage +and its threshold sensitivity, but still does not authorize promotion. + +Non-overlap is not overstated as statistical independence: the positive tiles +remain adjacent and come from only two AOIs, edge objects can remain split, and +there is only one real background AOI. The production model remains unchanged. diff --git a/scripts/README.md b/scripts/README.md index 79a01937..3cd7c979 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -35,6 +35,14 @@ excludes rather than relabels any positive tile that would become empty, and writes complete source/output hashes plus a row-level removal manifest. It never creates a training-release manifest or human-review claim. +`derive_yolo_nonoverlap_evaluation_view.py` selects the complete non-overlap +grid from an existing overlapping validation corpus, records every excluded +low-variance/no-data tile, leaves train directories empty and writes a +checksum-bound `NO_TRAINING.json`. The operator training wrapper rejects that +marker before release verification or CUDA allocation. Non-overlap proves only +that tiles do not share pixels; checkpoint reports explicitly avoid claiming +statistical independence for adjacent tiles from the same AOI. + Setup-, import-, demo- en maintenance-scripts voor GeoIntel. ## WALOUS source provisioning diff --git a/scripts/derive_yolo_nonoverlap_evaluation_view.py b/scripts/derive_yolo_nonoverlap_evaluation_view.py new file mode 100644 index 00000000..867176d5 --- /dev/null +++ b/scripts/derive_yolo_nonoverlap_evaluation_view.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Derive a checksum-bound, training-disabled non-overlapping YOLO val view.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +from collections import defaultdict +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from PIL import Image + +try: + from scripts.audit_yolo_cross_tile_repetition import tile_offsets +except ModuleNotFoundError: # Standalone operator-tool copy beside the auditor. + from audit_yolo_cross_tile_repetition import tile_offsets + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def select_nonoverlap_tiles( + tiles: list[dict[str, Any]], tile_size: int +) -> list[dict[str, Any]]: + selected: list[dict[str, Any]] = [] + for tile in tiles: + if not isinstance(tile, dict) or not tile.get("kept", True): + continue + if str(tile.get("split") or "") != "val": + continue + row_offset, column_offset = tile_offsets(str(tile.get("image_path") or "")) + if row_offset % tile_size == 0 and column_offset % tile_size == 0: + selected.append(tile) + return sorted( + selected, + key=lambda tile: ( + str(tile.get("sample_slug") or ""), + int(tile.get("tile_index") or 0), + ), + ) + + +def aggregate_hash(paths: list[Path], root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(paths): + digest.update(path.relative_to(root).as_posix().encode("utf-8")) + digest.update(b"\0") + digest.update(sha256_file(path).encode("ascii")) + digest.update(b"\n") + return digest.hexdigest() + + +def image_has_low_visual_variance(image_path: Path, blank_range_threshold: int) -> bool: + image = Image.open(image_path).convert("L") + minimum, maximum = image.getextrema() + return maximum - minimum <= blank_range_threshold + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source-dir", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument( + "--exclude-low-variance", + action=argparse.BooleanOptionalAction, + default=True, + ) + parser.add_argument("--blank-range-threshold", type=int, default=3) + args = parser.parse_args() + + source_dir = args.source_dir.expanduser().resolve(strict=True) + output_dir = args.output_dir.expanduser().resolve(strict=False) + if output_dir.exists(): + parser.error(f"output directory already exists: {output_dir}") + source_summary_path = source_dir / "yolo_tile_dataset_summary.json" + source_summary = json.loads(source_summary_path.read_text(encoding="utf-8")) + tiles = source_summary.get("tiles") + tile_size = source_summary.get("tile_size") + if not isinstance(tiles, list): + raise ValueError("source summary must contain a tiles list") + if not isinstance(tile_size, int) or tile_size <= 0: + raise ValueError("source summary must contain a positive integer tile_size") + selected_candidates = select_nonoverlap_tiles(tiles, tile_size) + if not selected_candidates: + raise ValueError("no non-overlapping validation tiles were selected") + + selected: list[dict[str, Any]] = [] + excluded_tiles: list[dict[str, Any]] = [] + for tile in selected_candidates: + source_image = Path(str(tile.get("image_path") or "")).resolve(strict=True) + if args.exclude_low_variance and image_has_low_visual_variance( + source_image, args.blank_range_threshold + ): + excluded_tiles.append( + { + "sample_slug": str(tile.get("sample_slug") or "unknown"), + "tile_index": int(tile.get("tile_index") or 0), + "source_image_path": str(source_image), + "reason": "low_visual_variance_no_data", + } + ) + continue + selected.append(tile) + if not selected: + raise ValueError("every non-overlapping validation tile was excluded") + + offsets_by_sample: dict[str, set[tuple[int, int]]] = defaultdict(set) + source_val_counts: dict[str, int] = defaultdict(int) + for tile in tiles: + if ( + isinstance(tile, dict) + and tile.get("kept", True) + and tile.get("split") == "val" + ): + source_val_counts[str(tile.get("sample_slug") or "unknown")] += 1 + for tile in selected: + offsets_by_sample[str(tile.get("sample_slug") or "unknown")].add( + tile_offsets(str(tile.get("image_path") or "")) + ) + accounted_samples = set(offsets_by_sample) | { + tile["sample_slug"] for tile in excluded_tiles + } + if accounted_samples != set(source_val_counts): + raise ValueError("non-overlap selection has an unaccounted validation sample") + + output_dir.mkdir(parents=True) + (output_dir / "images" / "train").mkdir(parents=True) + (output_dir / "labels" / "train").mkdir(parents=True) + output_tiles: list[dict[str, Any]] = [] + output_files: list[Path] = [] + for tile in selected: + source_image = Path(str(tile.get("image_path") or "")).resolve(strict=True) + source_label = Path(str(tile.get("label_path") or "")).resolve(strict=True) + target_image = output_dir / "images" / "val" / source_image.name + target_label = output_dir / "labels" / "val" / source_label.name + target_image.parent.mkdir(parents=True, exist_ok=True) + target_label.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_image, target_image) + shutil.copy2(source_label, target_label) + output_files.extend((target_image, target_label)) + derived_tile = dict(tile) + derived_tile.update( + { + "image_path": str(target_image), + "label_path": str(target_label), + "derived_from_image_path": str(source_image), + "derived_from_label_path": str(source_label), + } + ) + output_tiles.append(derived_tile) + + dataset_yaml = output_dir / "dataset.yaml" + dataset_yaml.write_text( + "\n".join( + ( + f"path: {output_dir}", + "train: images/train", + "val: images/val", + "names:", + " 0: building", + "", + ) + ), + encoding="utf-8", + ) + no_training = { + "schema_version": 1, + "training_prohibited": True, + "reason": "Immutable non-overlapping evaluation view; train directories are intentionally empty.", + "source_summary_sha256": sha256_file(source_summary_path), + } + no_training_path = output_dir / "NO_TRAINING.json" + no_training_path.write_text( + json.dumps(no_training, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + summary = { + "schema_version": 1, + "status": "ready_evaluation_only", + "claim_boundary": ( + "Non-protected, non-overlapping tile-level validation view; no " + "training, protected-test, object-level independence or promotion claim." + ), + "dataset_yaml": str(dataset_yaml), + "output_dir": str(output_dir), + "source_dataset_dir": str(source_dir), + "source_summary_path": str(source_summary_path), + "source_summary_sha256": sha256_file(source_summary_path), + "class_names": source_summary.get("class_names", ["building"]), + "tile_size": tile_size, + "stride": tile_size, + "source_stride": source_summary.get("stride"), + "training_prohibited": True, + "training_marker": str(no_training_path), + "tile_count": len(output_tiles), + "val_tile_count": len(output_tiles), + "train_tile_count": 0, + "positive_tile_count": sum( + int(tile.get("label_count") or 0) > 0 for tile in output_tiles + ), + "negative_tile_count": sum( + int(tile.get("label_count") or 0) == 0 for tile in output_tiles + ), + "label_count": sum(int(tile.get("label_count") or 0) for tile in output_tiles), + "selected_offsets_by_sample": { + sample: [list(offset) for offset in sorted(offsets)] + for sample, offsets in sorted(offsets_by_sample.items()) + }, + "source_val_tile_counts_by_sample": dict(sorted(source_val_counts.items())), + "excluded_tile_count": len(excluded_tiles), + "excluded_tiles": excluded_tiles, + "excluded_sample_slugs": sorted( + {tile["sample_slug"] for tile in excluded_tiles} + ), + "exclude_low_variance": args.exclude_low_variance, + "blank_range_threshold": args.blank_range_threshold, + "tiles": output_tiles, + } + summary_path = output_dir / "yolo_tile_dataset_summary.json" + summary_path.write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + manifest = { + "schema_version": 1, + "generated_at": datetime.now(UTC).isoformat(), + "status": "complete_evaluation_only", + "source_summary_path": str(source_summary_path), + "source_summary_sha256": sha256_file(source_summary_path), + "output_summary_path": str(summary_path), + "output_summary_sha256": sha256_file(summary_path), + "dataset_yaml_sha256": sha256_file(dataset_yaml), + "evaluation_file_set_aggregate_sha256": aggregate_hash( + output_files, output_dir + ), + "training_marker_sha256": sha256_file(no_training_path), + "training_prohibited": True, + "selected_tile_count": len(output_tiles), + "selected_sample_count": len(offsets_by_sample), + "excluded_tile_count": len(excluded_tiles), + "excluded_tiles": excluded_tiles, + } + manifest_path = output_dir / "evaluation_view_manifest.json" + manifest_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(json.dumps(manifest, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluate_yolo_checkpoint_matrix.py b/scripts/evaluate_yolo_checkpoint_matrix.py index 83f6d771..e4983fa0 100644 --- a/scripts/evaluate_yolo_checkpoint_matrix.py +++ b/scripts/evaluate_yolo_checkpoint_matrix.py @@ -59,7 +59,8 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]: return { "status": "unavailable", "summary_path": str(summary_path), - "validation_rows_independent": None, + "validation_tiles_non_overlapping": None, + "statistical_independence_established": False, } payload = json.loads(summary_path.read_text(encoding="utf-8")) tile_size = payload.get("tile_size") @@ -69,7 +70,8 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]: "status": "invalid", "summary_path": str(summary_path), "summary_sha256": sha256_file(summary_path), - "validation_rows_independent": None, + "validation_tiles_non_overlapping": None, + "statistical_independence_established": False, } overlap_pixels = max(tile_size - stride, 0) return { @@ -79,12 +81,15 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]: "tile_size": tile_size, "stride": stride, "overlap_pixels": overlap_pixels, - "validation_rows_independent": overlap_pixels == 0, + "validation_tiles_non_overlapping": overlap_pixels == 0, + "statistical_independence_established": False, "interpretation": ( "Tile metrics can repeat the same source object and are valid for " "candidate ranking only, not independent object-level uncertainty." if overlap_pixels - else "Tile rows do not overlap according to the dataset summary." + else "Tiles do not overlap, but spatial/statistical independence is " + "not established because adjacent tiles share AOI context and edge " + "objects can remain split." ), } diff --git a/scripts/train_operator_yolo_detector.sh b/scripts/train_operator_yolo_detector.sh index 76a042ce..5764a148 100644 --- a/scripts/train_operator_yolo_detector.sh +++ b/scripts/train_operator_yolo_detector.sh @@ -56,6 +56,7 @@ fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml" +NO_TRAINING_MARKER="${OPERATOR_YOLO_DATASET_DIR%/}/NO_TRAINING.json" SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json" export DATASET_YAML export YOLO_BASE_MODEL_PATH @@ -76,6 +77,11 @@ if [[ ! -f "${DATASET_YAML}" ]]; then exit 1 fi +if [[ -f "${NO_TRAINING_MARKER}" ]]; then + echo "Training is prohibited for this evaluation-only dataset: ${NO_TRAINING_MARKER}" >&2 + exit 1 +fi + if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then echo "Base model file not found: ${YOLO_BASE_MODEL_PATH}" >&2 exit 1 diff --git a/tests/test_derive_yolo_nonoverlap_evaluation_view.py b/tests/test_derive_yolo_nonoverlap_evaluation_view.py new file mode 100644 index 00000000..ed5de918 --- /dev/null +++ b/tests/test_derive_yolo_nonoverlap_evaluation_view.py @@ -0,0 +1,49 @@ +from pathlib import Path + +from PIL import Image + +from scripts.derive_yolo_nonoverlap_evaluation_view import ( + image_has_low_visual_variance, + select_nonoverlap_tiles, +) + + +def tile(name: str, index: int, split: str = "val") -> dict[str, object]: + return { + "image_path": f"/data/sample_{index:04d}_{name}.png", + "tile_index": index, + "sample_slug": "sample", + "split": split, + "kept": True, + } + + +def test_select_nonoverlap_tiles_keeps_complete_512_grid() -> None: + tiles = [ + tile("r0_c0", 0), + tile("r0_c256", 1), + tile("r0_c512", 2), + tile("r256_c0", 3), + tile("r256_c256", 4), + tile("r256_c512", 5), + tile("r512_c0", 6), + tile("r512_c256", 7), + tile("r512_c512", 8), + tile("r0_c0", 9, split="train"), + ] + + selected = select_nonoverlap_tiles(tiles, 512) + + assert [item["tile_index"] for item in selected] == [0, 2, 6, 8] + + +def test_image_has_low_visual_variance_rejects_blank_no_data(tmp_path: Path) -> None: + blank = tmp_path / "blank.png" + real = tmp_path / "real.png" + Image.new("L", (8, 8), color=255).save(blank) + image = Image.new("L", (8, 8), color=100) + image.putpixel((0, 0), 120) + image.save(real) + + assert image_has_low_visual_variance(blank, 3) is True + assert image_has_low_visual_variance(real, 3) is False diff --git a/tests/test_evaluate_yolo_checkpoint_matrix.py b/tests/test_evaluate_yolo_checkpoint_matrix.py index bad4a43e..8888632d 100644 --- a/tests/test_evaluate_yolo_checkpoint_matrix.py +++ b/tests/test_evaluate_yolo_checkpoint_matrix.py @@ -17,7 +17,8 @@ def test_dataset_overlap_evidence_marks_repeated_validation_rows( assert evidence["status"] == "overlapping" assert evidence["overlap_pixels"] == 256 - assert evidence["validation_rows_independent"] is False + assert evidence["validation_tiles_non_overlapping"] is False + assert evidence["statistical_independence_established"] is False def test_dataset_overlap_evidence_is_explicit_when_summary_missing( @@ -26,4 +27,21 @@ def test_dataset_overlap_evidence_is_explicit_when_summary_missing( evidence = dataset_overlap_evidence(tmp_path / "dataset.yaml") assert evidence["status"] == "unavailable" - assert evidence["validation_rows_independent"] is None + assert evidence["validation_tiles_non_overlapping"] is None + assert evidence["statistical_independence_established"] is False + + +def test_nonoverlap_does_not_overclaim_statistical_independence( + tmp_path: Path, +) -> None: + dataset_yaml = tmp_path / "dataset.yaml" + dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8") + (tmp_path / "yolo_tile_dataset_summary.json").write_text( + json.dumps({"tile_size": 512, "stride": 512}), encoding="utf-8" + ) + + evidence = dataset_overlap_evidence(dataset_yaml) + + assert evidence["validation_tiles_non_overlapping"] is True + assert evidence["statistical_independence_established"] is False + assert "not established" in evidence["interpretation"]