correct overlapping checkpoint evaluation
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Jens
2026-08-09 20:41:54 +02:00
parent 2de438b9cc
commit 48084799c9
11 changed files with 539 additions and 10 deletions
@@ -0,0 +1,72 @@
{
"schema_version": 1,
"run_id": "building-kempen-v69-nonoverlap-checkpoint-review-r2",
"generated_at": "2026-08-09T20:35:51+02:00",
"status": "completed_no_production_change",
"claim_boundary": "Non-protected two-positive-AOI and one-background-AOI validation refinement only; tiles do not overlap, but statistical independence, protected-test performance, national accuracy and promotion eligibility are not established.",
"correction_to_prior_evidence": {
"prior_validation_tile_count": 36,
"prior_tile_overlap_pixels": 256,
"prior_background_tile_count": 18,
"invalid_background_sample": "arendonk_heide",
"invalid_background_reason": "all nine source tiles are visually blank/low-variance no-data; four non-overlap representatives were excluded with explicit reason codes",
"effect": "The prior matrix remains a historical overlapping-tile ranking, but its 18-image background count must not be interpreted as 18 real background observations."
},
"evaluation_view": {
"path": "/app/storage/operator-data/yolo-building-aoi1024-bgaware512r8-nonoverlap-eval-r2",
"manifest_sha256": "69f555e57fa52d8b7cdfc5632167bde96b64030ee37e9bf49ff2d9891d66574a",
"summary_sha256": "c17a8c47eaecc2dfb38778ff5a612f9e6c4d67c9e90e31d3a6a66d8857adec4c",
"cross_tile_audit_sha256": "b0a49d7f5983ccaa07754a1e3d6d8f7b10345d3c57b9f613bb856de9d6f70b74",
"contact_sheet_sha256": "f5954a2f6ad3dc59642bf3a51d98ca94343491f522968ab38d60c999ed21c797",
"tile_size": 512,
"stride": 512,
"tile_count": 12,
"positive_tile_count": 8,
"real_background_tile_count": 4,
"positive_aoi_count": 2,
"background_aoi_count": 1,
"label_count": 2551,
"unique_interior_object_count": 2384,
"edge_label_count": 167,
"repeated_interior_object_count": 0,
"excluded_low_variance_tile_count": 4,
"excluded_sample_slug": "arendonk_heide",
"training_prohibited": true,
"statistical_independence_established": false
},
"runtime": {
"device": "cuda:0",
"gpu": "NVIDIA GeForce RTX 4080 SUPER",
"torch": "2.11.0+cu128",
"ultralytics": "8.4.99"
},
"checkpoint_metrics": {
"active": {
"model_sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1",
"precision": 0.5426324241859346,
"recall": 0.4353171786761234,
"map50": 0.342034346792601,
"map50_95": 0.1413160008545276,
"background_detections_confidence_0_15": 0,
"background_detections_confidence_0_25": 0
},
"challenger": {
"model_sha256": "038f1f97a6afd534f29e1f392a730a58207b928ca01e31ab8d8fed6106705820",
"precision": 0.5817150200182469,
"recall": 0.462299518515736,
"map50": 0.3683635371499511,
"map50_95": 0.15531818619443444,
"background_detections_confidence_0_15": 1,
"background_detections_confidence_0_25": 0
},
"confidence_0_15_matrix_sha256": "915c8523fe71a2d88b44bcb75d9070d25940f3969213692cc121f1239b31489b",
"confidence_0_25_matrix_sha256": "173d8af94d5d3d805b67a1a99d36e196255166e2e32ecabbef40ebf065c92138"
},
"decision": {
"relative_metric_ranking": "challenger",
"strict_background_ranking_at_0_15": "active",
"background_tie_at_0_25": true,
"production_model_changed": false,
"reason": "The challenger remains stronger on this refined non-overlapping ranking and clears Postel at 0.25, but one real background AOI and two adjacent positive AOIs are insufficient for release; historical provenance and human-review gates also remain closed."
}
}
@@ -1,4 +1,5 @@
from pathlib import Path from pathlib import Path
import shlex
import subprocess import subprocess
import sys import sys
@@ -8,7 +9,9 @@ ROOT = Path(__file__).resolve().parents[2]
def test_operator_yolo_dataset_export_script_contract() -> None: def test_operator_yolo_dataset_export_script_contract() -> None:
script_path = ROOT / "scripts" / "export_operator_yolo_dataset.py" script_path = ROOT / "scripts" / "export_operator_yolo_dataset.py"
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
encoding="utf-8"
)
assert script_path.exists() assert script_path.exists()
script = script_path.read_text(encoding="utf-8") script = script_path.read_text(encoding="utf-8")
@@ -42,14 +45,18 @@ def test_operator_yolo_dataset_export_help_does_not_require_gis_dependencies() -
) )
assert result.returncode == 0 assert result.returncode == 0
assert "Export operator real-data samples to a YOLO detection dataset" in result.stdout assert (
"Export operator real-data samples to a YOLO detection dataset" in result.stdout
)
assert "--manifest-path" in result.stdout assert "--manifest-path" in result.stdout
assert "--val-samples" in result.stdout assert "--val-samples" in result.stdout
def test_operator_yolo_train_smoke_script_contract() -> None: def test_operator_yolo_train_smoke_script_contract() -> None:
script_path = ROOT / "scripts" / "train_operator_yolo_detector.sh" script_path = ROOT / "scripts" / "train_operator_yolo_detector.sh"
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
encoding="utf-8"
)
assert script_path.exists() assert script_path.exists()
script = script_path.read_text(encoding="utf-8") script = script_path.read_text(encoding="utf-8")
@@ -62,7 +69,7 @@ def test_operator_yolo_train_smoke_script_contract() -> None:
assert "TRAIN_EPOCHS" in script assert "TRAIN_EPOCHS" in script
assert "TRAIN_IMGSZ" in script assert "TRAIN_IMGSZ" in script
assert "/opt/geointel/venv/bin/python" in script assert "/opt/geointel/venv/bin/python" in script
assert "PYTHON_BIN=\"python3\"" in script assert 'PYTHON_BIN="python3"' in script
assert "dataset.yaml" in script assert "dataset.yaml" in script
assert "from ultralytics import YOLO" in script assert "from ultralytics import YOLO" in script
assert "model.train" in script assert "model.train" in script
@@ -76,6 +83,42 @@ def test_operator_yolo_train_smoke_script_contract() -> None:
assert '"base_model_sha256"' in script assert '"base_model_sha256"' in script
assert '"trained_model_sha256"' in script assert '"trained_model_sha256"' in script
assert "training_release_manifest.py" in script assert "training_release_manifest.py" in script
assert "NO_TRAINING.json" in script
assert "Training is prohibited for this evaluation-only dataset" in script
assert "verify" in script assert "verify" in script
assert "download" not in script.lower() assert "download" not in script.lower()
assert "fixture_mode" not in script assert "fixture_mode" not in script
def test_operator_yolo_train_rejects_evaluation_only_marker(tmp_path: Path) -> None:
dataset_dir = tmp_path / "evaluation-only"
dataset_dir.mkdir()
(dataset_dir / "dataset.yaml").write_text("val: images/val\n", encoding="utf-8")
(dataset_dir / "NO_TRAINING.json").write_text(
'{"training_prohibited": true}\n', encoding="utf-8"
)
def shell_path(path: Path) -> str:
if path.drive:
return f"/mnt/{path.drive[0].lower()}/{path.as_posix()[3:]}"
return str(path)
dataset_shell_path = shell_path(dataset_dir)
model_shell_path = shell_path(tmp_path / "missing.pt")
script_shell_path = shell_path(ROOT / "scripts" / "train_operator_yolo_detector.sh")
command = (
f"OPERATOR_YOLO_DATASET_DIR={shlex.quote(dataset_shell_path)} "
f"YOLO_BASE_MODEL_PATH={shlex.quote(model_shell_path)} "
f"bash {shlex.quote(script_shell_path)}"
)
result = subprocess.run(
["bash", "-lc", command],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 1
assert "Training is prohibited for this evaluation-only dataset" in result.stderr
assert "Base model file not found" not in result.stderr
+33
View File
@@ -12671,3 +12671,36 @@ Open:
- The min-4px corpus is preferred for future experimental work, but remains - The min-4px corpus is preferred for future experimental work, but remains
explicitly training/release-ineligible until real human review and governed explicitly training/release-ineligible until real human review and governed
release evidence exist. The active GPU model was not changed. release evidence exist. The active GPU model was not changed.
## 2026-08-09 - Non-overlapping checkpoint correction
### Corrected
- Traced the prior 36-image checkpoint matrix to four AOIs tiled at 512/256.
The same interiors were repeated and nine Arendonk-heide tiles were visually
blank no-data, despite being counted as background observations.
- Added a deterministic non-overlap evaluation-view builder. It selects four
complete 512 px grid tiles per 1024 px AOI, excludes low-variance no-data
with reason codes, creates empty train directories and writes a hashed
`NO_TRAINING.json` marker.
- Added a fail-fast marker check to the operator YOLO training wrapper before
release verification, model loading or CUDA allocation.
- Corrected checkpoint overlap semantics: zero pixel overlap is reported, but
statistical independence is explicitly not established for adjacent tiles.
### GPU evidence
- The clean r2 view contains 12 images: eight positive Turnhout/Westerlo tiles
and four real Postel backgrounds. It has 2,551 labels, 2,384 unique interior
objects, 167 edge labels and zero repeated interior objects.
- Active: precision 0.542632, recall 0.435317, mAP50 0.342034, mAP50-95
0.141316. Challenger: 0.581715, 0.462300, 0.368364 and 0.155318.
- At confidence 0.15 the active model has zero Postel detections and the
challenger one; at 0.25 both have zero. The threshold-sensitive relative
challenger advantage is confirmed without changing production.
### Remaining limitations
- Two adjacent positive AOIs and one real background AOI are not representative
release evidence. Human review, broader independent geography and exact
challenger provenance remain blocking.
+7
View File
@@ -1170,6 +1170,13 @@ This file now starts with the current implementation status. Older preparation/b
- [ ] Compare the existing 0.35 edge-visibility policy against a separately - [ ] Compare the existing 0.35 edge-visibility policy against a separately
versioned stricter ablation from original source geometry before any new versioned stricter ablation from original source geometry before any new
governed training; do not approximate this by dropping every edge label. governed training; do not approximate this by dropping every edge label.
- [x] Replace the overlapping checkpoint ranking with a checksum-bound,
training-disabled non-overlap view: 12 real tiles, 2,384 unique interior
objects, zero repeated interiors and explicit removal of blank Arendonk
no-data imagery.
- [ ] Add multiple independent real-background AOIs and more geographically
separated positive validation AOIs before interpreting non-overlap ranking
as release evidence; Postel alone is not representative.
- [ ] Convert the AI-assisted ledger into no stronger claim than experimental - [ ] Convert the AI-assisted ledger into no stronger claim than experimental
triage; a real human must independently review and sign the frozen artifacts triage; a real human must independently review and sign the frozen artifacts
before the governed training wrapper may unlock. before the governed training wrapper may unlock.
@@ -187,3 +187,32 @@ visible; it was not suppressed or relabelled as success.
This min-4px version is the preferred experimental successor to the legacy This min-4px version is the preferred experimental successor to the legacy
min-3px corpus. It explicitly remains ineligible for governed training until min-3px corpus. It explicitly remains ineligible for governed training until
real human review and all release-contract evidence exist. real human review and all release-contract evidence exist.
## Non-overlapping checkpoint re-evaluation
The 36-tile checkpoint matrix was traced to a 512 px validation corpus with
stride 256. It contained nine views each of Turnhout, Westerlo, Postel-bos and
Arendonk-heide. Visual inspection then exposed that all Arendonk-heide views
are blank/no-data imagery, not meaningful pure-background observations. The
historical claim of 18 background images is therefore corrected: nine were
blank no-data and nine represented Postel under overlap.
A training-disabled validation view now covers each 1024 px AOI with four
non-overlapping 512 px tiles. Four blank Arendonk representatives are excluded
with explicit reason codes. The resulting set contains eight positive tiles
from Turnhout/Westerlo, four real Postel background tiles and 2,551 labels.
Exact reconstruction finds 2,384 unique interior objects, 167 edge labels and
zero repeated interior objects. The view has empty train directories, a
checksum-bound `NO_TRAINING.json`, and the training wrapper rejects that marker.
On the Tower RTX 4080 SUPER, the active checkpoint measures precision
`0.542632`, recall `0.435317`, mAP50 `0.342034` and mAP50-95 `0.141316`.
The reviewedexp6 challenger measures `0.581715`, `0.462300`, `0.368364` and
`0.155318` respectively. At background confidence 0.15 the active checkpoint
has zero Postel detections and the challenger has one. At 0.25 both have zero.
Thus the refined evidence confirms the challenger's relative metric advantage
and its threshold sensitivity, but still does not authorize promotion.
Non-overlap is not overstated as statistical independence: the positive tiles
remain adjacent and come from only two AOIs, edge objects can remain split, and
there is only one real background AOI. The production model remains unchanged.
+8
View File
@@ -35,6 +35,14 @@ excludes rather than relabels any positive tile that would become empty, and
writes complete source/output hashes plus a row-level removal manifest. It writes complete source/output hashes plus a row-level removal manifest. It
never creates a training-release manifest or human-review claim. never creates a training-release manifest or human-review claim.
`derive_yolo_nonoverlap_evaluation_view.py` selects the complete non-overlap
grid from an existing overlapping validation corpus, records every excluded
low-variance/no-data tile, leaves train directories empty and writes a
checksum-bound `NO_TRAINING.json`. The operator training wrapper rejects that
marker before release verification or CUDA allocation. Non-overlap proves only
that tiles do not share pixels; checkpoint reports explicitly avoid claiming
statistical independence for adjacent tiles from the same AOI.
Setup-, import-, demo- en maintenance-scripts voor GeoIntel. Setup-, import-, demo- en maintenance-scripts voor GeoIntel.
## WALOUS source provisioning ## WALOUS source provisioning
@@ -0,0 +1,259 @@
#!/usr/bin/env python3
"""Derive a checksum-bound, training-disabled non-overlapping YOLO val view."""
from __future__ import annotations
import argparse
import hashlib
import json
import shutil
from collections import defaultdict
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from PIL import Image
try:
from scripts.audit_yolo_cross_tile_repetition import tile_offsets
except ModuleNotFoundError: # Standalone operator-tool copy beside the auditor.
from audit_yolo_cross_tile_repetition import tile_offsets
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def select_nonoverlap_tiles(
tiles: list[dict[str, Any]], tile_size: int
) -> list[dict[str, Any]]:
selected: list[dict[str, Any]] = []
for tile in tiles:
if not isinstance(tile, dict) or not tile.get("kept", True):
continue
if str(tile.get("split") or "") != "val":
continue
row_offset, column_offset = tile_offsets(str(tile.get("image_path") or ""))
if row_offset % tile_size == 0 and column_offset % tile_size == 0:
selected.append(tile)
return sorted(
selected,
key=lambda tile: (
str(tile.get("sample_slug") or ""),
int(tile.get("tile_index") or 0),
),
)
def aggregate_hash(paths: list[Path], root: Path) -> str:
digest = hashlib.sha256()
for path in sorted(paths):
digest.update(path.relative_to(root).as_posix().encode("utf-8"))
digest.update(b"\0")
digest.update(sha256_file(path).encode("ascii"))
digest.update(b"\n")
return digest.hexdigest()
def image_has_low_visual_variance(image_path: Path, blank_range_threshold: int) -> bool:
image = Image.open(image_path).convert("L")
minimum, maximum = image.getextrema()
return maximum - minimum <= blank_range_threshold
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--source-dir", required=True, type=Path)
parser.add_argument("--output-dir", required=True, type=Path)
parser.add_argument(
"--exclude-low-variance",
action=argparse.BooleanOptionalAction,
default=True,
)
parser.add_argument("--blank-range-threshold", type=int, default=3)
args = parser.parse_args()
source_dir = args.source_dir.expanduser().resolve(strict=True)
output_dir = args.output_dir.expanduser().resolve(strict=False)
if output_dir.exists():
parser.error(f"output directory already exists: {output_dir}")
source_summary_path = source_dir / "yolo_tile_dataset_summary.json"
source_summary = json.loads(source_summary_path.read_text(encoding="utf-8"))
tiles = source_summary.get("tiles")
tile_size = source_summary.get("tile_size")
if not isinstance(tiles, list):
raise ValueError("source summary must contain a tiles list")
if not isinstance(tile_size, int) or tile_size <= 0:
raise ValueError("source summary must contain a positive integer tile_size")
selected_candidates = select_nonoverlap_tiles(tiles, tile_size)
if not selected_candidates:
raise ValueError("no non-overlapping validation tiles were selected")
selected: list[dict[str, Any]] = []
excluded_tiles: list[dict[str, Any]] = []
for tile in selected_candidates:
source_image = Path(str(tile.get("image_path") or "")).resolve(strict=True)
if args.exclude_low_variance and image_has_low_visual_variance(
source_image, args.blank_range_threshold
):
excluded_tiles.append(
{
"sample_slug": str(tile.get("sample_slug") or "unknown"),
"tile_index": int(tile.get("tile_index") or 0),
"source_image_path": str(source_image),
"reason": "low_visual_variance_no_data",
}
)
continue
selected.append(tile)
if not selected:
raise ValueError("every non-overlapping validation tile was excluded")
offsets_by_sample: dict[str, set[tuple[int, int]]] = defaultdict(set)
source_val_counts: dict[str, int] = defaultdict(int)
for tile in tiles:
if (
isinstance(tile, dict)
and tile.get("kept", True)
and tile.get("split") == "val"
):
source_val_counts[str(tile.get("sample_slug") or "unknown")] += 1
for tile in selected:
offsets_by_sample[str(tile.get("sample_slug") or "unknown")].add(
tile_offsets(str(tile.get("image_path") or ""))
)
accounted_samples = set(offsets_by_sample) | {
tile["sample_slug"] for tile in excluded_tiles
}
if accounted_samples != set(source_val_counts):
raise ValueError("non-overlap selection has an unaccounted validation sample")
output_dir.mkdir(parents=True)
(output_dir / "images" / "train").mkdir(parents=True)
(output_dir / "labels" / "train").mkdir(parents=True)
output_tiles: list[dict[str, Any]] = []
output_files: list[Path] = []
for tile in selected:
source_image = Path(str(tile.get("image_path") or "")).resolve(strict=True)
source_label = Path(str(tile.get("label_path") or "")).resolve(strict=True)
target_image = output_dir / "images" / "val" / source_image.name
target_label = output_dir / "labels" / "val" / source_label.name
target_image.parent.mkdir(parents=True, exist_ok=True)
target_label.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source_image, target_image)
shutil.copy2(source_label, target_label)
output_files.extend((target_image, target_label))
derived_tile = dict(tile)
derived_tile.update(
{
"image_path": str(target_image),
"label_path": str(target_label),
"derived_from_image_path": str(source_image),
"derived_from_label_path": str(source_label),
}
)
output_tiles.append(derived_tile)
dataset_yaml = output_dir / "dataset.yaml"
dataset_yaml.write_text(
"\n".join(
(
f"path: {output_dir}",
"train: images/train",
"val: images/val",
"names:",
" 0: building",
"",
)
),
encoding="utf-8",
)
no_training = {
"schema_version": 1,
"training_prohibited": True,
"reason": "Immutable non-overlapping evaluation view; train directories are intentionally empty.",
"source_summary_sha256": sha256_file(source_summary_path),
}
no_training_path = output_dir / "NO_TRAINING.json"
no_training_path.write_text(
json.dumps(no_training, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
summary = {
"schema_version": 1,
"status": "ready_evaluation_only",
"claim_boundary": (
"Non-protected, non-overlapping tile-level validation view; no "
"training, protected-test, object-level independence or promotion claim."
),
"dataset_yaml": str(dataset_yaml),
"output_dir": str(output_dir),
"source_dataset_dir": str(source_dir),
"source_summary_path": str(source_summary_path),
"source_summary_sha256": sha256_file(source_summary_path),
"class_names": source_summary.get("class_names", ["building"]),
"tile_size": tile_size,
"stride": tile_size,
"source_stride": source_summary.get("stride"),
"training_prohibited": True,
"training_marker": str(no_training_path),
"tile_count": len(output_tiles),
"val_tile_count": len(output_tiles),
"train_tile_count": 0,
"positive_tile_count": sum(
int(tile.get("label_count") or 0) > 0 for tile in output_tiles
),
"negative_tile_count": sum(
int(tile.get("label_count") or 0) == 0 for tile in output_tiles
),
"label_count": sum(int(tile.get("label_count") or 0) for tile in output_tiles),
"selected_offsets_by_sample": {
sample: [list(offset) for offset in sorted(offsets)]
for sample, offsets in sorted(offsets_by_sample.items())
},
"source_val_tile_counts_by_sample": dict(sorted(source_val_counts.items())),
"excluded_tile_count": len(excluded_tiles),
"excluded_tiles": excluded_tiles,
"excluded_sample_slugs": sorted(
{tile["sample_slug"] for tile in excluded_tiles}
),
"exclude_low_variance": args.exclude_low_variance,
"blank_range_threshold": args.blank_range_threshold,
"tiles": output_tiles,
}
summary_path = output_dir / "yolo_tile_dataset_summary.json"
summary_path.write_text(
json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
manifest = {
"schema_version": 1,
"generated_at": datetime.now(UTC).isoformat(),
"status": "complete_evaluation_only",
"source_summary_path": str(source_summary_path),
"source_summary_sha256": sha256_file(source_summary_path),
"output_summary_path": str(summary_path),
"output_summary_sha256": sha256_file(summary_path),
"dataset_yaml_sha256": sha256_file(dataset_yaml),
"evaluation_file_set_aggregate_sha256": aggregate_hash(
output_files, output_dir
),
"training_marker_sha256": sha256_file(no_training_path),
"training_prohibited": True,
"selected_tile_count": len(output_tiles),
"selected_sample_count": len(offsets_by_sample),
"excluded_tile_count": len(excluded_tiles),
"excluded_tiles": excluded_tiles,
}
manifest_path = output_dir / "evaluation_view_manifest.json"
manifest_path.write_text(
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
print(json.dumps(manifest, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+9 -4
View File
@@ -59,7 +59,8 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
return { return {
"status": "unavailable", "status": "unavailable",
"summary_path": str(summary_path), "summary_path": str(summary_path),
"validation_rows_independent": None, "validation_tiles_non_overlapping": None,
"statistical_independence_established": False,
} }
payload = json.loads(summary_path.read_text(encoding="utf-8")) payload = json.loads(summary_path.read_text(encoding="utf-8"))
tile_size = payload.get("tile_size") tile_size = payload.get("tile_size")
@@ -69,7 +70,8 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
"status": "invalid", "status": "invalid",
"summary_path": str(summary_path), "summary_path": str(summary_path),
"summary_sha256": sha256_file(summary_path), "summary_sha256": sha256_file(summary_path),
"validation_rows_independent": None, "validation_tiles_non_overlapping": None,
"statistical_independence_established": False,
} }
overlap_pixels = max(tile_size - stride, 0) overlap_pixels = max(tile_size - stride, 0)
return { return {
@@ -79,12 +81,15 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
"tile_size": tile_size, "tile_size": tile_size,
"stride": stride, "stride": stride,
"overlap_pixels": overlap_pixels, "overlap_pixels": overlap_pixels,
"validation_rows_independent": overlap_pixels == 0, "validation_tiles_non_overlapping": overlap_pixels == 0,
"statistical_independence_established": False,
"interpretation": ( "interpretation": (
"Tile metrics can repeat the same source object and are valid for " "Tile metrics can repeat the same source object and are valid for "
"candidate ranking only, not independent object-level uncertainty." "candidate ranking only, not independent object-level uncertainty."
if overlap_pixels if overlap_pixels
else "Tile rows do not overlap according to the dataset summary." else "Tiles do not overlap, but spatial/statistical independence is "
"not established because adjacent tiles share AOI context and edge "
"objects can remain split."
), ),
} }
+6
View File
@@ -56,6 +56,7 @@ fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml" DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
NO_TRAINING_MARKER="${OPERATOR_YOLO_DATASET_DIR%/}/NO_TRAINING.json"
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json" SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
export DATASET_YAML export DATASET_YAML
export YOLO_BASE_MODEL_PATH export YOLO_BASE_MODEL_PATH
@@ -76,6 +77,11 @@ if [[ ! -f "${DATASET_YAML}" ]]; then
exit 1 exit 1
fi fi
if [[ -f "${NO_TRAINING_MARKER}" ]]; then
echo "Training is prohibited for this evaluation-only dataset: ${NO_TRAINING_MARKER}" >&2
exit 1
fi
if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
echo "Base model file not found: ${YOLO_BASE_MODEL_PATH}" >&2 echo "Base model file not found: ${YOLO_BASE_MODEL_PATH}" >&2
exit 1 exit 1
@@ -0,0 +1,49 @@
from pathlib import Path
from PIL import Image
from scripts.derive_yolo_nonoverlap_evaluation_view import (
image_has_low_visual_variance,
select_nonoverlap_tiles,
)
def tile(name: str, index: int, split: str = "val") -> dict[str, object]:
return {
"image_path": f"/data/sample_{index:04d}_{name}.png",
"tile_index": index,
"sample_slug": "sample",
"split": split,
"kept": True,
}
def test_select_nonoverlap_tiles_keeps_complete_512_grid() -> None:
tiles = [
tile("r0_c0", 0),
tile("r0_c256", 1),
tile("r0_c512", 2),
tile("r256_c0", 3),
tile("r256_c256", 4),
tile("r256_c512", 5),
tile("r512_c0", 6),
tile("r512_c256", 7),
tile("r512_c512", 8),
tile("r0_c0", 9, split="train"),
]
selected = select_nonoverlap_tiles(tiles, 512)
assert [item["tile_index"] for item in selected] == [0, 2, 6, 8]
def test_image_has_low_visual_variance_rejects_blank_no_data(tmp_path: Path) -> None:
blank = tmp_path / "blank.png"
real = tmp_path / "real.png"
Image.new("L", (8, 8), color=255).save(blank)
image = Image.new("L", (8, 8), color=100)
image.putpixel((0, 0), 120)
image.save(real)
assert image_has_low_visual_variance(blank, 3) is True
assert image_has_low_visual_variance(real, 3) is False
+20 -2
View File
@@ -17,7 +17,8 @@ def test_dataset_overlap_evidence_marks_repeated_validation_rows(
assert evidence["status"] == "overlapping" assert evidence["status"] == "overlapping"
assert evidence["overlap_pixels"] == 256 assert evidence["overlap_pixels"] == 256
assert evidence["validation_rows_independent"] is False assert evidence["validation_tiles_non_overlapping"] is False
assert evidence["statistical_independence_established"] is False
def test_dataset_overlap_evidence_is_explicit_when_summary_missing( def test_dataset_overlap_evidence_is_explicit_when_summary_missing(
@@ -26,4 +27,21 @@ def test_dataset_overlap_evidence_is_explicit_when_summary_missing(
evidence = dataset_overlap_evidence(tmp_path / "dataset.yaml") evidence = dataset_overlap_evidence(tmp_path / "dataset.yaml")
assert evidence["status"] == "unavailable" assert evidence["status"] == "unavailable"
assert evidence["validation_rows_independent"] is None assert evidence["validation_tiles_non_overlapping"] is None
assert evidence["statistical_independence_established"] is False
def test_nonoverlap_does_not_overclaim_statistical_independence(
tmp_path: Path,
) -> None:
dataset_yaml = tmp_path / "dataset.yaml"
dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8")
(tmp_path / "yolo_tile_dataset_summary.json").write_text(
json.dumps({"tile_size": 512, "stride": 512}), encoding="utf-8"
)
evidence = dataset_overlap_evidence(dataset_yaml)
assert evidence["validation_tiles_non_overlapping"] is True
assert evidence["statistical_independence_established"] is False
assert "not established" in evidence["interpretation"]