correct overlapping checkpoint evaluation
This commit is contained in:
+72
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"run_id": "building-kempen-v69-nonoverlap-checkpoint-review-r2",
|
||||
"generated_at": "2026-08-09T20:35:51+02:00",
|
||||
"status": "completed_no_production_change",
|
||||
"claim_boundary": "Non-protected two-positive-AOI and one-background-AOI validation refinement only; tiles do not overlap, but statistical independence, protected-test performance, national accuracy and promotion eligibility are not established.",
|
||||
"correction_to_prior_evidence": {
|
||||
"prior_validation_tile_count": 36,
|
||||
"prior_tile_overlap_pixels": 256,
|
||||
"prior_background_tile_count": 18,
|
||||
"invalid_background_sample": "arendonk_heide",
|
||||
"invalid_background_reason": "all nine source tiles are visually blank/low-variance no-data; four non-overlap representatives were excluded with explicit reason codes",
|
||||
"effect": "The prior matrix remains a historical overlapping-tile ranking, but its 18-image background count must not be interpreted as 18 real background observations."
|
||||
},
|
||||
"evaluation_view": {
|
||||
"path": "/app/storage/operator-data/yolo-building-aoi1024-bgaware512r8-nonoverlap-eval-r2",
|
||||
"manifest_sha256": "69f555e57fa52d8b7cdfc5632167bde96b64030ee37e9bf49ff2d9891d66574a",
|
||||
"summary_sha256": "c17a8c47eaecc2dfb38778ff5a612f9e6c4d67c9e90e31d3a6a66d8857adec4c",
|
||||
"cross_tile_audit_sha256": "b0a49d7f5983ccaa07754a1e3d6d8f7b10345d3c57b9f613bb856de9d6f70b74",
|
||||
"contact_sheet_sha256": "f5954a2f6ad3dc59642bf3a51d98ca94343491f522968ab38d60c999ed21c797",
|
||||
"tile_size": 512,
|
||||
"stride": 512,
|
||||
"tile_count": 12,
|
||||
"positive_tile_count": 8,
|
||||
"real_background_tile_count": 4,
|
||||
"positive_aoi_count": 2,
|
||||
"background_aoi_count": 1,
|
||||
"label_count": 2551,
|
||||
"unique_interior_object_count": 2384,
|
||||
"edge_label_count": 167,
|
||||
"repeated_interior_object_count": 0,
|
||||
"excluded_low_variance_tile_count": 4,
|
||||
"excluded_sample_slug": "arendonk_heide",
|
||||
"training_prohibited": true,
|
||||
"statistical_independence_established": false
|
||||
},
|
||||
"runtime": {
|
||||
"device": "cuda:0",
|
||||
"gpu": "NVIDIA GeForce RTX 4080 SUPER",
|
||||
"torch": "2.11.0+cu128",
|
||||
"ultralytics": "8.4.99"
|
||||
},
|
||||
"checkpoint_metrics": {
|
||||
"active": {
|
||||
"model_sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1",
|
||||
"precision": 0.5426324241859346,
|
||||
"recall": 0.4353171786761234,
|
||||
"map50": 0.342034346792601,
|
||||
"map50_95": 0.1413160008545276,
|
||||
"background_detections_confidence_0_15": 0,
|
||||
"background_detections_confidence_0_25": 0
|
||||
},
|
||||
"challenger": {
|
||||
"model_sha256": "038f1f97a6afd534f29e1f392a730a58207b928ca01e31ab8d8fed6106705820",
|
||||
"precision": 0.5817150200182469,
|
||||
"recall": 0.462299518515736,
|
||||
"map50": 0.3683635371499511,
|
||||
"map50_95": 0.15531818619443444,
|
||||
"background_detections_confidence_0_15": 1,
|
||||
"background_detections_confidence_0_25": 0
|
||||
},
|
||||
"confidence_0_15_matrix_sha256": "915c8523fe71a2d88b44bcb75d9070d25940f3969213692cc121f1239b31489b",
|
||||
"confidence_0_25_matrix_sha256": "173d8af94d5d3d805b67a1a99d36e196255166e2e32ecabbef40ebf065c92138"
|
||||
},
|
||||
"decision": {
|
||||
"relative_metric_ranking": "challenger",
|
||||
"strict_background_ranking_at_0_15": "active",
|
||||
"background_tie_at_0_25": true,
|
||||
"production_model_changed": false,
|
||||
"reason": "The challenger remains stronger on this refined non-overlapping ranking and clears Postel at 0.25, but one real background AOI and two adjacent positive AOIs are insufficient for release; historical provenance and human-review gates also remain closed."
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
@@ -8,7 +9,9 @@ ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
def test_operator_yolo_dataset_export_script_contract() -> None:
|
||||
script_path = ROOT / "scripts" / "export_operator_yolo_dataset.py"
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert script_path.exists()
|
||||
script = script_path.read_text(encoding="utf-8")
|
||||
@@ -42,14 +45,18 @@ def test_operator_yolo_dataset_export_help_does_not_require_gis_dependencies() -
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "Export operator real-data samples to a YOLO detection dataset" in result.stdout
|
||||
assert (
|
||||
"Export operator real-data samples to a YOLO detection dataset" in result.stdout
|
||||
)
|
||||
assert "--manifest-path" in result.stdout
|
||||
assert "--val-samples" in result.stdout
|
||||
|
||||
|
||||
def test_operator_yolo_train_smoke_script_contract() -> None:
|
||||
script_path = ROOT / "scripts" / "train_operator_yolo_detector.sh"
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert script_path.exists()
|
||||
script = script_path.read_text(encoding="utf-8")
|
||||
@@ -62,7 +69,7 @@ def test_operator_yolo_train_smoke_script_contract() -> None:
|
||||
assert "TRAIN_EPOCHS" in script
|
||||
assert "TRAIN_IMGSZ" in script
|
||||
assert "/opt/geointel/venv/bin/python" in script
|
||||
assert "PYTHON_BIN=\"python3\"" in script
|
||||
assert 'PYTHON_BIN="python3"' in script
|
||||
assert "dataset.yaml" in script
|
||||
assert "from ultralytics import YOLO" in script
|
||||
assert "model.train" in script
|
||||
@@ -76,6 +83,42 @@ def test_operator_yolo_train_smoke_script_contract() -> None:
|
||||
assert '"base_model_sha256"' in script
|
||||
assert '"trained_model_sha256"' in script
|
||||
assert "training_release_manifest.py" in script
|
||||
assert "NO_TRAINING.json" in script
|
||||
assert "Training is prohibited for this evaluation-only dataset" in script
|
||||
assert "verify" in script
|
||||
assert "download" not in script.lower()
|
||||
assert "fixture_mode" not in script
|
||||
|
||||
|
||||
def test_operator_yolo_train_rejects_evaluation_only_marker(tmp_path: Path) -> None:
|
||||
dataset_dir = tmp_path / "evaluation-only"
|
||||
dataset_dir.mkdir()
|
||||
(dataset_dir / "dataset.yaml").write_text("val: images/val\n", encoding="utf-8")
|
||||
(dataset_dir / "NO_TRAINING.json").write_text(
|
||||
'{"training_prohibited": true}\n', encoding="utf-8"
|
||||
)
|
||||
|
||||
def shell_path(path: Path) -> str:
|
||||
if path.drive:
|
||||
return f"/mnt/{path.drive[0].lower()}/{path.as_posix()[3:]}"
|
||||
return str(path)
|
||||
|
||||
dataset_shell_path = shell_path(dataset_dir)
|
||||
model_shell_path = shell_path(tmp_path / "missing.pt")
|
||||
script_shell_path = shell_path(ROOT / "scripts" / "train_operator_yolo_detector.sh")
|
||||
command = (
|
||||
f"OPERATOR_YOLO_DATASET_DIR={shlex.quote(dataset_shell_path)} "
|
||||
f"YOLO_BASE_MODEL_PATH={shlex.quote(model_shell_path)} "
|
||||
f"bash {shlex.quote(script_shell_path)}"
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", "-lc", command],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "Training is prohibited for this evaluation-only dataset" in result.stderr
|
||||
assert "Base model file not found" not in result.stderr
|
||||
|
||||
@@ -12671,3 +12671,36 @@ Open:
|
||||
- The min-4px corpus is preferred for future experimental work, but remains
|
||||
explicitly training/release-ineligible until real human review and governed
|
||||
release evidence exist. The active GPU model was not changed.
|
||||
|
||||
## 2026-08-09 - Non-overlapping checkpoint correction
|
||||
|
||||
### Corrected
|
||||
|
||||
- Traced the prior 36-image checkpoint matrix to four AOIs tiled at 512/256.
|
||||
The same interiors were repeated and nine Arendonk-heide tiles were visually
|
||||
blank no-data, despite being counted as background observations.
|
||||
- Added a deterministic non-overlap evaluation-view builder. It selects four
|
||||
complete 512 px grid tiles per 1024 px AOI, excludes low-variance no-data
|
||||
with reason codes, creates empty train directories and writes a hashed
|
||||
`NO_TRAINING.json` marker.
|
||||
- Added a fail-fast marker check to the operator YOLO training wrapper before
|
||||
release verification, model loading or CUDA allocation.
|
||||
- Corrected checkpoint overlap semantics: zero pixel overlap is reported, but
|
||||
statistical independence is explicitly not established for adjacent tiles.
|
||||
|
||||
### GPU evidence
|
||||
|
||||
- The clean r2 view contains 12 images: eight positive Turnhout/Westerlo tiles
|
||||
and four real Postel backgrounds. It has 2,551 labels, 2,384 unique interior
|
||||
objects, 167 edge labels and zero repeated interior objects.
|
||||
- Active: precision 0.542632, recall 0.435317, mAP50 0.342034, mAP50-95
|
||||
0.141316. Challenger: 0.581715, 0.462300, 0.368364 and 0.155318.
|
||||
- At confidence 0.15 the active model has zero Postel detections and the
|
||||
challenger one; at 0.25 both have zero. The threshold-sensitive relative
|
||||
challenger advantage is confirmed without changing production.
|
||||
|
||||
### Remaining limitations
|
||||
|
||||
- Two adjacent positive AOIs and one real background AOI are not representative
|
||||
release evidence. Human review, broader independent geography and exact
|
||||
challenger provenance remain blocking.
|
||||
|
||||
@@ -1170,6 +1170,13 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
- [ ] Compare the existing 0.35 edge-visibility policy against a separately
|
||||
versioned stricter ablation from original source geometry before any new
|
||||
governed training; do not approximate this by dropping every edge label.
|
||||
- [x] Replace the overlapping checkpoint ranking with a checksum-bound,
|
||||
training-disabled non-overlap view: 12 real tiles, 2,384 unique interior
|
||||
objects, zero repeated interiors and explicit removal of blank Arendonk
|
||||
no-data imagery.
|
||||
- [ ] Add multiple independent real-background AOIs and more geographically
|
||||
separated positive validation AOIs before interpreting non-overlap ranking
|
||||
as release evidence; Postel alone is not representative.
|
||||
- [ ] Convert the AI-assisted ledger into no stronger claim than experimental
|
||||
triage; a real human must independently review and sign the frozen artifacts
|
||||
before the governed training wrapper may unlock.
|
||||
|
||||
@@ -187,3 +187,32 @@ visible; it was not suppressed or relabelled as success.
|
||||
This min-4px version is the preferred experimental successor to the legacy
|
||||
min-3px corpus. It explicitly remains ineligible for governed training until
|
||||
real human review and all release-contract evidence exist.
|
||||
|
||||
## Non-overlapping checkpoint re-evaluation
|
||||
|
||||
The 36-tile checkpoint matrix was traced to a 512 px validation corpus with
|
||||
stride 256. It contained nine views each of Turnhout, Westerlo, Postel-bos and
|
||||
Arendonk-heide. Visual inspection then exposed that all Arendonk-heide views
|
||||
are blank/no-data imagery, not meaningful pure-background observations. The
|
||||
historical claim of 18 background images is therefore corrected: nine were
|
||||
blank no-data and nine represented Postel under overlap.
|
||||
|
||||
A training-disabled validation view now covers each 1024 px AOI with four
|
||||
non-overlapping 512 px tiles. Four blank Arendonk representatives are excluded
|
||||
with explicit reason codes. The resulting set contains eight positive tiles
|
||||
from Turnhout/Westerlo, four real Postel background tiles and 2,551 labels.
|
||||
Exact reconstruction finds 2,384 unique interior objects, 167 edge labels and
|
||||
zero repeated interior objects. The view has empty train directories, a
|
||||
checksum-bound `NO_TRAINING.json`, and the training wrapper rejects that marker.
|
||||
|
||||
On the Tower RTX 4080 SUPER, the active checkpoint measures precision
|
||||
`0.542632`, recall `0.435317`, mAP50 `0.342034` and mAP50-95 `0.141316`.
|
||||
The reviewedexp6 challenger measures `0.581715`, `0.462300`, `0.368364` and
|
||||
`0.155318` respectively. At background confidence 0.15 the active checkpoint
|
||||
has zero Postel detections and the challenger has one. At 0.25 both have zero.
|
||||
Thus the refined evidence confirms the challenger's relative metric advantage
|
||||
and its threshold sensitivity, but still does not authorize promotion.
|
||||
|
||||
Non-overlap is not overstated as statistical independence: the positive tiles
|
||||
remain adjacent and come from only two AOIs, edge objects can remain split, and
|
||||
there is only one real background AOI. The production model remains unchanged.
|
||||
|
||||
@@ -35,6 +35,14 @@ excludes rather than relabels any positive tile that would become empty, and
|
||||
writes complete source/output hashes plus a row-level removal manifest. It
|
||||
never creates a training-release manifest or human-review claim.
|
||||
|
||||
`derive_yolo_nonoverlap_evaluation_view.py` selects the complete non-overlap
|
||||
grid from an existing overlapping validation corpus, records every excluded
|
||||
low-variance/no-data tile, leaves train directories empty and writes a
|
||||
checksum-bound `NO_TRAINING.json`. The operator training wrapper rejects that
|
||||
marker before release verification or CUDA allocation. Non-overlap proves only
|
||||
that tiles do not share pixels; checkpoint reports explicitly avoid claiming
|
||||
statistical independence for adjacent tiles from the same AOI.
|
||||
|
||||
Setup-, import-, demo- en maintenance-scripts voor GeoIntel.
|
||||
|
||||
## WALOUS source provisioning
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Derive a checksum-bound, training-disabled non-overlapping YOLO val view."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
from collections import defaultdict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image
|
||||
|
||||
try:
|
||||
from scripts.audit_yolo_cross_tile_repetition import tile_offsets
|
||||
except ModuleNotFoundError: # Standalone operator-tool copy beside the auditor.
|
||||
from audit_yolo_cross_tile_repetition import tile_offsets
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def select_nonoverlap_tiles(
|
||||
tiles: list[dict[str, Any]], tile_size: int
|
||||
) -> list[dict[str, Any]]:
|
||||
selected: list[dict[str, Any]] = []
|
||||
for tile in tiles:
|
||||
if not isinstance(tile, dict) or not tile.get("kept", True):
|
||||
continue
|
||||
if str(tile.get("split") or "") != "val":
|
||||
continue
|
||||
row_offset, column_offset = tile_offsets(str(tile.get("image_path") or ""))
|
||||
if row_offset % tile_size == 0 and column_offset % tile_size == 0:
|
||||
selected.append(tile)
|
||||
return sorted(
|
||||
selected,
|
||||
key=lambda tile: (
|
||||
str(tile.get("sample_slug") or ""),
|
||||
int(tile.get("tile_index") or 0),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def aggregate_hash(paths: list[Path], root: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
for path in sorted(paths):
|
||||
digest.update(path.relative_to(root).as_posix().encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(sha256_file(path).encode("ascii"))
|
||||
digest.update(b"\n")
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def image_has_low_visual_variance(image_path: Path, blank_range_threshold: int) -> bool:
|
||||
image = Image.open(image_path).convert("L")
|
||||
minimum, maximum = image.getextrema()
|
||||
return maximum - minimum <= blank_range_threshold
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-dir", required=True, type=Path)
|
||||
parser.add_argument("--output-dir", required=True, type=Path)
|
||||
parser.add_argument(
|
||||
"--exclude-low-variance",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=True,
|
||||
)
|
||||
parser.add_argument("--blank-range-threshold", type=int, default=3)
|
||||
args = parser.parse_args()
|
||||
|
||||
source_dir = args.source_dir.expanduser().resolve(strict=True)
|
||||
output_dir = args.output_dir.expanduser().resolve(strict=False)
|
||||
if output_dir.exists():
|
||||
parser.error(f"output directory already exists: {output_dir}")
|
||||
source_summary_path = source_dir / "yolo_tile_dataset_summary.json"
|
||||
source_summary = json.loads(source_summary_path.read_text(encoding="utf-8"))
|
||||
tiles = source_summary.get("tiles")
|
||||
tile_size = source_summary.get("tile_size")
|
||||
if not isinstance(tiles, list):
|
||||
raise ValueError("source summary must contain a tiles list")
|
||||
if not isinstance(tile_size, int) or tile_size <= 0:
|
||||
raise ValueError("source summary must contain a positive integer tile_size")
|
||||
selected_candidates = select_nonoverlap_tiles(tiles, tile_size)
|
||||
if not selected_candidates:
|
||||
raise ValueError("no non-overlapping validation tiles were selected")
|
||||
|
||||
selected: list[dict[str, Any]] = []
|
||||
excluded_tiles: list[dict[str, Any]] = []
|
||||
for tile in selected_candidates:
|
||||
source_image = Path(str(tile.get("image_path") or "")).resolve(strict=True)
|
||||
if args.exclude_low_variance and image_has_low_visual_variance(
|
||||
source_image, args.blank_range_threshold
|
||||
):
|
||||
excluded_tiles.append(
|
||||
{
|
||||
"sample_slug": str(tile.get("sample_slug") or "unknown"),
|
||||
"tile_index": int(tile.get("tile_index") or 0),
|
||||
"source_image_path": str(source_image),
|
||||
"reason": "low_visual_variance_no_data",
|
||||
}
|
||||
)
|
||||
continue
|
||||
selected.append(tile)
|
||||
if not selected:
|
||||
raise ValueError("every non-overlapping validation tile was excluded")
|
||||
|
||||
offsets_by_sample: dict[str, set[tuple[int, int]]] = defaultdict(set)
|
||||
source_val_counts: dict[str, int] = defaultdict(int)
|
||||
for tile in tiles:
|
||||
if (
|
||||
isinstance(tile, dict)
|
||||
and tile.get("kept", True)
|
||||
and tile.get("split") == "val"
|
||||
):
|
||||
source_val_counts[str(tile.get("sample_slug") or "unknown")] += 1
|
||||
for tile in selected:
|
||||
offsets_by_sample[str(tile.get("sample_slug") or "unknown")].add(
|
||||
tile_offsets(str(tile.get("image_path") or ""))
|
||||
)
|
||||
accounted_samples = set(offsets_by_sample) | {
|
||||
tile["sample_slug"] for tile in excluded_tiles
|
||||
}
|
||||
if accounted_samples != set(source_val_counts):
|
||||
raise ValueError("non-overlap selection has an unaccounted validation sample")
|
||||
|
||||
output_dir.mkdir(parents=True)
|
||||
(output_dir / "images" / "train").mkdir(parents=True)
|
||||
(output_dir / "labels" / "train").mkdir(parents=True)
|
||||
output_tiles: list[dict[str, Any]] = []
|
||||
output_files: list[Path] = []
|
||||
for tile in selected:
|
||||
source_image = Path(str(tile.get("image_path") or "")).resolve(strict=True)
|
||||
source_label = Path(str(tile.get("label_path") or "")).resolve(strict=True)
|
||||
target_image = output_dir / "images" / "val" / source_image.name
|
||||
target_label = output_dir / "labels" / "val" / source_label.name
|
||||
target_image.parent.mkdir(parents=True, exist_ok=True)
|
||||
target_label.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source_image, target_image)
|
||||
shutil.copy2(source_label, target_label)
|
||||
output_files.extend((target_image, target_label))
|
||||
derived_tile = dict(tile)
|
||||
derived_tile.update(
|
||||
{
|
||||
"image_path": str(target_image),
|
||||
"label_path": str(target_label),
|
||||
"derived_from_image_path": str(source_image),
|
||||
"derived_from_label_path": str(source_label),
|
||||
}
|
||||
)
|
||||
output_tiles.append(derived_tile)
|
||||
|
||||
dataset_yaml = output_dir / "dataset.yaml"
|
||||
dataset_yaml.write_text(
|
||||
"\n".join(
|
||||
(
|
||||
f"path: {output_dir}",
|
||||
"train: images/train",
|
||||
"val: images/val",
|
||||
"names:",
|
||||
" 0: building",
|
||||
"",
|
||||
)
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
no_training = {
|
||||
"schema_version": 1,
|
||||
"training_prohibited": True,
|
||||
"reason": "Immutable non-overlapping evaluation view; train directories are intentionally empty.",
|
||||
"source_summary_sha256": sha256_file(source_summary_path),
|
||||
}
|
||||
no_training_path = output_dir / "NO_TRAINING.json"
|
||||
no_training_path.write_text(
|
||||
json.dumps(no_training, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
summary = {
|
||||
"schema_version": 1,
|
||||
"status": "ready_evaluation_only",
|
||||
"claim_boundary": (
|
||||
"Non-protected, non-overlapping tile-level validation view; no "
|
||||
"training, protected-test, object-level independence or promotion claim."
|
||||
),
|
||||
"dataset_yaml": str(dataset_yaml),
|
||||
"output_dir": str(output_dir),
|
||||
"source_dataset_dir": str(source_dir),
|
||||
"source_summary_path": str(source_summary_path),
|
||||
"source_summary_sha256": sha256_file(source_summary_path),
|
||||
"class_names": source_summary.get("class_names", ["building"]),
|
||||
"tile_size": tile_size,
|
||||
"stride": tile_size,
|
||||
"source_stride": source_summary.get("stride"),
|
||||
"training_prohibited": True,
|
||||
"training_marker": str(no_training_path),
|
||||
"tile_count": len(output_tiles),
|
||||
"val_tile_count": len(output_tiles),
|
||||
"train_tile_count": 0,
|
||||
"positive_tile_count": sum(
|
||||
int(tile.get("label_count") or 0) > 0 for tile in output_tiles
|
||||
),
|
||||
"negative_tile_count": sum(
|
||||
int(tile.get("label_count") or 0) == 0 for tile in output_tiles
|
||||
),
|
||||
"label_count": sum(int(tile.get("label_count") or 0) for tile in output_tiles),
|
||||
"selected_offsets_by_sample": {
|
||||
sample: [list(offset) for offset in sorted(offsets)]
|
||||
for sample, offsets in sorted(offsets_by_sample.items())
|
||||
},
|
||||
"source_val_tile_counts_by_sample": dict(sorted(source_val_counts.items())),
|
||||
"excluded_tile_count": len(excluded_tiles),
|
||||
"excluded_tiles": excluded_tiles,
|
||||
"excluded_sample_slugs": sorted(
|
||||
{tile["sample_slug"] for tile in excluded_tiles}
|
||||
),
|
||||
"exclude_low_variance": args.exclude_low_variance,
|
||||
"blank_range_threshold": args.blank_range_threshold,
|
||||
"tiles": output_tiles,
|
||||
}
|
||||
summary_path = output_dir / "yolo_tile_dataset_summary.json"
|
||||
summary_path.write_text(
|
||||
json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(UTC).isoformat(),
|
||||
"status": "complete_evaluation_only",
|
||||
"source_summary_path": str(source_summary_path),
|
||||
"source_summary_sha256": sha256_file(source_summary_path),
|
||||
"output_summary_path": str(summary_path),
|
||||
"output_summary_sha256": sha256_file(summary_path),
|
||||
"dataset_yaml_sha256": sha256_file(dataset_yaml),
|
||||
"evaluation_file_set_aggregate_sha256": aggregate_hash(
|
||||
output_files, output_dir
|
||||
),
|
||||
"training_marker_sha256": sha256_file(no_training_path),
|
||||
"training_prohibited": True,
|
||||
"selected_tile_count": len(output_tiles),
|
||||
"selected_sample_count": len(offsets_by_sample),
|
||||
"excluded_tile_count": len(excluded_tiles),
|
||||
"excluded_tiles": excluded_tiles,
|
||||
}
|
||||
manifest_path = output_dir / "evaluation_view_manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
print(json.dumps(manifest, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -59,7 +59,8 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
|
||||
return {
|
||||
"status": "unavailable",
|
||||
"summary_path": str(summary_path),
|
||||
"validation_rows_independent": None,
|
||||
"validation_tiles_non_overlapping": None,
|
||||
"statistical_independence_established": False,
|
||||
}
|
||||
payload = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||
tile_size = payload.get("tile_size")
|
||||
@@ -69,7 +70,8 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
|
||||
"status": "invalid",
|
||||
"summary_path": str(summary_path),
|
||||
"summary_sha256": sha256_file(summary_path),
|
||||
"validation_rows_independent": None,
|
||||
"validation_tiles_non_overlapping": None,
|
||||
"statistical_independence_established": False,
|
||||
}
|
||||
overlap_pixels = max(tile_size - stride, 0)
|
||||
return {
|
||||
@@ -79,12 +81,15 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
|
||||
"tile_size": tile_size,
|
||||
"stride": stride,
|
||||
"overlap_pixels": overlap_pixels,
|
||||
"validation_rows_independent": overlap_pixels == 0,
|
||||
"validation_tiles_non_overlapping": overlap_pixels == 0,
|
||||
"statistical_independence_established": False,
|
||||
"interpretation": (
|
||||
"Tile metrics can repeat the same source object and are valid for "
|
||||
"candidate ranking only, not independent object-level uncertainty."
|
||||
if overlap_pixels
|
||||
else "Tile rows do not overlap according to the dataset summary."
|
||||
else "Tiles do not overlap, but spatial/statistical independence is "
|
||||
"not established because adjacent tiles share AOI context and edge "
|
||||
"objects can remain split."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -56,6 +56,7 @@ fi
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
|
||||
NO_TRAINING_MARKER="${OPERATOR_YOLO_DATASET_DIR%/}/NO_TRAINING.json"
|
||||
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
|
||||
export DATASET_YAML
|
||||
export YOLO_BASE_MODEL_PATH
|
||||
@@ -76,6 +77,11 @@ if [[ ! -f "${DATASET_YAML}" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -f "${NO_TRAINING_MARKER}" ]]; then
|
||||
echo "Training is prohibited for this evaluation-only dataset: ${NO_TRAINING_MARKER}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
|
||||
echo "Base model file not found: ${YOLO_BASE_MODEL_PATH}" >&2
|
||||
exit 1
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from scripts.derive_yolo_nonoverlap_evaluation_view import (
|
||||
image_has_low_visual_variance,
|
||||
select_nonoverlap_tiles,
|
||||
)
|
||||
|
||||
|
||||
def tile(name: str, index: int, split: str = "val") -> dict[str, object]:
|
||||
return {
|
||||
"image_path": f"/data/sample_{index:04d}_{name}.png",
|
||||
"tile_index": index,
|
||||
"sample_slug": "sample",
|
||||
"split": split,
|
||||
"kept": True,
|
||||
}
|
||||
|
||||
|
||||
def test_select_nonoverlap_tiles_keeps_complete_512_grid() -> None:
|
||||
tiles = [
|
||||
tile("r0_c0", 0),
|
||||
tile("r0_c256", 1),
|
||||
tile("r0_c512", 2),
|
||||
tile("r256_c0", 3),
|
||||
tile("r256_c256", 4),
|
||||
tile("r256_c512", 5),
|
||||
tile("r512_c0", 6),
|
||||
tile("r512_c256", 7),
|
||||
tile("r512_c512", 8),
|
||||
tile("r0_c0", 9, split="train"),
|
||||
]
|
||||
|
||||
selected = select_nonoverlap_tiles(tiles, 512)
|
||||
|
||||
assert [item["tile_index"] for item in selected] == [0, 2, 6, 8]
|
||||
|
||||
|
||||
def test_image_has_low_visual_variance_rejects_blank_no_data(tmp_path: Path) -> None:
|
||||
blank = tmp_path / "blank.png"
|
||||
real = tmp_path / "real.png"
|
||||
Image.new("L", (8, 8), color=255).save(blank)
|
||||
image = Image.new("L", (8, 8), color=100)
|
||||
image.putpixel((0, 0), 120)
|
||||
image.save(real)
|
||||
|
||||
assert image_has_low_visual_variance(blank, 3) is True
|
||||
assert image_has_low_visual_variance(real, 3) is False
|
||||
@@ -17,7 +17,8 @@ def test_dataset_overlap_evidence_marks_repeated_validation_rows(
|
||||
|
||||
assert evidence["status"] == "overlapping"
|
||||
assert evidence["overlap_pixels"] == 256
|
||||
assert evidence["validation_rows_independent"] is False
|
||||
assert evidence["validation_tiles_non_overlapping"] is False
|
||||
assert evidence["statistical_independence_established"] is False
|
||||
|
||||
|
||||
def test_dataset_overlap_evidence_is_explicit_when_summary_missing(
|
||||
@@ -26,4 +27,21 @@ def test_dataset_overlap_evidence_is_explicit_when_summary_missing(
|
||||
evidence = dataset_overlap_evidence(tmp_path / "dataset.yaml")
|
||||
|
||||
assert evidence["status"] == "unavailable"
|
||||
assert evidence["validation_rows_independent"] is None
|
||||
assert evidence["validation_tiles_non_overlapping"] is None
|
||||
assert evidence["statistical_independence_established"] is False
|
||||
|
||||
|
||||
def test_nonoverlap_does_not_overclaim_statistical_independence(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
dataset_yaml = tmp_path / "dataset.yaml"
|
||||
dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8")
|
||||
(tmp_path / "yolo_tile_dataset_summary.json").write_text(
|
||||
json.dumps({"tile_size": 512, "stride": 512}), encoding="utf-8"
|
||||
)
|
||||
|
||||
evidence = dataset_overlap_evidence(dataset_yaml)
|
||||
|
||||
assert evidence["validation_tiles_non_overlapping"] is True
|
||||
assert evidence["statistical_independence_established"] is False
|
||||
assert "not established" in evidence["interpretation"]
|
||||
|
||||
Reference in New Issue
Block a user