correct overlapping checkpoint evaluation
This commit is contained in:
+72
@@ -0,0 +1,72 @@
|
|||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"run_id": "building-kempen-v69-nonoverlap-checkpoint-review-r2",
|
||||||
|
"generated_at": "2026-08-09T20:35:51+02:00",
|
||||||
|
"status": "completed_no_production_change",
|
||||||
|
"claim_boundary": "Non-protected two-positive-AOI and one-background-AOI validation refinement only; tiles do not overlap, but statistical independence, protected-test performance, national accuracy and promotion eligibility are not established.",
|
||||||
|
"correction_to_prior_evidence": {
|
||||||
|
"prior_validation_tile_count": 36,
|
||||||
|
"prior_tile_overlap_pixels": 256,
|
||||||
|
"prior_background_tile_count": 18,
|
||||||
|
"invalid_background_sample": "arendonk_heide",
|
||||||
|
"invalid_background_reason": "all nine source tiles are visually blank/low-variance no-data; four non-overlap representatives were excluded with explicit reason codes",
|
||||||
|
"effect": "The prior matrix remains a historical overlapping-tile ranking, but its 18-image background count must not be interpreted as 18 real background observations."
|
||||||
|
},
|
||||||
|
"evaluation_view": {
|
||||||
|
"path": "/app/storage/operator-data/yolo-building-aoi1024-bgaware512r8-nonoverlap-eval-r2",
|
||||||
|
"manifest_sha256": "69f555e57fa52d8b7cdfc5632167bde96b64030ee37e9bf49ff2d9891d66574a",
|
||||||
|
"summary_sha256": "c17a8c47eaecc2dfb38778ff5a612f9e6c4d67c9e90e31d3a6a66d8857adec4c",
|
||||||
|
"cross_tile_audit_sha256": "b0a49d7f5983ccaa07754a1e3d6d8f7b10345d3c57b9f613bb856de9d6f70b74",
|
||||||
|
"contact_sheet_sha256": "f5954a2f6ad3dc59642bf3a51d98ca94343491f522968ab38d60c999ed21c797",
|
||||||
|
"tile_size": 512,
|
||||||
|
"stride": 512,
|
||||||
|
"tile_count": 12,
|
||||||
|
"positive_tile_count": 8,
|
||||||
|
"real_background_tile_count": 4,
|
||||||
|
"positive_aoi_count": 2,
|
||||||
|
"background_aoi_count": 1,
|
||||||
|
"label_count": 2551,
|
||||||
|
"unique_interior_object_count": 2384,
|
||||||
|
"edge_label_count": 167,
|
||||||
|
"repeated_interior_object_count": 0,
|
||||||
|
"excluded_low_variance_tile_count": 4,
|
||||||
|
"excluded_sample_slug": "arendonk_heide",
|
||||||
|
"training_prohibited": true,
|
||||||
|
"statistical_independence_established": false
|
||||||
|
},
|
||||||
|
"runtime": {
|
||||||
|
"device": "cuda:0",
|
||||||
|
"gpu": "NVIDIA GeForce RTX 4080 SUPER",
|
||||||
|
"torch": "2.11.0+cu128",
|
||||||
|
"ultralytics": "8.4.99"
|
||||||
|
},
|
||||||
|
"checkpoint_metrics": {
|
||||||
|
"active": {
|
||||||
|
"model_sha256": "a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1",
|
||||||
|
"precision": 0.5426324241859346,
|
||||||
|
"recall": 0.4353171786761234,
|
||||||
|
"map50": 0.342034346792601,
|
||||||
|
"map50_95": 0.1413160008545276,
|
||||||
|
"background_detections_confidence_0_15": 0,
|
||||||
|
"background_detections_confidence_0_25": 0
|
||||||
|
},
|
||||||
|
"challenger": {
|
||||||
|
"model_sha256": "038f1f97a6afd534f29e1f392a730a58207b928ca01e31ab8d8fed6106705820",
|
||||||
|
"precision": 0.5817150200182469,
|
||||||
|
"recall": 0.462299518515736,
|
||||||
|
"map50": 0.3683635371499511,
|
||||||
|
"map50_95": 0.15531818619443444,
|
||||||
|
"background_detections_confidence_0_15": 1,
|
||||||
|
"background_detections_confidence_0_25": 0
|
||||||
|
},
|
||||||
|
"confidence_0_15_matrix_sha256": "915c8523fe71a2d88b44bcb75d9070d25940f3969213692cc121f1239b31489b",
|
||||||
|
"confidence_0_25_matrix_sha256": "173d8af94d5d3d805b67a1a99d36e196255166e2e32ecabbef40ebf065c92138"
|
||||||
|
},
|
||||||
|
"decision": {
|
||||||
|
"relative_metric_ranking": "challenger",
|
||||||
|
"strict_background_ranking_at_0_15": "active",
|
||||||
|
"background_tie_at_0_25": true,
|
||||||
|
"production_model_changed": false,
|
||||||
|
"reason": "The challenger remains stronger on this refined non-overlapping ranking and clears Postel at 0.25, but one real background AOI and two adjacent positive AOIs are insufficient for release; historical provenance and human-review gates also remain closed."
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
import shlex
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
@@ -8,7 +9,9 @@ ROOT = Path(__file__).resolve().parents[2]
|
|||||||
|
|
||||||
def test_operator_yolo_dataset_export_script_contract() -> None:
|
def test_operator_yolo_dataset_export_script_contract() -> None:
|
||||||
script_path = ROOT / "scripts" / "export_operator_yolo_dataset.py"
|
script_path = ROOT / "scripts" / "export_operator_yolo_dataset.py"
|
||||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
assert script_path.exists()
|
assert script_path.exists()
|
||||||
script = script_path.read_text(encoding="utf-8")
|
script = script_path.read_text(encoding="utf-8")
|
||||||
@@ -42,14 +45,18 @@ def test_operator_yolo_dataset_export_help_does_not_require_gis_dependencies() -
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert result.returncode == 0
|
assert result.returncode == 0
|
||||||
assert "Export operator real-data samples to a YOLO detection dataset" in result.stdout
|
assert (
|
||||||
|
"Export operator real-data samples to a YOLO detection dataset" in result.stdout
|
||||||
|
)
|
||||||
assert "--manifest-path" in result.stdout
|
assert "--manifest-path" in result.stdout
|
||||||
assert "--val-samples" in result.stdout
|
assert "--val-samples" in result.stdout
|
||||||
|
|
||||||
|
|
||||||
def test_operator_yolo_train_smoke_script_contract() -> None:
|
def test_operator_yolo_train_smoke_script_contract() -> None:
|
||||||
script_path = ROOT / "scripts" / "train_operator_yolo_detector.sh"
|
script_path = ROOT / "scripts" / "train_operator_yolo_detector.sh"
|
||||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
assert script_path.exists()
|
assert script_path.exists()
|
||||||
script = script_path.read_text(encoding="utf-8")
|
script = script_path.read_text(encoding="utf-8")
|
||||||
@@ -62,7 +69,7 @@ def test_operator_yolo_train_smoke_script_contract() -> None:
|
|||||||
assert "TRAIN_EPOCHS" in script
|
assert "TRAIN_EPOCHS" in script
|
||||||
assert "TRAIN_IMGSZ" in script
|
assert "TRAIN_IMGSZ" in script
|
||||||
assert "/opt/geointel/venv/bin/python" in script
|
assert "/opt/geointel/venv/bin/python" in script
|
||||||
assert "PYTHON_BIN=\"python3\"" in script
|
assert 'PYTHON_BIN="python3"' in script
|
||||||
assert "dataset.yaml" in script
|
assert "dataset.yaml" in script
|
||||||
assert "from ultralytics import YOLO" in script
|
assert "from ultralytics import YOLO" in script
|
||||||
assert "model.train" in script
|
assert "model.train" in script
|
||||||
@@ -76,6 +83,42 @@ def test_operator_yolo_train_smoke_script_contract() -> None:
|
|||||||
assert '"base_model_sha256"' in script
|
assert '"base_model_sha256"' in script
|
||||||
assert '"trained_model_sha256"' in script
|
assert '"trained_model_sha256"' in script
|
||||||
assert "training_release_manifest.py" in script
|
assert "training_release_manifest.py" in script
|
||||||
|
assert "NO_TRAINING.json" in script
|
||||||
|
assert "Training is prohibited for this evaluation-only dataset" in script
|
||||||
assert "verify" in script
|
assert "verify" in script
|
||||||
assert "download" not in script.lower()
|
assert "download" not in script.lower()
|
||||||
assert "fixture_mode" not in script
|
assert "fixture_mode" not in script
|
||||||
|
|
||||||
|
|
||||||
|
def test_operator_yolo_train_rejects_evaluation_only_marker(tmp_path: Path) -> None:
|
||||||
|
dataset_dir = tmp_path / "evaluation-only"
|
||||||
|
dataset_dir.mkdir()
|
||||||
|
(dataset_dir / "dataset.yaml").write_text("val: images/val\n", encoding="utf-8")
|
||||||
|
(dataset_dir / "NO_TRAINING.json").write_text(
|
||||||
|
'{"training_prohibited": true}\n', encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
def shell_path(path: Path) -> str:
|
||||||
|
if path.drive:
|
||||||
|
return f"/mnt/{path.drive[0].lower()}/{path.as_posix()[3:]}"
|
||||||
|
return str(path)
|
||||||
|
|
||||||
|
dataset_shell_path = shell_path(dataset_dir)
|
||||||
|
model_shell_path = shell_path(tmp_path / "missing.pt")
|
||||||
|
script_shell_path = shell_path(ROOT / "scripts" / "train_operator_yolo_detector.sh")
|
||||||
|
command = (
|
||||||
|
f"OPERATOR_YOLO_DATASET_DIR={shlex.quote(dataset_shell_path)} "
|
||||||
|
f"YOLO_BASE_MODEL_PATH={shlex.quote(model_shell_path)} "
|
||||||
|
f"bash {shlex.quote(script_shell_path)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
["bash", "-lc", command],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.returncode == 1
|
||||||
|
assert "Training is prohibited for this evaluation-only dataset" in result.stderr
|
||||||
|
assert "Base model file not found" not in result.stderr
|
||||||
|
|||||||
@@ -12671,3 +12671,36 @@ Open:
|
|||||||
- The min-4px corpus is preferred for future experimental work, but remains
|
- The min-4px corpus is preferred for future experimental work, but remains
|
||||||
explicitly training/release-ineligible until real human review and governed
|
explicitly training/release-ineligible until real human review and governed
|
||||||
release evidence exist. The active GPU model was not changed.
|
release evidence exist. The active GPU model was not changed.
|
||||||
|
|
||||||
|
## 2026-08-09 - Non-overlapping checkpoint correction
|
||||||
|
|
||||||
|
### Corrected
|
||||||
|
|
||||||
|
- Traced the prior 36-image checkpoint matrix to four AOIs tiled at 512/256.
|
||||||
|
The same interiors were repeated and nine Arendonk-heide tiles were visually
|
||||||
|
blank no-data, despite being counted as background observations.
|
||||||
|
- Added a deterministic non-overlap evaluation-view builder. It selects four
|
||||||
|
complete 512 px grid tiles per 1024 px AOI, excludes low-variance no-data
|
||||||
|
with reason codes, creates empty train directories and writes a hashed
|
||||||
|
`NO_TRAINING.json` marker.
|
||||||
|
- Added a fail-fast marker check to the operator YOLO training wrapper before
|
||||||
|
release verification, model loading or CUDA allocation.
|
||||||
|
- Corrected checkpoint overlap semantics: zero pixel overlap is reported, but
|
||||||
|
statistical independence is explicitly not established for adjacent tiles.
|
||||||
|
|
||||||
|
### GPU evidence
|
||||||
|
|
||||||
|
- The clean r2 view contains 12 images: eight positive Turnhout/Westerlo tiles
|
||||||
|
and four real Postel backgrounds. It has 2,551 labels, 2,384 unique interior
|
||||||
|
objects, 167 edge labels and zero repeated interior objects.
|
||||||
|
- Active: precision 0.542632, recall 0.435317, mAP50 0.342034, mAP50-95
|
||||||
|
0.141316. Challenger: 0.581715, 0.462300, 0.368364 and 0.155318.
|
||||||
|
- At confidence 0.15 the active model has zero Postel detections and the
|
||||||
|
challenger one; at 0.25 both have zero. The threshold-sensitive relative
|
||||||
|
challenger advantage is confirmed without changing production.
|
||||||
|
|
||||||
|
### Remaining limitations
|
||||||
|
|
||||||
|
- Two adjacent positive AOIs and one real background AOI are not representative
|
||||||
|
release evidence. Human review, broader independent geography and exact
|
||||||
|
challenger provenance remain blocking.
|
||||||
|
|||||||
@@ -1170,6 +1170,13 @@ This file now starts with the current implementation status. Older preparation/b
|
|||||||
- [ ] Compare the existing 0.35 edge-visibility policy against a separately
|
- [ ] Compare the existing 0.35 edge-visibility policy against a separately
|
||||||
versioned stricter ablation from original source geometry before any new
|
versioned stricter ablation from original source geometry before any new
|
||||||
governed training; do not approximate this by dropping every edge label.
|
governed training; do not approximate this by dropping every edge label.
|
||||||
|
- [x] Replace the overlapping checkpoint ranking with a checksum-bound,
|
||||||
|
training-disabled non-overlap view: 12 real tiles, 2,384 unique interior
|
||||||
|
objects, zero repeated interiors and explicit removal of blank Arendonk
|
||||||
|
no-data imagery.
|
||||||
|
- [ ] Add multiple independent real-background AOIs and more geographically
|
||||||
|
separated positive validation AOIs before interpreting non-overlap ranking
|
||||||
|
as release evidence; Postel alone is not representative.
|
||||||
- [ ] Convert the AI-assisted ledger into no stronger claim than experimental
|
- [ ] Convert the AI-assisted ledger into no stronger claim than experimental
|
||||||
triage; a real human must independently review and sign the frozen artifacts
|
triage; a real human must independently review and sign the frozen artifacts
|
||||||
before the governed training wrapper may unlock.
|
before the governed training wrapper may unlock.
|
||||||
|
|||||||
@@ -187,3 +187,32 @@ visible; it was not suppressed or relabelled as success.
|
|||||||
This min-4px version is the preferred experimental successor to the legacy
|
This min-4px version is the preferred experimental successor to the legacy
|
||||||
min-3px corpus. It explicitly remains ineligible for governed training until
|
min-3px corpus. It explicitly remains ineligible for governed training until
|
||||||
real human review and all release-contract evidence exist.
|
real human review and all release-contract evidence exist.
|
||||||
|
|
||||||
|
## Non-overlapping checkpoint re-evaluation
|
||||||
|
|
||||||
|
The 36-tile checkpoint matrix was traced to a 512 px validation corpus with
|
||||||
|
stride 256. It contained nine views each of Turnhout, Westerlo, Postel-bos and
|
||||||
|
Arendonk-heide. Visual inspection then exposed that all Arendonk-heide views
|
||||||
|
are blank/no-data imagery, not meaningful pure-background observations. The
|
||||||
|
historical claim of 18 background images is therefore corrected: nine were
|
||||||
|
blank no-data and nine represented Postel under overlap.
|
||||||
|
|
||||||
|
A training-disabled validation view now covers each 1024 px AOI with four
|
||||||
|
non-overlapping 512 px tiles. Four blank Arendonk representatives are excluded
|
||||||
|
with explicit reason codes. The resulting set contains eight positive tiles
|
||||||
|
from Turnhout/Westerlo, four real Postel background tiles and 2,551 labels.
|
||||||
|
Exact reconstruction finds 2,384 unique interior objects, 167 edge labels and
|
||||||
|
zero repeated interior objects. The view has empty train directories, a
|
||||||
|
checksum-bound `NO_TRAINING.json`, and the training wrapper rejects that marker.
|
||||||
|
|
||||||
|
On the Tower RTX 4080 SUPER, the active checkpoint measures precision
|
||||||
|
`0.542632`, recall `0.435317`, mAP50 `0.342034` and mAP50-95 `0.141316`.
|
||||||
|
The reviewedexp6 challenger measures `0.581715`, `0.462300`, `0.368364` and
|
||||||
|
`0.155318` respectively. At background confidence 0.15 the active checkpoint
|
||||||
|
has zero Postel detections and the challenger has one. At 0.25 both have zero.
|
||||||
|
Thus the refined evidence confirms the challenger's relative metric advantage
|
||||||
|
and its threshold sensitivity, but still does not authorize promotion.
|
||||||
|
|
||||||
|
Non-overlap is not overstated as statistical independence: the positive tiles
|
||||||
|
remain adjacent and come from only two AOIs, edge objects can remain split, and
|
||||||
|
there is only one real background AOI. The production model remains unchanged.
|
||||||
|
|||||||
@@ -35,6 +35,14 @@ excludes rather than relabels any positive tile that would become empty, and
|
|||||||
writes complete source/output hashes plus a row-level removal manifest. It
|
writes complete source/output hashes plus a row-level removal manifest. It
|
||||||
never creates a training-release manifest or human-review claim.
|
never creates a training-release manifest or human-review claim.
|
||||||
|
|
||||||
|
`derive_yolo_nonoverlap_evaluation_view.py` selects the complete non-overlap
|
||||||
|
grid from an existing overlapping validation corpus, records every excluded
|
||||||
|
low-variance/no-data tile, leaves train directories empty and writes a
|
||||||
|
checksum-bound `NO_TRAINING.json`. The operator training wrapper rejects that
|
||||||
|
marker before release verification or CUDA allocation. Non-overlap proves only
|
||||||
|
that tiles do not share pixels; checkpoint reports explicitly avoid claiming
|
||||||
|
statistical independence for adjacent tiles from the same AOI.
|
||||||
|
|
||||||
Setup-, import-, demo- en maintenance-scripts voor GeoIntel.
|
Setup-, import-, demo- en maintenance-scripts voor GeoIntel.
|
||||||
|
|
||||||
## WALOUS source provisioning
|
## WALOUS source provisioning
|
||||||
|
|||||||
@@ -0,0 +1,259 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Derive a checksum-bound, training-disabled non-overlapping YOLO val view."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
from collections import defaultdict
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
try:
|
||||||
|
from scripts.audit_yolo_cross_tile_repetition import tile_offsets
|
||||||
|
except ModuleNotFoundError: # Standalone operator-tool copy beside the auditor.
|
||||||
|
from audit_yolo_cross_tile_repetition import tile_offsets
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def select_nonoverlap_tiles(
|
||||||
|
tiles: list[dict[str, Any]], tile_size: int
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
selected: list[dict[str, Any]] = []
|
||||||
|
for tile in tiles:
|
||||||
|
if not isinstance(tile, dict) or not tile.get("kept", True):
|
||||||
|
continue
|
||||||
|
if str(tile.get("split") or "") != "val":
|
||||||
|
continue
|
||||||
|
row_offset, column_offset = tile_offsets(str(tile.get("image_path") or ""))
|
||||||
|
if row_offset % tile_size == 0 and column_offset % tile_size == 0:
|
||||||
|
selected.append(tile)
|
||||||
|
return sorted(
|
||||||
|
selected,
|
||||||
|
key=lambda tile: (
|
||||||
|
str(tile.get("sample_slug") or ""),
|
||||||
|
int(tile.get("tile_index") or 0),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def aggregate_hash(paths: list[Path], root: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
for path in sorted(paths):
|
||||||
|
digest.update(path.relative_to(root).as_posix().encode("utf-8"))
|
||||||
|
digest.update(b"\0")
|
||||||
|
digest.update(sha256_file(path).encode("ascii"))
|
||||||
|
digest.update(b"\n")
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def image_has_low_visual_variance(image_path: Path, blank_range_threshold: int) -> bool:
|
||||||
|
image = Image.open(image_path).convert("L")
|
||||||
|
minimum, maximum = image.getextrema()
|
||||||
|
return maximum - minimum <= blank_range_threshold
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--source-dir", required=True, type=Path)
|
||||||
|
parser.add_argument("--output-dir", required=True, type=Path)
|
||||||
|
parser.add_argument(
|
||||||
|
"--exclude-low-variance",
|
||||||
|
action=argparse.BooleanOptionalAction,
|
||||||
|
default=True,
|
||||||
|
)
|
||||||
|
parser.add_argument("--blank-range-threshold", type=int, default=3)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
source_dir = args.source_dir.expanduser().resolve(strict=True)
|
||||||
|
output_dir = args.output_dir.expanduser().resolve(strict=False)
|
||||||
|
if output_dir.exists():
|
||||||
|
parser.error(f"output directory already exists: {output_dir}")
|
||||||
|
source_summary_path = source_dir / "yolo_tile_dataset_summary.json"
|
||||||
|
source_summary = json.loads(source_summary_path.read_text(encoding="utf-8"))
|
||||||
|
tiles = source_summary.get("tiles")
|
||||||
|
tile_size = source_summary.get("tile_size")
|
||||||
|
if not isinstance(tiles, list):
|
||||||
|
raise ValueError("source summary must contain a tiles list")
|
||||||
|
if not isinstance(tile_size, int) or tile_size <= 0:
|
||||||
|
raise ValueError("source summary must contain a positive integer tile_size")
|
||||||
|
selected_candidates = select_nonoverlap_tiles(tiles, tile_size)
|
||||||
|
if not selected_candidates:
|
||||||
|
raise ValueError("no non-overlapping validation tiles were selected")
|
||||||
|
|
||||||
|
selected: list[dict[str, Any]] = []
|
||||||
|
excluded_tiles: list[dict[str, Any]] = []
|
||||||
|
for tile in selected_candidates:
|
||||||
|
source_image = Path(str(tile.get("image_path") or "")).resolve(strict=True)
|
||||||
|
if args.exclude_low_variance and image_has_low_visual_variance(
|
||||||
|
source_image, args.blank_range_threshold
|
||||||
|
):
|
||||||
|
excluded_tiles.append(
|
||||||
|
{
|
||||||
|
"sample_slug": str(tile.get("sample_slug") or "unknown"),
|
||||||
|
"tile_index": int(tile.get("tile_index") or 0),
|
||||||
|
"source_image_path": str(source_image),
|
||||||
|
"reason": "low_visual_variance_no_data",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
selected.append(tile)
|
||||||
|
if not selected:
|
||||||
|
raise ValueError("every non-overlapping validation tile was excluded")
|
||||||
|
|
||||||
|
offsets_by_sample: dict[str, set[tuple[int, int]]] = defaultdict(set)
|
||||||
|
source_val_counts: dict[str, int] = defaultdict(int)
|
||||||
|
for tile in tiles:
|
||||||
|
if (
|
||||||
|
isinstance(tile, dict)
|
||||||
|
and tile.get("kept", True)
|
||||||
|
and tile.get("split") == "val"
|
||||||
|
):
|
||||||
|
source_val_counts[str(tile.get("sample_slug") or "unknown")] += 1
|
||||||
|
for tile in selected:
|
||||||
|
offsets_by_sample[str(tile.get("sample_slug") or "unknown")].add(
|
||||||
|
tile_offsets(str(tile.get("image_path") or ""))
|
||||||
|
)
|
||||||
|
accounted_samples = set(offsets_by_sample) | {
|
||||||
|
tile["sample_slug"] for tile in excluded_tiles
|
||||||
|
}
|
||||||
|
if accounted_samples != set(source_val_counts):
|
||||||
|
raise ValueError("non-overlap selection has an unaccounted validation sample")
|
||||||
|
|
||||||
|
output_dir.mkdir(parents=True)
|
||||||
|
(output_dir / "images" / "train").mkdir(parents=True)
|
||||||
|
(output_dir / "labels" / "train").mkdir(parents=True)
|
||||||
|
output_tiles: list[dict[str, Any]] = []
|
||||||
|
output_files: list[Path] = []
|
||||||
|
for tile in selected:
|
||||||
|
source_image = Path(str(tile.get("image_path") or "")).resolve(strict=True)
|
||||||
|
source_label = Path(str(tile.get("label_path") or "")).resolve(strict=True)
|
||||||
|
target_image = output_dir / "images" / "val" / source_image.name
|
||||||
|
target_label = output_dir / "labels" / "val" / source_label.name
|
||||||
|
target_image.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target_label.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(source_image, target_image)
|
||||||
|
shutil.copy2(source_label, target_label)
|
||||||
|
output_files.extend((target_image, target_label))
|
||||||
|
derived_tile = dict(tile)
|
||||||
|
derived_tile.update(
|
||||||
|
{
|
||||||
|
"image_path": str(target_image),
|
||||||
|
"label_path": str(target_label),
|
||||||
|
"derived_from_image_path": str(source_image),
|
||||||
|
"derived_from_label_path": str(source_label),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
output_tiles.append(derived_tile)
|
||||||
|
|
||||||
|
dataset_yaml = output_dir / "dataset.yaml"
|
||||||
|
dataset_yaml.write_text(
|
||||||
|
"\n".join(
|
||||||
|
(
|
||||||
|
f"path: {output_dir}",
|
||||||
|
"train: images/train",
|
||||||
|
"val: images/val",
|
||||||
|
"names:",
|
||||||
|
" 0: building",
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
no_training = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"training_prohibited": True,
|
||||||
|
"reason": "Immutable non-overlapping evaluation view; train directories are intentionally empty.",
|
||||||
|
"source_summary_sha256": sha256_file(source_summary_path),
|
||||||
|
}
|
||||||
|
no_training_path = output_dir / "NO_TRAINING.json"
|
||||||
|
no_training_path.write_text(
|
||||||
|
json.dumps(no_training, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
summary = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"status": "ready_evaluation_only",
|
||||||
|
"claim_boundary": (
|
||||||
|
"Non-protected, non-overlapping tile-level validation view; no "
|
||||||
|
"training, protected-test, object-level independence or promotion claim."
|
||||||
|
),
|
||||||
|
"dataset_yaml": str(dataset_yaml),
|
||||||
|
"output_dir": str(output_dir),
|
||||||
|
"source_dataset_dir": str(source_dir),
|
||||||
|
"source_summary_path": str(source_summary_path),
|
||||||
|
"source_summary_sha256": sha256_file(source_summary_path),
|
||||||
|
"class_names": source_summary.get("class_names", ["building"]),
|
||||||
|
"tile_size": tile_size,
|
||||||
|
"stride": tile_size,
|
||||||
|
"source_stride": source_summary.get("stride"),
|
||||||
|
"training_prohibited": True,
|
||||||
|
"training_marker": str(no_training_path),
|
||||||
|
"tile_count": len(output_tiles),
|
||||||
|
"val_tile_count": len(output_tiles),
|
||||||
|
"train_tile_count": 0,
|
||||||
|
"positive_tile_count": sum(
|
||||||
|
int(tile.get("label_count") or 0) > 0 for tile in output_tiles
|
||||||
|
),
|
||||||
|
"negative_tile_count": sum(
|
||||||
|
int(tile.get("label_count") or 0) == 0 for tile in output_tiles
|
||||||
|
),
|
||||||
|
"label_count": sum(int(tile.get("label_count") or 0) for tile in output_tiles),
|
||||||
|
"selected_offsets_by_sample": {
|
||||||
|
sample: [list(offset) for offset in sorted(offsets)]
|
||||||
|
for sample, offsets in sorted(offsets_by_sample.items())
|
||||||
|
},
|
||||||
|
"source_val_tile_counts_by_sample": dict(sorted(source_val_counts.items())),
|
||||||
|
"excluded_tile_count": len(excluded_tiles),
|
||||||
|
"excluded_tiles": excluded_tiles,
|
||||||
|
"excluded_sample_slugs": sorted(
|
||||||
|
{tile["sample_slug"] for tile in excluded_tiles}
|
||||||
|
),
|
||||||
|
"exclude_low_variance": args.exclude_low_variance,
|
||||||
|
"blank_range_threshold": args.blank_range_threshold,
|
||||||
|
"tiles": output_tiles,
|
||||||
|
}
|
||||||
|
summary_path = output_dir / "yolo_tile_dataset_summary.json"
|
||||||
|
summary_path.write_text(
|
||||||
|
json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
manifest = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"generated_at": datetime.now(UTC).isoformat(),
|
||||||
|
"status": "complete_evaluation_only",
|
||||||
|
"source_summary_path": str(source_summary_path),
|
||||||
|
"source_summary_sha256": sha256_file(source_summary_path),
|
||||||
|
"output_summary_path": str(summary_path),
|
||||||
|
"output_summary_sha256": sha256_file(summary_path),
|
||||||
|
"dataset_yaml_sha256": sha256_file(dataset_yaml),
|
||||||
|
"evaluation_file_set_aggregate_sha256": aggregate_hash(
|
||||||
|
output_files, output_dir
|
||||||
|
),
|
||||||
|
"training_marker_sha256": sha256_file(no_training_path),
|
||||||
|
"training_prohibited": True,
|
||||||
|
"selected_tile_count": len(output_tiles),
|
||||||
|
"selected_sample_count": len(offsets_by_sample),
|
||||||
|
"excluded_tile_count": len(excluded_tiles),
|
||||||
|
"excluded_tiles": excluded_tiles,
|
||||||
|
}
|
||||||
|
manifest_path = output_dir / "evaluation_view_manifest.json"
|
||||||
|
manifest_path.write_text(
|
||||||
|
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
print(json.dumps(manifest, indent=2))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -59,7 +59,8 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
|
|||||||
return {
|
return {
|
||||||
"status": "unavailable",
|
"status": "unavailable",
|
||||||
"summary_path": str(summary_path),
|
"summary_path": str(summary_path),
|
||||||
"validation_rows_independent": None,
|
"validation_tiles_non_overlapping": None,
|
||||||
|
"statistical_independence_established": False,
|
||||||
}
|
}
|
||||||
payload = json.loads(summary_path.read_text(encoding="utf-8"))
|
payload = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||||
tile_size = payload.get("tile_size")
|
tile_size = payload.get("tile_size")
|
||||||
@@ -69,7 +70,8 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
|
|||||||
"status": "invalid",
|
"status": "invalid",
|
||||||
"summary_path": str(summary_path),
|
"summary_path": str(summary_path),
|
||||||
"summary_sha256": sha256_file(summary_path),
|
"summary_sha256": sha256_file(summary_path),
|
||||||
"validation_rows_independent": None,
|
"validation_tiles_non_overlapping": None,
|
||||||
|
"statistical_independence_established": False,
|
||||||
}
|
}
|
||||||
overlap_pixels = max(tile_size - stride, 0)
|
overlap_pixels = max(tile_size - stride, 0)
|
||||||
return {
|
return {
|
||||||
@@ -79,12 +81,15 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
|
|||||||
"tile_size": tile_size,
|
"tile_size": tile_size,
|
||||||
"stride": stride,
|
"stride": stride,
|
||||||
"overlap_pixels": overlap_pixels,
|
"overlap_pixels": overlap_pixels,
|
||||||
"validation_rows_independent": overlap_pixels == 0,
|
"validation_tiles_non_overlapping": overlap_pixels == 0,
|
||||||
|
"statistical_independence_established": False,
|
||||||
"interpretation": (
|
"interpretation": (
|
||||||
"Tile metrics can repeat the same source object and are valid for "
|
"Tile metrics can repeat the same source object and are valid for "
|
||||||
"candidate ranking only, not independent object-level uncertainty."
|
"candidate ranking only, not independent object-level uncertainty."
|
||||||
if overlap_pixels
|
if overlap_pixels
|
||||||
else "Tile rows do not overlap according to the dataset summary."
|
else "Tiles do not overlap, but spatial/statistical independence is "
|
||||||
|
"not established because adjacent tiles share AOI context and edge "
|
||||||
|
"objects can remain split."
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ fi
|
|||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
|
DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
|
||||||
|
NO_TRAINING_MARKER="${OPERATOR_YOLO_DATASET_DIR%/}/NO_TRAINING.json"
|
||||||
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
|
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
|
||||||
export DATASET_YAML
|
export DATASET_YAML
|
||||||
export YOLO_BASE_MODEL_PATH
|
export YOLO_BASE_MODEL_PATH
|
||||||
@@ -76,6 +77,11 @@ if [[ ! -f "${DATASET_YAML}" ]]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [[ -f "${NO_TRAINING_MARKER}" ]]; then
|
||||||
|
echo "Training is prohibited for this evaluation-only dataset: ${NO_TRAINING_MARKER}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
|
if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
|
||||||
echo "Base model file not found: ${YOLO_BASE_MODEL_PATH}" >&2
|
echo "Base model file not found: ${YOLO_BASE_MODEL_PATH}" >&2
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from scripts.derive_yolo_nonoverlap_evaluation_view import (
|
||||||
|
image_has_low_visual_variance,
|
||||||
|
select_nonoverlap_tiles,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def tile(name: str, index: int, split: str = "val") -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"image_path": f"/data/sample_{index:04d}_{name}.png",
|
||||||
|
"tile_index": index,
|
||||||
|
"sample_slug": "sample",
|
||||||
|
"split": split,
|
||||||
|
"kept": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_select_nonoverlap_tiles_keeps_complete_512_grid() -> None:
|
||||||
|
tiles = [
|
||||||
|
tile("r0_c0", 0),
|
||||||
|
tile("r0_c256", 1),
|
||||||
|
tile("r0_c512", 2),
|
||||||
|
tile("r256_c0", 3),
|
||||||
|
tile("r256_c256", 4),
|
||||||
|
tile("r256_c512", 5),
|
||||||
|
tile("r512_c0", 6),
|
||||||
|
tile("r512_c256", 7),
|
||||||
|
tile("r512_c512", 8),
|
||||||
|
tile("r0_c0", 9, split="train"),
|
||||||
|
]
|
||||||
|
|
||||||
|
selected = select_nonoverlap_tiles(tiles, 512)
|
||||||
|
|
||||||
|
assert [item["tile_index"] for item in selected] == [0, 2, 6, 8]
|
||||||
|
|
||||||
|
|
||||||
|
def test_image_has_low_visual_variance_rejects_blank_no_data(tmp_path: Path) -> None:
|
||||||
|
blank = tmp_path / "blank.png"
|
||||||
|
real = tmp_path / "real.png"
|
||||||
|
Image.new("L", (8, 8), color=255).save(blank)
|
||||||
|
image = Image.new("L", (8, 8), color=100)
|
||||||
|
image.putpixel((0, 0), 120)
|
||||||
|
image.save(real)
|
||||||
|
|
||||||
|
assert image_has_low_visual_variance(blank, 3) is True
|
||||||
|
assert image_has_low_visual_variance(real, 3) is False
|
||||||
@@ -17,7 +17,8 @@ def test_dataset_overlap_evidence_marks_repeated_validation_rows(
|
|||||||
|
|
||||||
assert evidence["status"] == "overlapping"
|
assert evidence["status"] == "overlapping"
|
||||||
assert evidence["overlap_pixels"] == 256
|
assert evidence["overlap_pixels"] == 256
|
||||||
assert evidence["validation_rows_independent"] is False
|
assert evidence["validation_tiles_non_overlapping"] is False
|
||||||
|
assert evidence["statistical_independence_established"] is False
|
||||||
|
|
||||||
|
|
||||||
def test_dataset_overlap_evidence_is_explicit_when_summary_missing(
|
def test_dataset_overlap_evidence_is_explicit_when_summary_missing(
|
||||||
@@ -26,4 +27,21 @@ def test_dataset_overlap_evidence_is_explicit_when_summary_missing(
|
|||||||
evidence = dataset_overlap_evidence(tmp_path / "dataset.yaml")
|
evidence = dataset_overlap_evidence(tmp_path / "dataset.yaml")
|
||||||
|
|
||||||
assert evidence["status"] == "unavailable"
|
assert evidence["status"] == "unavailable"
|
||||||
assert evidence["validation_rows_independent"] is None
|
assert evidence["validation_tiles_non_overlapping"] is None
|
||||||
|
assert evidence["statistical_independence_established"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_nonoverlap_does_not_overclaim_statistical_independence(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
dataset_yaml = tmp_path / "dataset.yaml"
|
||||||
|
dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8")
|
||||||
|
(tmp_path / "yolo_tile_dataset_summary.json").write_text(
|
||||||
|
json.dumps({"tile_size": 512, "stride": 512}), encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
evidence = dataset_overlap_evidence(dataset_yaml)
|
||||||
|
|
||||||
|
assert evidence["validation_tiles_non_overlapping"] is True
|
||||||
|
assert evidence["statistical_independence_established"] is False
|
||||||
|
assert "not established" in evidence["interpretation"]
|
||||||
|
|||||||
Reference in New Issue
Block a user