From fd45f37a38c7c30a3a0b89ae46470f3c36ade0df Mon Sep 17 00:00:00 2001 From: Jens Date: Sun, 9 Aug 2026 21:01:12 +0200 Subject: [PATCH] block training-seen checkpoint evaluation --- ...0809-v70-evaluation-independence-gate.json | 43 ++++++ docs/CODEX_EXECUTION_LOG.md | 33 ++++ docs/TODO.md | 6 +- ...026-08-09-ai-assisted-checkpoint-review.md | 21 +++ scripts/README.md | 7 + scripts/evaluate_yolo_checkpoint_matrix.py | 145 +++++++++++++++++- tests/test_evaluate_yolo_checkpoint_matrix.py | 110 ++++++++++++- 7 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 artifacts/evidence/accuracy/model-training/20260809-v70-evaluation-independence-gate.json diff --git a/artifacts/evidence/accuracy/model-training/20260809-v70-evaluation-independence-gate.json b/artifacts/evidence/accuracy/model-training/20260809-v70-evaluation-independence-gate.json new file mode 100644 index 00000000..8b8d3b5f --- /dev/null +++ b/artifacts/evidence/accuracy/model-training/20260809-v70-evaluation-independence-gate.json @@ -0,0 +1,43 @@ +{ + "claim_boundary": "No checkpoint ranking or release claim is permitted.", + "dataset_yaml": "/app/storage/operator-data/yolo-building-aoi1024-bgaware512r8-nonoverlap-eval-r2/dataset.yaml", + "dataset_yaml_sha256": "c36f6d76aac4bc2f5e880e2f0133d176fb0dc277600e121570a944eee6db2c48", + "generated_at": "2026-08-09T18:59:13.410422+00:00", + "gpu_inference_attempted": false, + "model_loading_attempted": false, + "schema_version": 2, + "status": "blocked_training_sample_overlap", + "training_sample_independence_evidence": { + "evaluation_samples": [ + "postel_bos", + "turnhout", + "westerlo" + ], + "evaluation_summary_path": "/app/storage/operator-data/yolo-building-aoi1024-bgaware512r8-nonoverlap-eval-r2/yolo_tile_dataset_summary.json", + "evaluation_summary_sha256": "c17a8c47eaecc2dfb38778ff5a612f9e6c4d67c9e90e31d3a6a66d8857adec4c", + "independent_for_all_supplied_training_corpora": false, + "interpretation": "At least one evaluation AOI occurs in a supplied train split; the matrix is blocked before model loading.", + "overlapping_evaluation_samples": [ + "postel_bos" + ], + "status": "overlap", + "training_corpora": [ + { + "overlapping_evaluation_samples": [ + "postel_bos" + ], + "training_sample_count": 20, + "training_summary_path": "/app/storage/operator-data/yolo-building-aoi1024-smallbld-minpx3vis035/yolo_tile_dataset_summary.json", + "training_summary_sha256": "49b2a07d2105d08356431757b83eafc1498eaf1fb76965b1efe05b776824942a" + }, + { + "overlapping_evaluation_samples": [ + "postel_bos" + ], + "training_sample_count": 26, + "training_summary_path": "/app/storage/operator-data/yolo-building-aoi1024-reviewedexp6-minpx3vis035/yolo_tile_dataset_summary.json", + "training_summary_sha256": "7c917e31216d1df2174c0f9c736f88a81f3835aa991971f8fb8665e17ddf5c9c" + } + ] + } +} diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 09e0efc4..a0ba60bb 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -12704,3 +12704,36 @@ Open: - Two adjacent positive AOIs and one real background AOI are not representative release evidence. Human review, broader independent geography and exact challenger provenance remain blocking. + +## 2026-08-09 - Training/evaluation membership gate + +### Corrected + +- Compared the non-overlap evaluation sample membership against the exact + active and challenger training summaries. Turnhout and Westerlo are absent + from both train splits; Postel-bos is present in both. +- Reclassified the Postel detection counts as training-seen regression checks, + not independent background validation. No existing metric or checkpoint was + promoted on that evidence. +- Extended the checkpoint evaluator with a strict multi-corpus membership gate. + Governed runs now persist a blocked manifest and stop before PyTorch import, + model loading and GPU inference if any evaluation AOI is training-seen. + +### Reproduced on Tower + +- Active training summary SHA-256: + `49b2a07d2105d08356431757b83eafc1498eaf1fb76965b1efe05b776824942a`. +- Challenger training summary SHA-256: + `7c917e31216d1df2174c0f9c736f88a81f3835aa991971f8fb8665e17ddf5c9c`. +- Evaluation summary SHA-256: + `c17a8c47eaecc2dfb38778ff5a612f9e6c4d67c9e90e31d3a6a66d8857adec4c`. +- The strict run blocked on `postel_bos`, with + `model_loading_attempted=false` and `gpu_inference_attempted=false`. +- Blocked-manifest SHA-256: + `62e8a9377ac87552936f472b0acf55d1d659ebf27f84ba8c4dbcefe7a374e1da`. + +### Remaining limitation + +- Independent real-background AOIs are still required. Literal 100% model + correctness is not demonstrated; the system now refuses to misrepresent the + known overlap as proof. diff --git a/docs/TODO.md b/docs/TODO.md index 430f8ae3..1f007557 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1176,7 +1176,11 @@ This file now starts with the current implementation status. Older preparation/b no-data imagery. - [ ] Add multiple independent real-background AOIs and more geographically separated positive validation AOIs before interpreting non-overlap ranking - as release evidence; Postel alone is not representative. + as release evidence. Postel occurs in both compared train splits and is only + a training-seen regression check, not independent validation. +- [x] Add a fail-closed checkpoint gate that checks every evaluation AOI + against the exact train split of every supplied model corpus before PyTorch + import, model loading or GPU inference. - [ ] Convert the AI-assisted ledger into no stronger claim than experimental triage; a real human must independently review and sign the frozen artifacts before the governed training wrapper may unlock. diff --git a/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md b/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md index 34256554..2350533c 100644 --- a/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md +++ b/docs/reviews/2026-08-09-ai-assisted-checkpoint-review.md @@ -216,3 +216,24 @@ and its threshold sensitivity, but still does not authorize promotion. Non-overlap is not overstated as statistical independence: the positive tiles remain adjacent and come from only two AOIs, edge objects can remain split, and there is only one real background AOI. The production model remains unchanged. + +## Training/evaluation membership correction + +The evaluation AOIs were subsequently checked against the exact tile summaries +of both compared checkpoints. `turnhout` and `westerlo` do not occur in either +train split. `postel_bos`, however, occurs in both: the active corpus contains +20 training AOIs and the challenger corpus 26, with Postel included in each. +The Postel image is real rather than blank/no-data, but it is training-seen. + +Consequently the positive Turnhout/Westerlo metrics remain a non-protected, +adjacent-AOI candidate ranking; the Postel detection counts are only +training-seen sanity/regression observations. They are not independent +pure-background validation and cannot support release, threshold or +generalisation claims. The earlier wording about a "real background AOI" must +be read with this correction. + +The checkpoint evaluator now accepts exact training summaries and, in governed +mode, fails before PyTorch import, model loading or GPU inference when any +evaluation AOI overlaps any supplied train split. The reproduced gate blocked +on `postel_bos` for both checkpoints. Its immutable machine-readable record is +`artifacts/evidence/accuracy/model-training/20260809-v70-evaluation-independence-gate.json`. diff --git a/scripts/README.md b/scripts/README.md index 3cd7c979..24147923 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -5,6 +5,13 @@ weights on one declared, non-protected `val` split using CUDA. It records exact model and dataset hashes, standard Ultralytics detection metrics and a separate pure-background detection count. The output claim is validation ranking only; the script neither reads protected test data nor promotes a model. +Governed comparisons must pass every candidate's exact tile-summary with +`--training-summary` and enable `--require-training-sample-independence`. If +any validation AOI occurs in any supplied train split, the script writes a +blocked manifest and exits before importing PyTorch, loading a model or using +the GPU. A background detection count from a training-seen AOI is only a +regression check and must never be presented as independent background +evidence. `render_operator_yolo_label_qa_contact_sheets.py` paginates complete visual reviews with `--tiles-per-sheet` (default `64`). This keeps large corpora diff --git a/scripts/evaluate_yolo_checkpoint_matrix.py b/scripts/evaluate_yolo_checkpoint_matrix.py index e4983fa0..a69baa97 100644 --- a/scripts/evaluate_yolo_checkpoint_matrix.py +++ b/scripts/evaluate_yolo_checkpoint_matrix.py @@ -94,6 +94,132 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]: } +def _sample_slugs(payload: dict[str, Any], *, split: str | None) -> set[str]: + tiles = payload.get("tiles") + if not isinstance(tiles, list): + raise ValueError("dataset summary requires a tiles list") + samples: set[str] = set() + for tile in tiles: + if not isinstance(tile, dict): + raise ValueError("dataset summary tiles must contain mappings") + if split is not None and tile.get("split") != split: + continue + sample_slug = tile.get("sample_slug") + if not isinstance(sample_slug, str) or not sample_slug.strip(): + raise ValueError("every selected tile requires a sample_slug") + samples.add(sample_slug.strip()) + return samples + + +def training_sample_independence_evidence( + dataset_yaml: Path, training_summaries: list[Path] +) -> dict[str, Any]: + evaluation_summary = dataset_yaml.parent / "yolo_tile_dataset_summary.json" + if not evaluation_summary.is_file(): + return { + "status": "unavailable", + "reason": "evaluation dataset summary is unavailable", + "evaluation_summary_path": str(evaluation_summary), + "independent_for_all_supplied_training_corpora": False, + } + + try: + evaluation_payload = json.loads(evaluation_summary.read_text(encoding="utf-8")) + evaluation_samples = _sample_slugs(evaluation_payload, split="val") + if not evaluation_samples: + raise ValueError("evaluation summary contains no validation samples") + except (json.JSONDecodeError, OSError, ValueError) as exc: + return { + "status": "invalid", + "reason": str(exc), + "evaluation_summary_path": str(evaluation_summary), + "evaluation_summary_sha256": sha256_file(evaluation_summary), + "independent_for_all_supplied_training_corpora": False, + } + + rows: list[dict[str, Any]] = [] + union_overlap: set[str] = set() + for raw_summary in training_summaries: + summary = raw_summary.expanduser().resolve(strict=True) + try: + payload = json.loads(summary.read_text(encoding="utf-8")) + training_samples = _sample_slugs(payload, split="train") + if not training_samples: + raise ValueError("training summary contains no training samples") + except (json.JSONDecodeError, OSError, ValueError) as exc: + return { + "status": "invalid", + "reason": f"{summary}: {exc}", + "evaluation_summary_path": str(evaluation_summary), + "evaluation_summary_sha256": sha256_file(evaluation_summary), + "evaluation_samples": sorted(evaluation_samples), + "independent_for_all_supplied_training_corpora": False, + } + overlap = evaluation_samples & training_samples + union_overlap.update(overlap) + rows.append( + { + "training_summary_path": str(summary), + "training_summary_sha256": sha256_file(summary), + "training_sample_count": len(training_samples), + "overlapping_evaluation_samples": sorted(overlap), + } + ) + + if not rows: + return { + "status": "unavailable", + "reason": "no training summaries were supplied", + "evaluation_summary_path": str(evaluation_summary), + "evaluation_summary_sha256": sha256_file(evaluation_summary), + "evaluation_samples": sorted(evaluation_samples), + "training_corpora": [], + "overlapping_evaluation_samples": [], + "independent_for_all_supplied_training_corpora": False, + "interpretation": ( + "Training/evaluation independence cannot be established " + "without exact training summaries." + ), + } + + independent = not union_overlap + return { + "status": "independent" if independent else "overlap", + "evaluation_summary_path": str(evaluation_summary), + "evaluation_summary_sha256": sha256_file(evaluation_summary), + "evaluation_samples": sorted(evaluation_samples), + "training_corpora": rows, + "overlapping_evaluation_samples": sorted(union_overlap), + "independent_for_all_supplied_training_corpora": independent, + "interpretation": ( + "No evaluation AOI occurs in the train split of any supplied corpus." + if independent + else "At least one evaluation AOI occurs in a supplied train split; " + "the matrix is blocked before model loading." + ), + } + + +def write_blocked_manifest( + output: Path, dataset_yaml: Path, evidence: dict[str, Any] +) -> None: + payload = { + "schema_version": 2, + "generated_at": datetime.now(UTC).isoformat(), + "status": "blocked_training_sample_overlap", + "claim_boundary": "No checkpoint ranking or release claim is permitted.", + "dataset_yaml": str(dataset_yaml), + "dataset_yaml_sha256": sha256_file(dataset_yaml), + "training_sample_independence_evidence": evidence, + "model_loading_attempted": False, + "gpu_inference_attempted": False, + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + def metric_value(metrics: Any, attribute: str) -> float: value = getattr(metrics.box, attribute) return float(value) @@ -135,6 +261,8 @@ def main() -> int: parser.add_argument("--output", type=Path, required=True) parser.add_argument("--background-prefix", action="append", required=True) parser.add_argument("--background-confidence", type=float, default=0.15) + parser.add_argument("--training-summary", type=Path, action="append", default=[]) + parser.add_argument("--require-training-sample-independence", action="store_true") parser.add_argument("--device", default="cuda:0") parser.add_argument("--imgsz", type=int, default=640) parser.add_argument("--batch", type=int, default=8) @@ -147,6 +275,20 @@ def main() -> int: dataset_yaml = args.dataset_yaml.expanduser().resolve(strict=True) images = validation_images(dataset_yaml) overlap_evidence = dataset_overlap_evidence(dataset_yaml) + independence_evidence = training_sample_independence_evidence( + dataset_yaml, args.training_summary + ) + if args.require_training_sample_independence and not args.training_summary: + parser.error( + "--require-training-sample-independence requires at least one " + "--training-summary" + ) + if ( + args.require_training_sample_independence + and not independence_evidence["independent_for_all_supplied_training_corpora"] + ): + write_blocked_manifest(args.output, dataset_yaml, independence_evidence) + return 3 prefixes = tuple(value.casefold() for value in args.background_prefix) import torch @@ -232,13 +374,14 @@ def main() -> int: reverse=True, ) payload = { - "schema_version": 1, + "schema_version": 2, "generated_at": datetime.now(UTC).isoformat(), "status": "ok" if successful else "failed", "claim_boundary": "Non-protected validation ranking only; no test, challenge or promotion claim.", "dataset_yaml": str(dataset_yaml), "dataset_yaml_sha256": sha256_file(dataset_yaml), "dataset_overlap_evidence": overlap_evidence, + "training_sample_independence_evidence": independence_evidence, "validation_image_count": len(images), "pure_background_prefixes": list(prefixes), "pure_background_confidence": args.background_confidence, diff --git a/tests/test_evaluate_yolo_checkpoint_matrix.py b/tests/test_evaluate_yolo_checkpoint_matrix.py index 8888632d..136c3b51 100644 --- a/tests/test_evaluate_yolo_checkpoint_matrix.py +++ b/tests/test_evaluate_yolo_checkpoint_matrix.py @@ -1,7 +1,11 @@ import json from pathlib import Path -from scripts.evaluate_yolo_checkpoint_matrix import dataset_overlap_evidence +from scripts.evaluate_yolo_checkpoint_matrix import ( + dataset_overlap_evidence, + training_sample_independence_evidence, + write_blocked_manifest, +) def test_dataset_overlap_evidence_marks_repeated_validation_rows( @@ -45,3 +49,107 @@ def test_nonoverlap_does_not_overclaim_statistical_independence( assert evidence["validation_tiles_non_overlapping"] is True assert evidence["statistical_independence_established"] is False assert "not established" in evidence["interpretation"] + + +def _write_summary(path: Path, rows: list[tuple[str, str]]) -> None: + path.write_text( + json.dumps( + { + "tile_size": 512, + "stride": 512, + "tiles": [ + {"sample_slug": sample_slug, "split": split} + for sample_slug, split in rows + ], + } + ), + encoding="utf-8", + ) + + +def test_training_sample_independence_detects_overlap_for_each_corpus( + tmp_path: Path, +) -> None: + evaluation = tmp_path / "evaluation" + evaluation.mkdir() + dataset_yaml = evaluation / "dataset.yaml" + dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8") + _write_summary( + evaluation / "yolo_tile_dataset_summary.json", + [("turnhout", "val"), ("postel_bos", "val")], + ) + active = tmp_path / "active.json" + challenger = tmp_path / "challenger.json" + _write_summary(active, [("postel_bos", "train"), ("mol", "train")]) + _write_summary( + challenger, [("postel_bos", "train"), ("dessel", "train")] + ) + + evidence = training_sample_independence_evidence( + dataset_yaml, [active, challenger] + ) + + assert evidence["status"] == "overlap" + assert evidence["overlapping_evaluation_samples"] == ["postel_bos"] + assert evidence["independent_for_all_supplied_training_corpora"] is False + assert [ + row["overlapping_evaluation_samples"] for row in evidence["training_corpora"] + ] == [["postel_bos"], ["postel_bos"]] + + +def test_training_sample_independence_accepts_disjoint_samples(tmp_path: Path) -> None: + evaluation = tmp_path / "evaluation" + evaluation.mkdir() + dataset_yaml = evaluation / "dataset.yaml" + dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8") + _write_summary( + evaluation / "yolo_tile_dataset_summary.json", [("turnhout", "val")] + ) + training = tmp_path / "training.json" + _write_summary(training, [("mol", "train")]) + + evidence = training_sample_independence_evidence(dataset_yaml, [training]) + + assert evidence["status"] == "independent" + assert evidence["overlapping_evaluation_samples"] == [] + assert evidence["independent_for_all_supplied_training_corpora"] is True + + +def test_training_sample_independence_is_unavailable_without_training_corpus( + tmp_path: Path, +) -> None: + evaluation = tmp_path / "evaluation" + evaluation.mkdir() + dataset_yaml = evaluation / "dataset.yaml" + dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8") + _write_summary( + evaluation / "yolo_tile_dataset_summary.json", [("turnhout", "val")] + ) + + evidence = training_sample_independence_evidence(dataset_yaml, []) + + assert evidence["status"] == "unavailable" + assert evidence["overlapping_evaluation_samples"] == [] + assert evidence["independent_for_all_supplied_training_corpora"] is False + + +def test_blocked_manifest_records_that_models_and_gpu_were_not_used( + tmp_path: Path, +) -> None: + dataset_yaml = tmp_path / "dataset.yaml" + dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8") + output = tmp_path / "evidence" / "matrix.json" + + write_blocked_manifest( + output, + dataset_yaml, + { + "status": "overlap", + "independent_for_all_supplied_training_corpora": False, + }, + ) + + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["status"] == "blocked_training_sample_overlap" + assert payload["model_loading_attempted"] is False + assert payload["gpu_inference_attempted"] is False