Complete Belgian building corpus v2 workflow

scripts/audit_belgium_building_corpus.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Audit a frozen Belgian corpus and emit a deterministic review queue."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+REQUIRED_SPLITS = ("train", "val", "calibration", "test", "background-test")
+REGIONS = ("flanders", "wallonia", "brussels")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--corpus-dir", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--review-decisions", type=Path)
+    args = parser.parse_args()
+    manifest = json.loads((args.corpus_dir / "operator_samples_manifest.json").read_text(encoding="utf-8"))
+    leakage = json.loads((args.corpus_dir / "spatial-leakage-audit.json").read_text(encoding="utf-8"))
+    samples = manifest["samples"]
+    split_counts = Counter((sample["region"], sample["split"]) for sample in samples)
+    decision_counts: Counter[str] = Counter()
+    review_queue: list[dict[str, Any]] = []
+    total_input = 0
+    total_accepted = 0
+    temporal_unknown = 0
+    failures: list[str] = []
+    for region in REGIONS:
+        for split in REQUIRED_SPLITS:
+            minimum = 4 if split == "train" else 2
+            if split_counts[(region, split)] < minimum:
+                failures.append(f"{region}/{split} has {split_counts[(region, split)]}, requires {minimum}")
+    for sample in samples:
+        audit_path = args.corpus_dir / "pairs" / sample["sample_slug"] / "label-audit.json"
+        audit = json.loads(audit_path.read_text(encoding="utf-8"))
+        total_input += int(audit["input_feature_count"])
+        total_accepted += int(audit["accepted_feature_count"])
+        decision_counts.update(audit["decision_counts"])
+        if audit.get("temporal_alignment_status") == "unknown":
+            temporal_unknown += 1
+        expected_empty = bool(sample.get("sample_role") == "background_candidate" and sample.get("require_empty"))
+        if expected_empty and audit["accepted_feature_count"] != 0:
+            failures.append(f"{sample['sample_slug']} is not pure empty after normalization")
+        priority = "high" if audit["accepted_feature_count"] >= 200 or sample["split"] in {"test", "background-test"} else "normal"
+        review_queue.append(
+            {
+                "sample_slug": sample["sample_slug"],
+                "region": sample["region"],
+                "context": sample.get("context"),
+                "split": sample["split"],
+                "accepted_feature_count": audit["accepted_feature_count"],
+                "temporal_alignment_status": audit.get("temporal_alignment_status"),
+                "priority": priority,
+                "decision": "pending_human_review",
+            }
+        )
+    if leakage.get("status") != "ok":
+        failures.append("spatial leakage audit failed")
+    reviewed = 0
+    review_complete = False
+    if args.review_decisions and args.review_decisions.is_file():
+        decisions = json.loads(args.review_decisions.read_text(encoding="utf-8"))
+        by_slug = {item["sample_slug"]: item for item in decisions.get("decisions", [])}
+        for item in review_queue:
+            decision = by_slug.get(item["sample_slug"])
+            if decision:
+                item["decision"] = decision.get("decision")
+                item["reviewer"] = decision.get("reviewer")
+                item["notes"] = decision.get("notes")
+                if item["decision"] in {"accepted", "rejected"} and item.get("reviewer"):
+                    reviewed += 1
+        review_complete = reviewed == len(review_queue) and all(item["decision"] == "accepted" for item in review_queue)
+    status = "failed" if failures else ("ok" if review_complete else "needs_human_review")
+    report = {
+        "status": status,
+        "dataset_version": manifest["dataset_version"],
+        "manifest_immutable": manifest["immutable"],
+        "sample_count": len(samples),
+        "split_counts": {f"{region}/{split}": split_counts[(region, split)] for region in REGIONS for split in REQUIRED_SPLITS},
+        "input_feature_count": total_input,
+        "accepted_feature_count": total_accepted,
+        "decision_counts": dict(sorted(decision_counts.items())),
+        "temporal_unknown_sample_count": temporal_unknown,
+        "spatial_leakage_status": leakage.get("status"),
+        "reviewed_sample_count": reviewed,
+        "review_complete": review_complete,
+        "failures": failures,
+        "review_queue": review_queue,
+    }
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    (args.output_dir / "belgium-building-corpus-audit.json").write_text(
+        json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+    lines = [
+        f"# Belgian building corpus audit: {manifest['dataset_version']}",
+        "",
+        f"Status: `{status}`",
+        f"Samples: {len(samples)}; accepted labels: {total_accepted}/{total_input}.",
+        f"Spatial leakage: `{leakage.get('status')}`; human reviewed: {reviewed}/{len(samples)}.",
+        "",
+        "## Review queue",
+        "",
+        "| Sample | Region | Context | Split | Labels | Priority | Decision |",
+        "| --- | --- | --- | --- | ---: | --- | --- |",
+    ]
+    lines.extend(
+        f"| {item['sample_slug']} | {item['region']} | {item['context']} | {item['split']} | "
+        f"{item['accepted_feature_count']} | {item['priority']} | {item['decision']} |"
+        for item in review_queue
+    )
+    (args.output_dir / "belgium-building-corpus-audit.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print(json.dumps({key: value for key, value in report.items() if key != "review_queue"}, ensure_ascii=False, indent=2))
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())