#!/usr/bin/env python3 """Audit a frozen Belgian corpus and emit a deterministic review queue.""" from __future__ import annotations import argparse import json from collections import Counter from pathlib import Path from typing import Any REQUIRED_SPLITS = ("train", "val", "calibration", "test", "background-test") REGIONS = ("flanders", "wallonia", "brussels") def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--corpus-dir", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--review-decisions", type=Path) args = parser.parse_args() manifest = json.loads((args.corpus_dir / "operator_samples_manifest.json").read_text(encoding="utf-8")) leakage = json.loads((args.corpus_dir / "spatial-leakage-audit.json").read_text(encoding="utf-8")) samples = manifest["samples"] split_counts = Counter((sample["region"], sample["split"]) for sample in samples) decision_counts: Counter[str] = Counter() review_queue: list[dict[str, Any]] = [] total_input = 0 total_accepted = 0 temporal_unknown = 0 failures: list[str] = [] for region in REGIONS: for split in REQUIRED_SPLITS: minimum = 4 if split == "train" else 2 if split_counts[(region, split)] < minimum: failures.append(f"{region}/{split} has {split_counts[(region, split)]}, requires {minimum}") for sample in samples: audit_path = args.corpus_dir / "pairs" / sample["sample_slug"] / "label-audit.json" audit = json.loads(audit_path.read_text(encoding="utf-8")) total_input += int(audit["input_feature_count"]) total_accepted += int(audit["accepted_feature_count"]) decision_counts.update(audit["decision_counts"]) if audit.get("temporal_alignment_status") == "unknown": temporal_unknown += 1 expected_empty = bool(sample.get("sample_role") == "background_candidate" and sample.get("require_empty")) if expected_empty and audit["accepted_feature_count"] != 0: failures.append(f"{sample['sample_slug']} is not pure empty after normalization") priority = "high" if audit["accepted_feature_count"] >= 200 or sample["split"] in {"test", "background-test"} else "normal" review_queue.append( { "sample_slug": sample["sample_slug"], "region": sample["region"], "context": sample.get("context"), "split": sample["split"], "accepted_feature_count": audit["accepted_feature_count"], "temporal_alignment_status": audit.get("temporal_alignment_status"), "priority": priority, "decision": "pending_human_review", } ) if leakage.get("status") != "ok": failures.append("spatial leakage audit failed") reviewed = 0 review_complete = False if args.review_decisions and args.review_decisions.is_file(): decisions = json.loads(args.review_decisions.read_text(encoding="utf-8")) by_slug = {item["sample_slug"]: item for item in decisions.get("decisions", [])} for item in review_queue: decision = by_slug.get(item["sample_slug"]) if decision: item["decision"] = decision.get("decision") item["reviewer"] = decision.get("reviewer") item["notes"] = decision.get("notes") if item["decision"] in {"accepted", "rejected"} and item.get("reviewer"): reviewed += 1 review_complete = reviewed == len(review_queue) and all(item["decision"] == "accepted" for item in review_queue) status = "failed" if failures else ("ok" if review_complete else "needs_human_review") report = { "status": status, "dataset_version": manifest["dataset_version"], "manifest_immutable": manifest["immutable"], "sample_count": len(samples), "split_counts": {f"{region}/{split}": split_counts[(region, split)] for region in REGIONS for split in REQUIRED_SPLITS}, "input_feature_count": total_input, "accepted_feature_count": total_accepted, "decision_counts": dict(sorted(decision_counts.items())), "temporal_unknown_sample_count": temporal_unknown, "spatial_leakage_status": leakage.get("status"), "reviewed_sample_count": reviewed, "review_complete": review_complete, "failures": failures, "review_queue": review_queue, } args.output_dir.mkdir(parents=True, exist_ok=True) (args.output_dir / "belgium-building-corpus-audit.json").write_text( json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8" ) lines = [ f"# Belgian building corpus audit: {manifest['dataset_version']}", "", f"Status: `{status}`", f"Samples: {len(samples)}; accepted labels: {total_accepted}/{total_input}.", f"Spatial leakage: `{leakage.get('status')}`; human reviewed: {reviewed}/{len(samples)}.", "", "## Review queue", "", "| Sample | Region | Context | Split | Labels | Priority | Decision |", "| --- | --- | --- | --- | ---: | --- | --- |", ] lines.extend( f"| {item['sample_slug']} | {item['region']} | {item['context']} | {item['split']} | " f"{item['accepted_feature_count']} | {item['priority']} | {item['decision']} |" for item in review_queue ) (args.output_dir / "belgium-building-corpus-audit.md").write_text("\n".join(lines) + "\n", encoding="utf-8") print(json.dumps({key: value for key, value in report.items() if key != "review_queue"}, ensure_ascii=False, indent=2)) return 1 if failures else 0 if __name__ == "__main__": raise SystemExit(main())