122 lines
5.7 KiB
Python
122 lines
5.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Audit a frozen Belgian corpus and emit a deterministic review queue."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
REQUIRED_SPLITS = ("train", "val", "calibration", "test", "background-test")
|
|
REGIONS = ("flanders", "wallonia", "brussels")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--corpus-dir", type=Path, required=True)
|
|
parser.add_argument("--output-dir", type=Path, required=True)
|
|
parser.add_argument("--review-decisions", type=Path)
|
|
args = parser.parse_args()
|
|
manifest = json.loads((args.corpus_dir / "operator_samples_manifest.json").read_text(encoding="utf-8"))
|
|
leakage = json.loads((args.corpus_dir / "spatial-leakage-audit.json").read_text(encoding="utf-8"))
|
|
samples = manifest["samples"]
|
|
split_counts = Counter((sample["region"], sample["split"]) for sample in samples)
|
|
decision_counts: Counter[str] = Counter()
|
|
review_queue: list[dict[str, Any]] = []
|
|
total_input = 0
|
|
total_accepted = 0
|
|
temporal_unknown = 0
|
|
failures: list[str] = []
|
|
for region in REGIONS:
|
|
for split in REQUIRED_SPLITS:
|
|
minimum = 4 if split == "train" else 2
|
|
if split_counts[(region, split)] < minimum:
|
|
failures.append(f"{region}/{split} has {split_counts[(region, split)]}, requires {minimum}")
|
|
for sample in samples:
|
|
audit_path = args.corpus_dir / "pairs" / sample["sample_slug"] / "label-audit.json"
|
|
audit = json.loads(audit_path.read_text(encoding="utf-8"))
|
|
total_input += int(audit["input_feature_count"])
|
|
total_accepted += int(audit["accepted_feature_count"])
|
|
decision_counts.update(audit["decision_counts"])
|
|
if audit.get("temporal_alignment_status") == "unknown":
|
|
temporal_unknown += 1
|
|
expected_empty = bool(sample.get("sample_role") == "background_candidate" and sample.get("require_empty"))
|
|
if expected_empty and audit["accepted_feature_count"] != 0:
|
|
failures.append(f"{sample['sample_slug']} is not pure empty after normalization")
|
|
priority = "high" if audit["accepted_feature_count"] >= 200 or sample["split"] in {"test", "background-test"} else "normal"
|
|
review_queue.append(
|
|
{
|
|
"sample_slug": sample["sample_slug"],
|
|
"region": sample["region"],
|
|
"context": sample.get("context"),
|
|
"split": sample["split"],
|
|
"accepted_feature_count": audit["accepted_feature_count"],
|
|
"temporal_alignment_status": audit.get("temporal_alignment_status"),
|
|
"priority": priority,
|
|
"decision": "pending_human_review",
|
|
}
|
|
)
|
|
if leakage.get("status") != "ok":
|
|
failures.append("spatial leakage audit failed")
|
|
reviewed = 0
|
|
review_complete = False
|
|
if args.review_decisions and args.review_decisions.is_file():
|
|
decisions = json.loads(args.review_decisions.read_text(encoding="utf-8"))
|
|
by_slug = {item["sample_slug"]: item for item in decisions.get("decisions", [])}
|
|
for item in review_queue:
|
|
decision = by_slug.get(item["sample_slug"])
|
|
if decision:
|
|
item["decision"] = decision.get("decision")
|
|
item["reviewer"] = decision.get("reviewer")
|
|
item["notes"] = decision.get("notes")
|
|
if item["decision"] in {"accepted", "rejected"} and item.get("reviewer"):
|
|
reviewed += 1
|
|
review_complete = reviewed == len(review_queue) and all(item["decision"] == "accepted" for item in review_queue)
|
|
status = "failed" if failures else ("ok" if review_complete else "needs_human_review")
|
|
report = {
|
|
"status": status,
|
|
"dataset_version": manifest["dataset_version"],
|
|
"manifest_immutable": manifest["immutable"],
|
|
"sample_count": len(samples),
|
|
"split_counts": {f"{region}/{split}": split_counts[(region, split)] for region in REGIONS for split in REQUIRED_SPLITS},
|
|
"input_feature_count": total_input,
|
|
"accepted_feature_count": total_accepted,
|
|
"decision_counts": dict(sorted(decision_counts.items())),
|
|
"temporal_unknown_sample_count": temporal_unknown,
|
|
"spatial_leakage_status": leakage.get("status"),
|
|
"reviewed_sample_count": reviewed,
|
|
"review_complete": review_complete,
|
|
"failures": failures,
|
|
"review_queue": review_queue,
|
|
}
|
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
(args.output_dir / "belgium-building-corpus-audit.json").write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
)
|
|
lines = [
|
|
f"# Belgian building corpus audit: {manifest['dataset_version']}",
|
|
"",
|
|
f"Status: `{status}`",
|
|
f"Samples: {len(samples)}; accepted labels: {total_accepted}/{total_input}.",
|
|
f"Spatial leakage: `{leakage.get('status')}`; human reviewed: {reviewed}/{len(samples)}.",
|
|
"",
|
|
"## Review queue",
|
|
"",
|
|
"| Sample | Region | Context | Split | Labels | Priority | Decision |",
|
|
"| --- | --- | --- | --- | ---: | --- | --- |",
|
|
]
|
|
lines.extend(
|
|
f"| {item['sample_slug']} | {item['region']} | {item['context']} | {item['split']} | "
|
|
f"{item['accepted_feature_count']} | {item['priority']} | {item['decision']} |"
|
|
for item in review_queue
|
|
)
|
|
(args.output_dir / "belgium-building-corpus-audit.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
print(json.dumps({key: value for key, value in report.items() if key != "review_queue"}, ensure_ascii=False, indent=2))
|
|
return 1 if failures else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|