Harden demo golden QA smoke
GeoIntel CI / docs-smoke (push) Has been cancelled
GeoIntel CI / contract-smoke (push) Has been cancelled

This commit is contained in:
Codex
2026-06-17 06:02:03 +02:00
parent a28e516e17
commit 0bb2e828b1
9 changed files with 165 additions and 20 deletions
+35 -5
View File
@@ -2,6 +2,7 @@
set -euo pipefail
BASE_URL="${1:-http://localhost:1202}"
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TMP_DIR="$(mktemp -d)"
trap 'rm -rf "${TMP_DIR}"' EXIT
@@ -146,22 +147,51 @@ if [ "${quality_count}" -lt 1 ]; then
echo "Expected at least one persisted QA/QC result after demo workflow" >&2
exit 1
fi
"${PYTHON_BIN}" - "${TMP_DIR}/quality_checks.json" "${quality_check_id}" <<'PY'
"${PYTHON_BIN}" - "${TMP_DIR}/quality_checks.json" "${quality_check_id}" "${ROOT}/fixtures/golden/expected_qa_metrics.json" <<'PY'
import json
import sys
path, quality_check_id = sys.argv[1], sys.argv[2]
path, quality_check_id, expected_path = sys.argv[1], sys.argv[2], sys.argv[3]
with open(path, "r", encoding="utf-8") as handle:
payload = json.load(handle)
with open(expected_path, "r", encoding="utf-8") as handle:
expected = json.load(handle)
items = payload["data"]["items"]
matches = [item for item in items if item["id"] == quality_check_id]
if not matches:
raise SystemExit("Seeded quality_check_id is not listed in project QA/QC results")
metric_keys = {metric["metric_key"] for metric in matches[0].get("metrics", [])}
required = {"precision", "recall", "f1", "mean_iou", "false_positive_count", "false_negative_count"}
missing = required - metric_keys
quality_check = matches[0]
if quality_check.get("status") != "ok":
raise SystemExit(f"Seeded QA/QC result has unexpected status: {quality_check.get('status')}")
if abs(float(quality_check.get("score")) - float(expected["f1"])) > float(expected["tolerance"]):
raise SystemExit("Seeded QA/QC score does not match the golden F1 baseline")
metrics = {metric["metric_key"]: metric.get("metric_value") for metric in quality_check.get("metrics", [])}
required = {
"precision": expected["precision"],
"recall": expected["recall"],
"f1": expected["f1"],
"mean_iou": expected["mean_iou"],
"false_positive_count": expected["false_positive_count"],
"false_negative_count": expected["false_negative_count"],
}
missing = set(required) - set(metrics)
if missing:
raise SystemExit(f"QA/QC result is missing metrics: {sorted(missing)}")
for key, expected_value in required.items():
actual = metrics[key]
if actual is None:
raise SystemExit(f"QA/QC metric {key} is None")
if abs(float(actual) - float(expected_value)) > float(expected["tolerance"]):
raise SystemExit(
f"QA/QC metric {key} drifted: actual={actual}, expected={expected_value}, tolerance={expected['tolerance']}"
)
findings = quality_check.get("findings_json") or {}
if int(findings.get("matches", -1)) != int(expected["matches"]):
raise SystemExit("Seeded QA/QC match count does not match golden baseline")
if int(findings.get("false_positives", -1)) != int(expected["false_positive_count"]):
raise SystemExit("Seeded QA/QC false-positive count does not match golden baseline")
if int(findings.get("false_negatives", -1)) != int(expected["false_negative_count"]):
raise SystemExit("Seeded QA/QC false-negative count does not match golden baseline")
PY
curl -fsS -X POST "${BASE_URL%/}/api/v1/exports/metadata" \