Harden demo golden QA smoke
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${1:-http://localhost:1202}"
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "${TMP_DIR}"' EXIT
|
||||
|
||||
@@ -146,22 +147,51 @@ if [ "${quality_count}" -lt 1 ]; then
|
||||
echo "Expected at least one persisted QA/QC result after demo workflow" >&2
|
||||
exit 1
|
||||
fi
|
||||
"${PYTHON_BIN}" - "${TMP_DIR}/quality_checks.json" "${quality_check_id}" <<'PY'
|
||||
"${PYTHON_BIN}" - "${TMP_DIR}/quality_checks.json" "${quality_check_id}" "${ROOT}/fixtures/golden/expected_qa_metrics.json" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
path, quality_check_id = sys.argv[1], sys.argv[2]
|
||||
path, quality_check_id, expected_path = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
with open(expected_path, "r", encoding="utf-8") as handle:
|
||||
expected = json.load(handle)
|
||||
items = payload["data"]["items"]
|
||||
matches = [item for item in items if item["id"] == quality_check_id]
|
||||
if not matches:
|
||||
raise SystemExit("Seeded quality_check_id is not listed in project QA/QC results")
|
||||
metric_keys = {metric["metric_key"] for metric in matches[0].get("metrics", [])}
|
||||
required = {"precision", "recall", "f1", "mean_iou", "false_positive_count", "false_negative_count"}
|
||||
missing = required - metric_keys
|
||||
quality_check = matches[0]
|
||||
if quality_check.get("status") != "ok":
|
||||
raise SystemExit(f"Seeded QA/QC result has unexpected status: {quality_check.get('status')}")
|
||||
if abs(float(quality_check.get("score")) - float(expected["f1"])) > float(expected["tolerance"]):
|
||||
raise SystemExit("Seeded QA/QC score does not match the golden F1 baseline")
|
||||
metrics = {metric["metric_key"]: metric.get("metric_value") for metric in quality_check.get("metrics", [])}
|
||||
required = {
|
||||
"precision": expected["precision"],
|
||||
"recall": expected["recall"],
|
||||
"f1": expected["f1"],
|
||||
"mean_iou": expected["mean_iou"],
|
||||
"false_positive_count": expected["false_positive_count"],
|
||||
"false_negative_count": expected["false_negative_count"],
|
||||
}
|
||||
missing = set(required) - set(metrics)
|
||||
if missing:
|
||||
raise SystemExit(f"QA/QC result is missing metrics: {sorted(missing)}")
|
||||
for key, expected_value in required.items():
|
||||
actual = metrics[key]
|
||||
if actual is None:
|
||||
raise SystemExit(f"QA/QC metric {key} is None")
|
||||
if abs(float(actual) - float(expected_value)) > float(expected["tolerance"]):
|
||||
raise SystemExit(
|
||||
f"QA/QC metric {key} drifted: actual={actual}, expected={expected_value}, tolerance={expected['tolerance']}"
|
||||
)
|
||||
findings = quality_check.get("findings_json") or {}
|
||||
if int(findings.get("matches", -1)) != int(expected["matches"]):
|
||||
raise SystemExit("Seeded QA/QC match count does not match golden baseline")
|
||||
if int(findings.get("false_positives", -1)) != int(expected["false_positive_count"]):
|
||||
raise SystemExit("Seeded QA/QC false-positive count does not match golden baseline")
|
||||
if int(findings.get("false_negatives", -1)) != int(expected["false_negative_count"]):
|
||||
raise SystemExit("Seeded QA/QC false-negative count does not match golden baseline")
|
||||
PY
|
||||
|
||||
curl -fsS -X POST "${BASE_URL%/}/api/v1/exports/metadata" \
|
||||
|
||||
Reference in New Issue
Block a user