106 lines
4.1 KiB
JSON
106 lines
4.1 KiB
JSON
{
|
|
"benchmark_id": "geointel-p4-reference-harness-v2",
|
|
"claim_boundary": "Synthetic deterministic reference cases validate evaluator behavior, not production model accuracy.",
|
|
"code": [
|
|
{
|
|
"path": "scripts/accuracy_phase4_evaluator.py",
|
|
"sha256": "d2e40f7978d3ff4ce7025eaaaefb06ccc76c269d6874f8b0f528739d535a1da1",
|
|
"size_bytes": 93353
|
|
},
|
|
{
|
|
"path": "scripts/generate_accuracy_phase4_splits.py",
|
|
"sha256": "d12842922bcd99b1672176df15a9cef250128e7d7fc662ab15e16f1c084ba570",
|
|
"size_bytes": 75923
|
|
},
|
|
{
|
|
"path": "scripts/run_accuracy_phase4_benchmark.py",
|
|
"sha256": "003ecb4babc7f56a6d13e9e1c08dda902b9b26203cc7d53476d4d5e083b8b85b",
|
|
"size_bytes": 123717
|
|
},
|
|
{
|
|
"path": "scripts/run_golden_qa_benchmark.py",
|
|
"sha256": "2c8a19fd92db8f07f491b8f7b34d9736384f81e02a6b83019fac71b101ebdaa0",
|
|
"size_bytes": 10856
|
|
},
|
|
{
|
|
"path": "backend/app/services/qa_service.py",
|
|
"sha256": "2d4f645410fbf2972e55dddbf5a85db5368b2b1932792cd522ec51c483339244",
|
|
"size_bytes": 15206
|
|
}
|
|
],
|
|
"evaluation_results_canonical_json_sha256": "55cdba9cce4dcdc88cee0829cd510d7e624a3472feb9d33f7e681621a792eba3",
|
|
"evaluator_version": "2.1.0",
|
|
"evidence_run_id": "p4-2.0.1-9677d0ef37db82bcf39b",
|
|
"inference_and_selection": {
|
|
"background_test_used_for_selection": false,
|
|
"challenge_labels_available": false,
|
|
"production_model_inference_executed": false,
|
|
"raw_predictions_retained": true,
|
|
"synthetic_reference_harness": true,
|
|
"test_used_for_selection": false,
|
|
"threshold_source": "pre_registered_configuration_only"
|
|
},
|
|
"input_manifest": {
|
|
"path": "input-manifest.json",
|
|
"sha256": "44f1c81d76baf4dc965afc2cd109ac165ae8fe0308f202018ea21bac786bcd0b"
|
|
},
|
|
"inputs": {
|
|
"golden_qa_manifest": {
|
|
"path": "fixtures/golden/golden_qa_benchmarks.json",
|
|
"sha256": "d60576ff7048ca3a048dc166e8977f3a31e9f8fa352be5ad222a017260e179e2",
|
|
"size_bytes": 2758
|
|
},
|
|
"phase3_full_scan": {
|
|
"path": "artifacts/evidence/accuracy/P3/full-scan-manifest.json",
|
|
"sha256": "28c1022d125f30bfb4aa362b57d8a90cf8c88270b2a9ac6582b86b6a44a08038",
|
|
"size_bytes": 755527
|
|
},
|
|
"phase3_leakage": {
|
|
"path": "artifacts/evidence/accuracy/P3/leakage-report.json",
|
|
"sha256": "8e1b7a24a840549d442bc03132ee1f21c6d50f8714cbb8d33430391a3769cd4e",
|
|
"size_bytes": 580
|
|
},
|
|
"protected_cases": {
|
|
"path": "fixtures/accuracy/p4/protected-baseline-cases.json",
|
|
"sha256": "c49a9c8d74cae2062552b012d9c275f65156c81d75b3e453860f5b859c1c5473",
|
|
"size_bytes": 20091
|
|
},
|
|
"split_source": {
|
|
"path": "fixtures/accuracy/p4/split-source-manifest.json",
|
|
"sha256": "27ec60406ac57a02f1335013ee344a43d4e0093e7ee1329ec22876da623f5bfc",
|
|
"size_bytes": 17331
|
|
}
|
|
},
|
|
"manifest_sha256": "0ea5ab07f46c509a7a24943b31d5e9bfd6368920609227bc47613e18e52c4642",
|
|
"product_baseline": {
|
|
"artifacts": [],
|
|
"manifest": null,
|
|
"product_gate_evidence_sha256": "c9f5e3de3ab6d64911f8807b26caa661ccf911a5fd6b995107edda2004836e0c",
|
|
"requested_path": "artifacts/evidence/accuracy/P4/product-baseline-manifest.json",
|
|
"validation_status": "not_evaluable"
|
|
},
|
|
"product_gate_evidence_sha256": "c9f5e3de3ab6d64911f8807b26caa661ccf911a5fd6b995107edda2004836e0c",
|
|
"reference_baseline_sha256": "aeb0f6510352f4e205e4fc4e8c423110ca72c2142016c7af846d758865f258c6",
|
|
"repository_commit": "70fb4b94e5cb7c248beec5a936ce186f38cc183c",
|
|
"runtime": {
|
|
"cuda_used_for_reference_harness": false,
|
|
"dependencies": {
|
|
"numpy": "2.2.2",
|
|
"pyproj": "3.7.2",
|
|
"shapely": "2.1.2"
|
|
},
|
|
"execution_device": "CPU deterministic evaluator arithmetic; no production model inference",
|
|
"platform": "Windows-11-10.0.26200-SP0",
|
|
"python": "3.13.2",
|
|
"python_implementation": "CPython"
|
|
},
|
|
"schema_version": 2,
|
|
"split_generator_version": "1.3.0",
|
|
"split_manifests": {
|
|
"development_sha256": "64e4fc986c2390bedfbe100062d4b06e1098689f4d4a9a9da13a7f67357c0493",
|
|
"leakage_status": "pass",
|
|
"protected_sha256": "e57ca1529654ef1c2527e84122ff26ec3533e7c70658250983968fbc5d4af410"
|
|
},
|
|
"workflow_version": "2.0.1"
|
|
}
|