728 lines
37 KiB
Python
728 lines
37 KiB
Python
#!/usr/bin/env python3
|
|
"""Offline structural, semantic and cross-file validation for the DevRunbook build pack."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
|
|
import yaml
|
|
from jsonschema import Draft202012Validator, FormatChecker
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
AUTONOMY = ["observe", "diagnose", "plan", "implement", "verify", "repair"]
|
|
PUBLISHABLE_P0_COUNT = 28
|
|
CATALOG_COUNT = 72
|
|
GOLDEN_PROMPT_COUNT = 28
|
|
NORMATIVE_EXAMPLES = {
|
|
"repository-health-audit",
|
|
"root-cause-bugfix",
|
|
"repository-cleanup",
|
|
"gitea-best-practices",
|
|
"feature-from-spec",
|
|
"production-readiness-audit",
|
|
}
|
|
REQUIRED_DOCS = [
|
|
"README.md", "START_HERE_CODEX.md", "CODEX_EXECUTION_PROTOCOL.md", "AGENTS.md", "CODEX_MASTER_PROMPT.md", "IMPLEMENTATION_PLAN.md",
|
|
"CURRENT_STATE.md", "DECISIONS.md", "PACK_REVIEW.md", "CHANGELOG.md",
|
|
*[f"docs/{i:02d}-{name}.md" for i, name in [
|
|
(0,"product-vision"),(1,"product-requirements"),(2,"personas-and-jobs"),(3,"information-architecture"),
|
|
(4,"ux-design-system"),(5,"domain-model"),(6,"technical-architecture"),(7,"playbook-package-spec"),
|
|
(8,"prompt-composition-engine"),(9,"repository-intelligence"),(10,"gitea-integration"),(11,"codex-integration"),
|
|
(12,"quality-evaluation"),(13,"security-privacy-threat-model"),(14,"api-contract"),(15,"test-strategy"),
|
|
(16,"deployment-unraid"),(17,"observability-operations"),(18,"roadmap"),(19,"acceptance-criteria"),
|
|
(20,"content-governance"),(21,"seed-catalog"),(22,"brand-copy"),(23,"future-expansion"),(24,"sources"),
|
|
(25,"implementation-defaults"),(26,"authentication-authorization"),(27,"database-reference"),
|
|
(28,"conditions-and-policy-dsl"),(29,"package-integrity-canonicalization"),(30,"screen-state-specification"),
|
|
(31,"first-run-and-instance-lifecycle"),(32,"configuration-reference"),(33,"requirements-traceability"),
|
|
(34,"risk-register"),(35,"glossary"),(36,"seed-content-delivery"),(37,"build-pack-tooling"),
|
|
(38,"codex-native-build-workflow"),(39,"reference-composer-and-golden-fixtures"),(40,"bootstrap-repository-contract"),
|
|
(41,"release-evidence-contract")
|
|
]],
|
|
]
|
|
REQUIRED_OPENAPI_PATHS = {
|
|
"/instance/status", "/instance/setup", "/auth/login", "/auth/logout", "/playbooks",
|
|
"/playbooks/{slug}/versions/{version}", "/playbook-imports", "/repositories",
|
|
"/repositories/{repositoryId}/profile", "/repositories/{repositoryId}/snapshots",
|
|
"/compositions/preview", "/runs", "/runs/{runId}", "/runs/{runId}/artifacts",
|
|
"/artifacts/{artifactId}/download", "/integrations/gitea", "/integrations/gitea/{integrationId}/test",
|
|
"/jobs/{jobId}", "/audit-events", "/health/live", "/health/ready",
|
|
}
|
|
REQUIRED_SQL_TABLES = {
|
|
"users", "auth_sessions", "workspaces", "workspace_memberships", "playbooks", "playbook_versions",
|
|
"repositories", "repository_profile_revisions", "repository_snapshots", "repository_findings",
|
|
"composition_drafts", "generated_runs", "generated_artifacts", "integrations", "integration_secrets",
|
|
"evaluation_cases", "evaluation_results", "jobs", "audit_events",
|
|
}
|
|
REQUIRED_SUPPORT_FILES = {
|
|
"BUILD_PACK.json", "LICENSE", "CONTRIBUTING.md", "SECURITY.md", "api/openapi.yaml", "database/reference-schema.sql",
|
|
"config/env.example", "scripts/requirements-validate.txt", "scripts/build_archive.py", "scripts/verify_archive.py",
|
|
"scripts/reference_compose.py", "schemas/rendered-prompt-manifest.schema.json",
|
|
"schemas/release-evidence.schema.json", "templates/release-evidence.template.json",
|
|
}
|
|
|
|
DEVELOPMENT_EXCLUDED_PARTS = {
|
|
".git", ".next", ".turbo", ".venv", "__pycache__", "artifacts",
|
|
"coverage", "dist", "node_modules", "playwright-report", "test-results", "volumes",
|
|
}
|
|
|
|
|
|
def is_development_tree() -> bool:
|
|
"""Return whether the extracted specification has been bootstrapped.
|
|
|
|
Docker build contexts intentionally exclude `.git`, so Git metadata alone
|
|
cannot distinguish the mutable implementation tree from the immutable v1.2
|
|
source archive. The workspace file is part of the mandatory bootstrap
|
|
contract and did not exist in that archive.
|
|
"""
|
|
return (ROOT / ".git").is_dir() or (ROOT / "pnpm-workspace.yaml").is_file()
|
|
|
|
|
|
def repository_files(pattern: str = "*") -> Iterable[Path]:
|
|
"""Yield source-controlled candidates without traversing implementation outputs.
|
|
|
|
The archive integrity manifest is evidence for the extracted build pack. Once a
|
|
Git repository exists, implementation files and generated dependency trees are
|
|
expected and must not turn contract validation into a full-disk scan.
|
|
"""
|
|
for directory, child_directories, filenames in os.walk(ROOT):
|
|
child_directories[:] = [
|
|
name for name in child_directories if name not in DEVELOPMENT_EXCLUDED_PARTS
|
|
]
|
|
parent = Path(directory)
|
|
for filename in filenames:
|
|
path = parent / filename
|
|
if path.match(pattern):
|
|
yield path
|
|
|
|
|
|
def load_json(path: Path) -> Any:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def load_yaml(path: Path) -> Any:
|
|
return yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def canonical_json_bytes(value: Any) -> bytes:
|
|
# Fixtures use only JSON values for which sorted compact JSON is RFC 8785-equivalent.
|
|
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
|
|
|
|
def add(errors: list[str], message: str) -> None:
|
|
errors.append(message)
|
|
|
|
|
|
def validate_schema(path: Path, errors: list[str]) -> dict[str, Any]:
|
|
try:
|
|
schema = load_json(path)
|
|
Draft202012Validator.check_schema(schema)
|
|
return schema
|
|
except Exception as exc: # noqa: BLE001
|
|
add(errors, f"Invalid JSON Schema {path.relative_to(ROOT)}: {exc}")
|
|
return {}
|
|
|
|
|
|
def iter_conditions(value: Any) -> Iterable[dict[str, Any]]:
|
|
if isinstance(value, dict):
|
|
if set(value).intersection({"fact", "all", "any", "not"}):
|
|
yield value
|
|
for child in value.values():
|
|
yield from iter_conditions(child)
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
yield from iter_conditions(child)
|
|
|
|
|
|
def condition_depth(condition: dict[str, Any]) -> int:
|
|
if "fact" in condition:
|
|
return 1
|
|
if "not" in condition:
|
|
return 1 + condition_depth(condition["not"])
|
|
key = "all" if "all" in condition else "any"
|
|
return 1 + max(condition_depth(x) for x in condition[key])
|
|
|
|
|
|
def type_matches(value: Any, input_type: str) -> bool:
|
|
if input_type in {"string", "multiline", "path", "command", "enum"}:
|
|
return isinstance(value, str)
|
|
if input_type == "boolean":
|
|
return isinstance(value, bool)
|
|
if input_type == "integer":
|
|
return isinstance(value, int) and not isinstance(value, bool)
|
|
if input_type in {"string-list", "multiselect"}:
|
|
return isinstance(value, list) and all(isinstance(x, str) for x in value)
|
|
if input_type == "key-value-list":
|
|
return isinstance(value, list) and all(isinstance(x, dict) for x in value)
|
|
return True
|
|
|
|
|
|
def validate_playbook_tree(root: Path, schema: dict[str, Any], eval_schema: dict[str, Any], errors: list[str], label: str) -> dict[str, dict[str, Any]]:
|
|
validator = Draft202012Validator(schema, format_checker=FormatChecker())
|
|
eval_validator = Draft202012Validator(eval_schema, format_checker=FormatChecker())
|
|
manifests = sorted(root.glob("*/playbook.yaml"))
|
|
result: dict[str, dict[str, Any]] = {}
|
|
template_pattern = re.compile(r"{{[#/]?\s*([a-zA-Z][a-zA-Z0-9_.]*)")
|
|
|
|
for manifest_path in manifests:
|
|
rel = manifest_path.relative_to(ROOT)
|
|
try:
|
|
data = load_yaml(manifest_path)
|
|
except Exception as exc: # noqa: BLE001
|
|
add(errors, f"Cannot parse {rel}: {exc}")
|
|
continue
|
|
for issue in sorted(validator.iter_errors(data), key=lambda e: list(e.absolute_path)):
|
|
pointer = ".".join(str(p) for p in issue.absolute_path) or "<root>"
|
|
add(errors, f"{rel}:{pointer}: {issue.message}")
|
|
if not isinstance(data, dict) or not isinstance(data.get("metadata"), dict) or not isinstance(data.get("spec"), dict):
|
|
continue
|
|
meta, spec = data["metadata"], data["spec"]
|
|
slug = meta.get("slug", "")
|
|
if slug in result:
|
|
add(errors, f"Duplicate {label} slug: {slug}")
|
|
result[slug] = data
|
|
|
|
# Semantic identity and autonomy.
|
|
if spec.get("defaultMode") not in spec.get("modes", []):
|
|
add(errors, f"{rel}: defaultMode is not present in modes")
|
|
try:
|
|
a = spec["autonomy"]
|
|
lo, hi, default = AUTONOMY.index(a["min"]), AUTONOMY.index(a["max"]), AUTONOMY.index(a["default"])
|
|
if lo > hi or not lo <= default <= hi:
|
|
add(errors, f"{rel}: invalid autonomy range/default")
|
|
except Exception:
|
|
pass
|
|
|
|
# Unique IDs and input semantics.
|
|
for field, key in [("inputs", "key"), ("guardrails", "id"), ("workflow", "id")]:
|
|
vals = [x.get(key) for x in spec.get(field, []) if isinstance(x, dict)]
|
|
if len(vals) != len(set(vals)):
|
|
add(errors, f"{rel}: duplicate {field} identifiers")
|
|
for field in [spec.get("validation", {}).get("checks", []), spec.get("reporting", {}).get("sections", [])]:
|
|
vals = [x.get("id") for x in field if isinstance(x, dict)]
|
|
if len(vals) != len(set(vals)):
|
|
add(errors, f"{rel}: duplicate check/report identifiers")
|
|
input_map = {x.get("key"): x for x in spec.get("inputs", []) if isinstance(x, dict)}
|
|
for key, item in input_map.items():
|
|
typ = item.get("type")
|
|
if item.get("sensitive") and item.get("includeInOutput"):
|
|
add(errors, f"{rel}: sensitive input {key} cannot be included in output")
|
|
if typ in {"enum", "multiselect"} and not item.get("options"):
|
|
add(errors, f"{rel}: {typ} input {key} requires options")
|
|
if "default" in item and not type_matches(item["default"], typ):
|
|
add(errors, f"{rel}: default for {key} does not match {typ}")
|
|
if typ == "enum" and "default" in item and item["default"] not in item.get("options", []):
|
|
add(errors, f"{rel}: enum default for {key} is not in options")
|
|
if typ == "multiselect" and "default" in item and not set(item["default"]).issubset(set(item.get("options", []))):
|
|
add(errors, f"{rel}: multiselect default for {key} is not a subset of options")
|
|
if item.get("minLength") is not None and item.get("maxLength") is not None and item["minLength"] > item["maxLength"]:
|
|
add(errors, f"{rel}: input {key} minLength exceeds maxLength")
|
|
|
|
# Conditions reference declared inputs and remain bounded.
|
|
conditions = list(iter_conditions(data))
|
|
if len(conditions) > 100:
|
|
add(errors, f"{rel}: more than 100 condition nodes")
|
|
for condition in conditions:
|
|
if condition_depth(condition) > 12:
|
|
add(errors, f"{rel}: condition nesting exceeds 12")
|
|
if "fact" in condition:
|
|
fact = condition["fact"]
|
|
path = fact.get("path", "")
|
|
if path.startswith("inputs."):
|
|
key = path.split(".", 2)[1]
|
|
if key not in input_map:
|
|
add(errors, f"{rel}: condition references undeclared input {key}")
|
|
op = fact.get("operator")
|
|
if op in {"exists", "truthy", "falsy"} and "value" in fact:
|
|
add(errors, f"{rel}: operator {op} must not provide value")
|
|
if op not in {"exists", "truthy", "falsy"} and "value" not in fact:
|
|
add(errors, f"{rel}: operator {op} requires value")
|
|
|
|
# Exact file inventory.
|
|
folder = manifest_path.parent
|
|
declared = data.get("package", {}).get("files", [])
|
|
declared_paths = [x.get("path") for x in declared if isinstance(x, dict)]
|
|
if len(declared_paths) != len(set(declared_paths)):
|
|
add(errors, f"{rel}: duplicate package file path")
|
|
actual_paths = sorted(str(x.relative_to(folder)).replace("\\", "/") for x in folder.rglob("*") if x.is_file() and x.name != "playbook.yaml")
|
|
if sorted(declared_paths) != actual_paths:
|
|
missing = sorted(set(declared_paths) - set(actual_paths))
|
|
undeclared = sorted(set(actual_paths) - set(declared_paths))
|
|
add(errors, f"{rel}: package inventory mismatch; missing={missing}, undeclared={undeclared}")
|
|
for file_entry in declared:
|
|
path = folder / file_entry.get("path", "")
|
|
if not path.is_file() or path.is_symlink():
|
|
add(errors, f"{rel}: declared file is missing or not regular: {file_entry.get('path')}")
|
|
main = spec.get("template", {}).get("main")
|
|
roles = {x.get("path"): x.get("role") for x in declared}
|
|
if roles.get(main) != "template":
|
|
add(errors, f"{rel}: main template must be declared with role template")
|
|
for partial in spec.get("template", {}).get("partials", []):
|
|
if roles.get(partial) != "partial":
|
|
add(errors, f"{rel}: partial {partial} lacks partial role")
|
|
|
|
# Template references.
|
|
template_path = folder / str(main or "")
|
|
if template_path.is_file():
|
|
template = template_path.read_text(encoding="utf-8")
|
|
for ref_name in template_pattern.findall(template):
|
|
if ref_name.startswith("inputs."):
|
|
key = ref_name.split(".", 1)[1]
|
|
if key not in input_map:
|
|
add(errors, f"{template_path.relative_to(ROOT)}: unknown input reference {key}")
|
|
elif ref_name.split(".", 1)[0] not in {"repository", "platform", "autonomy", "inputs", "composition"}:
|
|
add(errors, f"{template_path.relative_to(ROOT)}: disallowed template root {ref_name}")
|
|
|
|
# Examples and evaluations.
|
|
evaluation_ids: set[str] = set()
|
|
for ep in sorted(folder.glob("evaluations/*.yaml")):
|
|
try:
|
|
ev = load_yaml(ep)
|
|
for issue in eval_validator.iter_errors(ev):
|
|
pointer = ".".join(str(p) for p in issue.absolute_path) or "<root>"
|
|
add(errors, f"{ep.relative_to(ROOT)}:{pointer}: {issue.message}")
|
|
eid = ev.get("metadata", {}).get("id")
|
|
if eid:
|
|
evaluation_ids.add(eid)
|
|
input_path = (ep.parent / ev.get("spec", {}).get("inputFile", "")).resolve()
|
|
if not input_path.is_file() or folder.resolve() not in input_path.parents:
|
|
add(errors, f"{ep.relative_to(ROOT)}: inputFile missing or escapes package")
|
|
if ev.get("spec", {}).get("playbookVersion") != meta.get("version"):
|
|
add(errors, f"{ep.relative_to(ROOT)}: evaluation version does not match package")
|
|
except Exception as exc: # noqa: BLE001
|
|
add(errors, f"Cannot validate {ep.relative_to(ROOT)}: {exc}")
|
|
claimed = set(data.get("quality", {}).get("evaluationCaseIds", []))
|
|
if claimed != evaluation_ids:
|
|
add(errors, f"{rel}: quality evaluation IDs {sorted(claimed)} do not match files {sorted(evaluation_ids)}")
|
|
if meta.get("lifecycle") in {"validated", "battle-tested"} and data.get("quality", {}).get("reviewStatus") != "evaluation-backed":
|
|
add(errors, f"{rel}: validated lifecycle requires evaluation-backed review status")
|
|
|
|
# Minimal example type validation.
|
|
for example_path in sorted(folder.glob("examples/*.yaml")):
|
|
ex = load_yaml(example_path)
|
|
if ex.get("playbook", {}).get("slug") != slug or ex.get("playbook", {}).get("version") != meta.get("version"):
|
|
add(errors, f"{example_path.relative_to(ROOT)}: playbook identity mismatch")
|
|
supplied = ex.get("inputs", {})
|
|
for key, item in input_map.items():
|
|
if item.get("required") and key not in supplied:
|
|
add(errors, f"{example_path.relative_to(ROOT)}: missing required input {key}")
|
|
if key in supplied and not type_matches(supplied[key], item.get("type")):
|
|
add(errors, f"{example_path.relative_to(ROOT)}: input {key} has wrong type")
|
|
return result
|
|
|
|
|
|
def validate_catalog(schema: dict[str, Any], content: dict[str, dict[str, Any]], errors: list[str]) -> dict[str, Any]:
|
|
path = ROOT / "catalog/seed-catalog.yaml"
|
|
data = load_yaml(path)
|
|
validator = Draft202012Validator(schema, format_checker=FormatChecker())
|
|
for issue in validator.iter_errors(data):
|
|
pointer = ".".join(str(p) for p in issue.absolute_path) or "<root>"
|
|
add(errors, f"{path.relative_to(ROOT)}:{pointer}: {issue.message}")
|
|
entries = data.get("playbooks", []) if isinstance(data, dict) else []
|
|
if data.get("metadata", {}).get("count") != len(entries) or len(entries) != CATALOG_COUNT:
|
|
add(errors, f"Seed catalog must contain and declare {CATALOG_COUNT} entries")
|
|
p0 = [x for x in entries if x.get("priority") == "P0"]
|
|
if len(p0) != PUBLISHABLE_P0_COUNT or data.get("metadata", {}).get("publishableCount") != PUBLISHABLE_P0_COUNT:
|
|
add(errors, f"Seed catalog must contain and declare {PUBLISHABLE_P0_COUNT} P0 publishable entries")
|
|
for field in ("id", "slug", "title"):
|
|
vals = [x.get(field) for x in entries]
|
|
if len(vals) != len(set(vals)):
|
|
add(errors, f"Seed catalog contains duplicate {field}")
|
|
by_slug = {x["slug"]: x for x in entries}
|
|
for entry in p0:
|
|
if entry.get("deliveryStatus") != "publishable-package":
|
|
add(errors, f"P0 {entry['slug']} is not marked publishable-package")
|
|
pkg = content.get(entry["slug"])
|
|
if not pkg:
|
|
add(errors, f"P0 {entry['slug']} has no content package")
|
|
continue
|
|
meta, spec = pkg["metadata"], pkg["spec"]
|
|
checks = {
|
|
"id": (meta.get("id"), entry.get("id")), "title": (meta.get("title"), entry.get("title")),
|
|
"category": (meta.get("category"), entry.get("category")), "riskTier": (meta.get("riskTier"), entry.get("riskTier")),
|
|
"type": (spec.get("type"), entry.get("type")), "defaultMode": (spec.get("defaultMode"), entry.get("defaultMode")),
|
|
"defaultAutonomy": (spec.get("autonomy", {}).get("default"), entry.get("defaultAutonomy")),
|
|
}
|
|
for field, (actual, expected) in checks.items():
|
|
if actual != expected:
|
|
add(errors, f"P0 {entry['slug']} {field} mismatch: {actual!r} != {expected!r}")
|
|
extra = set(content) - {x["slug"] for x in p0}
|
|
if extra:
|
|
add(errors, f"Runtime content includes non-P0 or unknown slugs: {sorted(extra)}")
|
|
for entry in entries:
|
|
expected = "publishable-package" if entry["priority"] == "P0" else "authored-backlog"
|
|
if entry.get("deliveryStatus") != expected:
|
|
add(errors, f"Catalog {entry['slug']} deliveryStatus must be {expected}")
|
|
return by_slug
|
|
|
|
|
|
def validate_examples_match(content: dict[str, dict[str, Any]], examples: dict[str, dict[str, Any]], errors: list[str]) -> None:
|
|
if set(examples) != NORMATIVE_EXAMPLES:
|
|
add(errors, f"Normative examples must be exactly {sorted(NORMATIVE_EXAMPLES)}")
|
|
for slug, ex in examples.items():
|
|
pkg = content.get(slug)
|
|
if not pkg:
|
|
continue
|
|
for relative in ["playbook.yaml", "prompt.md", "README.md", "CHANGELOG.md", "examples/minimal.yaml", "evaluations/static-structure.yaml"]:
|
|
a = ROOT / "examples/playbooks" / slug / relative
|
|
b = ROOT / "content/playbooks" / slug / relative
|
|
if a.read_bytes() != b.read_bytes():
|
|
add(errors, f"Normative example {slug}/{relative} differs from runtime package")
|
|
|
|
|
|
def validate_fixture_schemas(schemas: dict[str, dict[str, Any]], errors: list[str]) -> None:
|
|
profile_path = ROOT / "examples/repository-profiles/example-profile.yaml"
|
|
profile = load_yaml(profile_path)
|
|
for issue in Draft202012Validator(schemas["repository-profile"], format_checker=FormatChecker()).iter_errors(profile):
|
|
add(errors, f"{profile_path.relative_to(ROOT)}:{'.'.join(map(str, issue.path))}: {issue.message}")
|
|
expected = profile.get("metadata", {}).get("contentDigest")
|
|
raw = json.loads(json.dumps(profile))
|
|
raw.get("metadata", {}).pop("contentDigest", None)
|
|
actual = hashlib.sha256(canonical_json_bytes(raw)).hexdigest()
|
|
if expected != actual:
|
|
add(errors, f"Example Repository Profile digest mismatch: {expected} != {actual}")
|
|
|
|
config_path = ROOT / "examples/instance-config/example-config.yaml"
|
|
config = load_yaml(config_path)
|
|
for issue in Draft202012Validator(schemas["instance-config"], format_checker=FormatChecker()).iter_errors(config):
|
|
add(errors, f"{config_path.relative_to(ROOT)}:{'.'.join(map(str, issue.path))}: {issue.message}")
|
|
|
|
pack = ROOT / "examples/run-packs/root-cause-example"
|
|
manifest_path = pack / "manifest.json"
|
|
manifest = load_json(manifest_path)
|
|
for issue in Draft202012Validator(schemas["run-pack"], format_checker=FormatChecker()).iter_errors(manifest):
|
|
add(errors, f"{manifest_path.relative_to(ROOT)}:{'.'.join(map(str, issue.path))}: {issue.message}")
|
|
declared = {x["path"]: x for x in manifest.get("files", [])}
|
|
actual_files = {str(x.relative_to(pack)).replace("\\", "/") for x in pack.rglob("*") if x.is_file() and x.name != "manifest.json"}
|
|
if set(declared) != actual_files:
|
|
add(errors, f"Example Run Pack file set mismatch: declared={sorted(declared)}, actual={sorted(actual_files)}")
|
|
for path, item in declared.items():
|
|
b = (pack / path).read_bytes()
|
|
if len(b) != item.get("sizeBytes") or hashlib.sha256(b).hexdigest() != item.get("sha256"):
|
|
add(errors, f"Example Run Pack file digest/size mismatch: {path}")
|
|
raw_manifest = dict(manifest)
|
|
expected_manifest_digest = raw_manifest.pop("manifestDigest", None)
|
|
actual_manifest_digest = hashlib.sha256(canonical_json_bytes(raw_manifest)).hexdigest()
|
|
if expected_manifest_digest != actual_manifest_digest:
|
|
add(errors, f"Example Run Pack manifest digest mismatch: {expected_manifest_digest} != {actual_manifest_digest}")
|
|
|
|
|
|
def validate_release_evidence_template(schema: dict[str, Any], errors: list[str]) -> None:
|
|
path = ROOT / "templates/release-evidence.template.json"
|
|
try:
|
|
data = load_json(path)
|
|
except Exception as exc: # noqa: BLE001
|
|
add(errors, f"Invalid release evidence template: {exc}")
|
|
return
|
|
validator = Draft202012Validator(schema, format_checker=FormatChecker())
|
|
for issue in validator.iter_errors(data):
|
|
pointer = ".".join(str(p) for p in issue.absolute_path) or "<root>"
|
|
add(errors, f"{path.relative_to(ROOT)}:{pointer}: {issue.message}")
|
|
trace = (ROOT / "docs/33-requirements-traceability.md").read_text(encoding="utf-8")
|
|
expected_ids = re.findall(r"`((?:FR|NFR)-[A-Z]+-[0-9]{3})`", trace)
|
|
actual_ids = [item.get("requirementId") for item in data.get("requirements", [])]
|
|
if len(expected_ids) != 68 or len(set(expected_ids)) != 68:
|
|
add(errors, f"Traceability document must contain exactly 68 unique requirement IDs, found {len(set(expected_ids))}")
|
|
if actual_ids != expected_ids:
|
|
add(errors, "Release evidence template requirement IDs/order differ from document 33")
|
|
if data.get("summary", {}).get("blocked") != len(expected_ids):
|
|
add(errors, "Release evidence template blocked summary must equal requirement count")
|
|
if any(item.get("status") != "blocked" for item in data.get("requirements", [])):
|
|
add(errors, "Unreleased release evidence template requirements must start blocked")
|
|
|
|
|
|
def validate_golden_prompts(schema: dict[str, Any], content: dict[str, dict[str, Any]], errors: list[str]) -> None:
|
|
root = ROOT / "examples/rendered-prompts"
|
|
manifest_path = root / "manifest.json"
|
|
if not manifest_path.is_file():
|
|
add(errors, "Missing examples/rendered-prompts/manifest.json")
|
|
return
|
|
try:
|
|
manifest = load_json(manifest_path)
|
|
except Exception as exc: # noqa: BLE001
|
|
add(errors, f"Invalid rendered prompt manifest: {exc}")
|
|
return
|
|
validator = Draft202012Validator(schema, format_checker=FormatChecker())
|
|
for issue in validator.iter_errors(manifest):
|
|
pointer = ".".join(str(p) for p in issue.absolute_path) or "<root>"
|
|
add(errors, f"{manifest_path.relative_to(ROOT)}:{pointer}: {issue.message}")
|
|
fixtures = manifest.get("fixtures", [])
|
|
if manifest.get("count") != len(fixtures) or len(fixtures) != GOLDEN_PROMPT_COUNT:
|
|
add(errors, f"Rendered prompt manifest must contain and declare {GOLDEN_PROMPT_COUNT} fixtures")
|
|
slugs = [item.get("slug") for item in fixtures]
|
|
if len(slugs) != len(set(slugs)):
|
|
add(errors, "Rendered prompt manifest contains duplicate slugs")
|
|
if set(slugs) != set(content):
|
|
add(errors, f"Rendered prompt slugs differ from P0 content: missing={sorted(set(content)-set(slugs))}, extra={sorted(set(slugs)-set(content))}")
|
|
headings = manifest.get("canonicalHeadings", [])
|
|
required_headings = [
|
|
"Mission", "Repository context", "Required reconnaissance", "Scope",
|
|
"Constraints and guardrails", "Autonomy and decision policy", "Execution workflow",
|
|
"Validation plan", "Failure and recovery behavior", "Completion contract",
|
|
"Final reporting format",
|
|
]
|
|
if headings != required_headings:
|
|
add(errors, "Rendered prompt canonical heading order mismatch")
|
|
for item in fixtures:
|
|
slug = item.get("slug")
|
|
if not isinstance(slug, str):
|
|
continue
|
|
path = root / f"{slug}.md"
|
|
if not path.is_file():
|
|
add(errors, f"Missing golden rendered prompt: {path.relative_to(ROOT)}")
|
|
continue
|
|
raw = path.read_bytes()
|
|
if len(raw) != item.get("sizeBytes"):
|
|
add(errors, f"Golden rendered prompt size mismatch: {slug}")
|
|
if hashlib.sha256(raw).hexdigest() != item.get("sha256"):
|
|
add(errors, f"Golden rendered prompt digest mismatch: {slug}")
|
|
try:
|
|
text = raw.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
add(errors, f"Golden rendered prompt is not UTF-8: {slug}")
|
|
continue
|
|
if "\r" in text:
|
|
add(errors, f"Golden rendered prompt contains CR newline: {slug}")
|
|
if "{{" in text or "}}" in text:
|
|
add(errors, f"Golden rendered prompt contains unresolved template marker: {slug}")
|
|
for heading in required_headings:
|
|
if f"## {heading}\n" not in text:
|
|
add(errors, f"Golden rendered prompt {slug} missing heading {heading}")
|
|
pkg = content.get(slug)
|
|
if pkg and item.get("playbookVersion") != pkg.get("metadata", {}).get("version"):
|
|
add(errors, f"Golden rendered prompt version mismatch: {slug}")
|
|
for ref_key in ("exampleFile", "repositoryProfileFile"):
|
|
ref = item.get(ref_key)
|
|
if ref is not None and not (ROOT / ref).is_file():
|
|
add(errors, f"Golden rendered prompt {slug} references missing {ref_key}: {ref}")
|
|
process = subprocess.run(
|
|
[sys.executable, str(ROOT / "scripts/reference_compose.py"), "--check"],
|
|
cwd=ROOT, capture_output=True, text=True, check=False,
|
|
)
|
|
if process.returncode != 0:
|
|
detail = (process.stdout + process.stderr).strip()
|
|
add(errors, f"Reference composer check failed: {detail}")
|
|
|
|
|
|
def resolve_ref(doc: dict[str, Any], ref: str) -> bool:
|
|
if not ref.startswith("#/"):
|
|
return True
|
|
value: Any = doc
|
|
try:
|
|
for token in ref[2:].split("/"):
|
|
token = token.replace("~1", "/").replace("~0", "~")
|
|
value = value[token]
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def walk_refs(value: Any) -> Iterable[str]:
|
|
if isinstance(value, dict):
|
|
if isinstance(value.get("$ref"), str):
|
|
yield value["$ref"]
|
|
for child in value.values():
|
|
yield from walk_refs(child)
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
yield from walk_refs(child)
|
|
|
|
|
|
def validate_openapi(errors: list[str]) -> None:
|
|
path = ROOT / "api/openapi.yaml"
|
|
doc = load_yaml(path)
|
|
if doc.get("openapi") != "3.1.0":
|
|
add(errors, "OpenAPI must use 3.1.0")
|
|
missing = REQUIRED_OPENAPI_PATHS - set(doc.get("paths", {}))
|
|
if missing:
|
|
add(errors, f"OpenAPI missing required paths: {sorted(missing)}")
|
|
ids: list[str] = []
|
|
for operations in doc.get("paths", {}).values():
|
|
for method, operation in operations.items():
|
|
if method.lower() in {"get", "post", "put", "patch", "delete"}:
|
|
oid = operation.get("operationId")
|
|
if not oid:
|
|
add(errors, "OpenAPI operation missing operationId")
|
|
ids.append(oid)
|
|
if len(ids) != len(set(ids)):
|
|
add(errors, "OpenAPI operationIds are not unique")
|
|
for ref in walk_refs(doc):
|
|
if not resolve_ref(doc, ref):
|
|
add(errors, f"OpenAPI unresolved local reference: {ref}")
|
|
|
|
|
|
def validate_sql(errors: list[str]) -> None:
|
|
path = ROOT / "database/reference-schema.sql"
|
|
sql = path.read_text(encoding="utf-8")
|
|
tables = set(re.findall(r"CREATE TABLE\s+([a-z_]+)", sql, flags=re.I))
|
|
missing = REQUIRED_SQL_TABLES - {x.lower() for x in tables}
|
|
if missing:
|
|
add(errors, f"Reference SQL missing tables: {sorted(missing)}")
|
|
|
|
|
|
|
|
def validate_build_metadata(errors: list[str]) -> None:
|
|
path = ROOT / "BUILD_PACK.json"
|
|
try:
|
|
data = load_json(path)
|
|
except Exception as exc: # noqa: BLE001
|
|
add(errors, f"Invalid BUILD_PACK.json: {exc}")
|
|
return
|
|
expected = {
|
|
"schemaVersion": 1,
|
|
"version": "1.2.0",
|
|
"status": "implementation-contract",
|
|
"publishablePlaybookPackages": PUBLISHABLE_P0_COUNT,
|
|
"normativeExamplePackages": len(NORMATIVE_EXAMPLES),
|
|
"roadmapCatalogEntries": CATALOG_COUNT,
|
|
"jsonSchemas": 9,
|
|
"goldenRenderedPrompts": GOLDEN_PROMPT_COUNT,
|
|
"releaseEvidenceRequirements": 68,
|
|
"codexNativeExecutionProtocol": True,
|
|
"minimumPythonVersion": "3.11",
|
|
"canonicalValidator": "scripts/validate_pack.py",
|
|
"canonicalArchiveBuilder": "scripts/build_archive.py",
|
|
"license": "MIT",
|
|
}
|
|
for key, value in expected.items():
|
|
if data.get(key) != value:
|
|
add(errors, f"BUILD_PACK.json {key} mismatch: {data.get(key)!r} != {value!r}")
|
|
if data.get("releaseDate") != "2026-07-27":
|
|
add(errors, "BUILD_PACK.json releaseDate must be 2026-07-27 for v1.2.0")
|
|
|
|
|
|
def validate_integrity_files(errors: list[str]) -> None:
|
|
index_path = ROOT / "FILE_INDEX.txt"
|
|
manifest_path = ROOT / "PACK_MANIFEST.sha256"
|
|
if not index_path.exists() and not manifest_path.exists():
|
|
return
|
|
if not index_path.is_file() or not manifest_path.is_file():
|
|
add(errors, "FILE_INDEX.txt and PACK_MANIFEST.sha256 must either both exist or both be absent")
|
|
return
|
|
if is_development_tree():
|
|
# The original archive digest was verified before repository initialization.
|
|
# Mutable state/docs and implementation files intentionally diverge afterward.
|
|
return
|
|
actual_files: set[str] = set()
|
|
for path in repository_files():
|
|
if path.is_symlink():
|
|
add(errors, f"Symlink included in build pack: {path.relative_to(ROOT)}")
|
|
elif path.is_file():
|
|
actual_files.add(path.relative_to(ROOT).as_posix())
|
|
index = {line for line in index_path.read_text(encoding="utf-8").splitlines() if line}
|
|
if index != actual_files:
|
|
add(errors, f"FILE_INDEX mismatch; missing={sorted(index - actual_files)}, extra={sorted(actual_files - index)}")
|
|
manifest: dict[str, str] = {}
|
|
for number, line in enumerate(manifest_path.read_text(encoding="utf-8").splitlines(), 1):
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
digest, relative = line.split(" ", 1)
|
|
except ValueError:
|
|
add(errors, f"PACK_MANIFEST invalid line {number}")
|
|
continue
|
|
if not re.fullmatch(r"[0-9a-f]{64}", digest):
|
|
add(errors, f"PACK_MANIFEST invalid SHA-256 at line {number}")
|
|
if relative in manifest:
|
|
add(errors, f"PACK_MANIFEST duplicate path {relative}")
|
|
manifest[relative] = digest
|
|
expected = actual_files - {"PACK_MANIFEST.sha256"}
|
|
if set(manifest) != expected:
|
|
add(errors, f"PACK_MANIFEST path mismatch; missing={sorted(expected - set(manifest))}, extra={sorted(set(manifest) - expected)}")
|
|
for relative, expected_digest in manifest.items():
|
|
path = ROOT / relative
|
|
if path.is_file():
|
|
actual = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
if actual != expected_digest:
|
|
add(errors, f"PACK_MANIFEST checksum mismatch for {relative}")
|
|
|
|
|
|
def validate_docs_and_secrets(errors: list[str]) -> None:
|
|
for relative in sorted(REQUIRED_SUPPORT_FILES):
|
|
path = ROOT / relative
|
|
if not path.is_file() or path.is_symlink() or not path.read_bytes():
|
|
add(errors, f"Missing, empty or unsafe required support file: {relative}")
|
|
for relative in REQUIRED_DOCS:
|
|
path = ROOT / relative
|
|
if not path.is_file() or not path.read_text(encoding="utf-8").strip():
|
|
add(errors, f"Missing or empty required document: {relative}")
|
|
# Backtick references to package-local paths and top-level contracts.
|
|
ref_pattern = re.compile(r"`((?:docs|schemas|templates|api|catalog|database|config|examples|scripts|adr|content)/[^`\n]+|(?:README|START_HERE_CODEX|CODEX_EXECUTION_PROTOCOL|AGENTS|CURRENT_STATE|DECISIONS|IMPLEMENTATION_PLAN|CODEX_MASTER_PROMPT|PACK_REVIEW|CHANGELOG)\.md)`")
|
|
for md in repository_files("*.md"):
|
|
text = md.read_text(encoding="utf-8")
|
|
for ref in ref_pattern.findall(text):
|
|
if any(ch in ref for ch in "*{}<>"):
|
|
continue
|
|
if not (ROOT / ref).exists():
|
|
add(errors, f"{md.relative_to(ROOT)} references missing path {ref}")
|
|
forbidden_names = {".env", "id_rsa", "id_ed25519", "secrets.json"}
|
|
private_key = re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")
|
|
for path in repository_files():
|
|
if path.name in forbidden_names:
|
|
add(errors, f"Forbidden secret-like file included: {path.relative_to(ROOT)}")
|
|
if path.stat().st_size <= 5_000_000:
|
|
try:
|
|
if private_key.search(path.read_text(encoding="utf-8")):
|
|
add(errors, f"Private-key material found: {path.relative_to(ROOT)}")
|
|
except UnicodeDecodeError:
|
|
pass
|
|
|
|
|
|
def main() -> int:
|
|
errors: list[str] = []
|
|
schema_files = {
|
|
"playbook": "playbook.schema.json",
|
|
"condition": "condition.schema.json",
|
|
"repository-profile": "repository-profile.schema.json",
|
|
"run-pack": "run-pack-manifest.schema.json",
|
|
"evaluation-case": "evaluation-case.schema.json",
|
|
"seed-catalog": "seed-catalog.schema.json",
|
|
"instance-config": "instance-config.schema.json",
|
|
"rendered-prompt-manifest": "rendered-prompt-manifest.schema.json",
|
|
"release-evidence": "release-evidence.schema.json",
|
|
}
|
|
schemas: dict[str, dict[str, Any]] = {}
|
|
for key, filename in schema_files.items():
|
|
schemas[key] = validate_schema(ROOT / "schemas" / filename, errors)
|
|
content = validate_playbook_tree(ROOT / "content/playbooks", schemas["playbook"], schemas["evaluation-case"], errors, "content")
|
|
examples = validate_playbook_tree(ROOT / "examples/playbooks", schemas["playbook"], schemas["evaluation-case"], errors, "example")
|
|
validate_catalog(schemas["seed-catalog"], content, errors)
|
|
validate_examples_match(content, examples, errors)
|
|
validate_fixture_schemas(schemas, errors)
|
|
validate_release_evidence_template(schemas["release-evidence"], errors)
|
|
validate_golden_prompts(schemas["rendered-prompt-manifest"], content, errors)
|
|
validate_openapi(errors)
|
|
validate_sql(errors)
|
|
validate_build_metadata(errors)
|
|
validate_integrity_files(errors)
|
|
validate_docs_and_secrets(errors)
|
|
|
|
if errors:
|
|
print("DevRunbook build-pack validation FAILED\n")
|
|
for error in errors:
|
|
print(f"- {error}")
|
|
return 1
|
|
|
|
print("DevRunbook build-pack validation PASSED")
|
|
print(f"- Publishable P0 packages: {len(content)}")
|
|
print(f"- Normative example packages: {len(examples)}")
|
|
print(f"- Roadmap catalog entries: {CATALOG_COUNT}")
|
|
print(f"- JSON Schemas: {len(schemas)} valid")
|
|
print("- Cross-file identity, package inventory and evaluation references: valid")
|
|
print("- Example profile, Run Pack, 28 golden prompts and 68-item release evidence template: valid")
|
|
print("- OpenAPI, reference SQL, documents, support files and secret guards: valid")
|
|
if is_development_tree():
|
|
print("- Embedded archive checksums: pre-initialization evidence retained; development tree excluded")
|
|
else:
|
|
print("- Embedded file index and checksums: valid when present")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|