#!/usr/bin/env python3 """Offline structural, semantic and cross-file validation for the DevRunbook build pack.""" from __future__ import annotations import hashlib import json import os import re import subprocess import sys from pathlib import Path from typing import Any, Iterable import yaml from jsonschema import Draft202012Validator, FormatChecker ROOT = Path(__file__).resolve().parents[1] AUTONOMY = ["observe", "diagnose", "plan", "implement", "verify", "repair"] PUBLISHABLE_P0_COUNT = 28 CATALOG_COUNT = 72 GOLDEN_PROMPT_COUNT = 28 NORMATIVE_EXAMPLES = { "repository-health-audit", "root-cause-bugfix", "repository-cleanup", "gitea-best-practices", "feature-from-spec", "production-readiness-audit", } REQUIRED_DOCS = [ "README.md", "START_HERE_CODEX.md", "CODEX_EXECUTION_PROTOCOL.md", "AGENTS.md", "CODEX_MASTER_PROMPT.md", "IMPLEMENTATION_PLAN.md", "CURRENT_STATE.md", "DECISIONS.md", "PACK_REVIEW.md", "CHANGELOG.md", *[f"docs/{i:02d}-{name}.md" for i, name in [ (0,"product-vision"),(1,"product-requirements"),(2,"personas-and-jobs"),(3,"information-architecture"), (4,"ux-design-system"),(5,"domain-model"),(6,"technical-architecture"),(7,"playbook-package-spec"), (8,"prompt-composition-engine"),(9,"repository-intelligence"),(10,"gitea-integration"),(11,"codex-integration"), (12,"quality-evaluation"),(13,"security-privacy-threat-model"),(14,"api-contract"),(15,"test-strategy"), (16,"deployment-unraid"),(17,"observability-operations"),(18,"roadmap"),(19,"acceptance-criteria"), (20,"content-governance"),(21,"seed-catalog"),(22,"brand-copy"),(23,"future-expansion"),(24,"sources"), (25,"implementation-defaults"),(26,"authentication-authorization"),(27,"database-reference"), (28,"conditions-and-policy-dsl"),(29,"package-integrity-canonicalization"),(30,"screen-state-specification"), (31,"first-run-and-instance-lifecycle"),(32,"configuration-reference"),(33,"requirements-traceability"), (34,"risk-register"),(35,"glossary"),(36,"seed-content-delivery"),(37,"build-pack-tooling"), (38,"codex-native-build-workflow"),(39,"reference-composer-and-golden-fixtures"),(40,"bootstrap-repository-contract"), (41,"release-evidence-contract") ]], ] REQUIRED_OPENAPI_PATHS = { "/instance/status", "/instance/setup", "/auth/login", "/auth/logout", "/playbooks", "/playbooks/{slug}/versions/{version}", "/playbook-imports", "/repositories", "/repositories/{repositoryId}/profile", "/repositories/{repositoryId}/snapshots", "/compositions/preview", "/runs", "/runs/{runId}", "/runs/{runId}/artifacts", "/artifacts/{artifactId}/download", "/integrations/gitea", "/integrations/gitea/{integrationId}/test", "/jobs/{jobId}", "/audit-events", "/health/live", "/health/ready", } REQUIRED_SQL_TABLES = { "users", "auth_sessions", "workspaces", "workspace_memberships", "playbooks", "playbook_versions", "repositories", "repository_profile_revisions", "repository_snapshots", "repository_findings", "composition_drafts", "generated_runs", "generated_artifacts", "integrations", "integration_secrets", "evaluation_cases", "evaluation_results", "jobs", "audit_events", } REQUIRED_SUPPORT_FILES = { "BUILD_PACK.json", "LICENSE", "CONTRIBUTING.md", "SECURITY.md", "api/openapi.yaml", "database/reference-schema.sql", "config/env.example", "scripts/requirements-validate.txt", "scripts/build_archive.py", "scripts/verify_archive.py", "scripts/reference_compose.py", "schemas/rendered-prompt-manifest.schema.json", "schemas/release-evidence.schema.json", "templates/release-evidence.template.json", } DEVELOPMENT_EXCLUDED_PARTS = { ".git", ".next", ".turbo", ".venv", "__pycache__", "artifacts", "coverage", "dist", "node_modules", "playwright-report", "test-results", "volumes", } def is_development_tree() -> bool: """Return whether the extracted specification has been bootstrapped. Docker build contexts intentionally exclude `.git`, so Git metadata alone cannot distinguish the mutable implementation tree from the immutable v1.2 source archive. The workspace file is part of the mandatory bootstrap contract and did not exist in that archive. """ return (ROOT / ".git").is_dir() or (ROOT / "pnpm-workspace.yaml").is_file() def repository_files(pattern: str = "*") -> Iterable[Path]: """Yield source-controlled candidates without traversing implementation outputs. The archive integrity manifest is evidence for the extracted build pack. Once a Git repository exists, implementation files and generated dependency trees are expected and must not turn contract validation into a full-disk scan. """ for directory, child_directories, filenames in os.walk(ROOT): child_directories[:] = [ name for name in child_directories if name not in DEVELOPMENT_EXCLUDED_PARTS ] parent = Path(directory) for filename in filenames: path = parent / filename if path.match(pattern): yield path def load_json(path: Path) -> Any: return json.loads(path.read_text(encoding="utf-8")) def load_yaml(path: Path) -> Any: return yaml.safe_load(path.read_text(encoding="utf-8")) def canonical_json_bytes(value: Any) -> bytes: # Fixtures use only JSON values for which sorted compact JSON is RFC 8785-equivalent. return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") def add(errors: list[str], message: str) -> None: errors.append(message) def validate_schema(path: Path, errors: list[str]) -> dict[str, Any]: try: schema = load_json(path) Draft202012Validator.check_schema(schema) return schema except Exception as exc: # noqa: BLE001 add(errors, f"Invalid JSON Schema {path.relative_to(ROOT)}: {exc}") return {} def iter_conditions(value: Any) -> Iterable[dict[str, Any]]: if isinstance(value, dict): if set(value).intersection({"fact", "all", "any", "not"}): yield value for child in value.values(): yield from iter_conditions(child) elif isinstance(value, list): for child in value: yield from iter_conditions(child) def condition_depth(condition: dict[str, Any]) -> int: if "fact" in condition: return 1 if "not" in condition: return 1 + condition_depth(condition["not"]) key = "all" if "all" in condition else "any" return 1 + max(condition_depth(x) for x in condition[key]) def type_matches(value: Any, input_type: str) -> bool: if input_type in {"string", "multiline", "path", "command", "enum"}: return isinstance(value, str) if input_type == "boolean": return isinstance(value, bool) if input_type == "integer": return isinstance(value, int) and not isinstance(value, bool) if input_type in {"string-list", "multiselect"}: return isinstance(value, list) and all(isinstance(x, str) for x in value) if input_type == "key-value-list": return isinstance(value, list) and all(isinstance(x, dict) for x in value) return True def validate_playbook_tree(root: Path, schema: dict[str, Any], eval_schema: dict[str, Any], errors: list[str], label: str) -> dict[str, dict[str, Any]]: validator = Draft202012Validator(schema, format_checker=FormatChecker()) eval_validator = Draft202012Validator(eval_schema, format_checker=FormatChecker()) manifests = sorted(root.glob("*/playbook.yaml")) result: dict[str, dict[str, Any]] = {} template_pattern = re.compile(r"{{[#/]?\s*([a-zA-Z][a-zA-Z0-9_.]*)") for manifest_path in manifests: rel = manifest_path.relative_to(ROOT) try: data = load_yaml(manifest_path) except Exception as exc: # noqa: BLE001 add(errors, f"Cannot parse {rel}: {exc}") continue for issue in sorted(validator.iter_errors(data), key=lambda e: list(e.absolute_path)): pointer = ".".join(str(p) for p in issue.absolute_path) or "" add(errors, f"{rel}:{pointer}: {issue.message}") if not isinstance(data, dict) or not isinstance(data.get("metadata"), dict) or not isinstance(data.get("spec"), dict): continue meta, spec = data["metadata"], data["spec"] slug = meta.get("slug", "") if slug in result: add(errors, f"Duplicate {label} slug: {slug}") result[slug] = data # Semantic identity and autonomy. if spec.get("defaultMode") not in spec.get("modes", []): add(errors, f"{rel}: defaultMode is not present in modes") try: a = spec["autonomy"] lo, hi, default = AUTONOMY.index(a["min"]), AUTONOMY.index(a["max"]), AUTONOMY.index(a["default"]) if lo > hi or not lo <= default <= hi: add(errors, f"{rel}: invalid autonomy range/default") except Exception: pass # Unique IDs and input semantics. for field, key in [("inputs", "key"), ("guardrails", "id"), ("workflow", "id")]: vals = [x.get(key) for x in spec.get(field, []) if isinstance(x, dict)] if len(vals) != len(set(vals)): add(errors, f"{rel}: duplicate {field} identifiers") for field in [spec.get("validation", {}).get("checks", []), spec.get("reporting", {}).get("sections", [])]: vals = [x.get("id") for x in field if isinstance(x, dict)] if len(vals) != len(set(vals)): add(errors, f"{rel}: duplicate check/report identifiers") input_map = {x.get("key"): x for x in spec.get("inputs", []) if isinstance(x, dict)} for key, item in input_map.items(): typ = item.get("type") if item.get("sensitive") and item.get("includeInOutput"): add(errors, f"{rel}: sensitive input {key} cannot be included in output") if typ in {"enum", "multiselect"} and not item.get("options"): add(errors, f"{rel}: {typ} input {key} requires options") if "default" in item and not type_matches(item["default"], typ): add(errors, f"{rel}: default for {key} does not match {typ}") if typ == "enum" and "default" in item and item["default"] not in item.get("options", []): add(errors, f"{rel}: enum default for {key} is not in options") if typ == "multiselect" and "default" in item and not set(item["default"]).issubset(set(item.get("options", []))): add(errors, f"{rel}: multiselect default for {key} is not a subset of options") if item.get("minLength") is not None and item.get("maxLength") is not None and item["minLength"] > item["maxLength"]: add(errors, f"{rel}: input {key} minLength exceeds maxLength") # Conditions reference declared inputs and remain bounded. conditions = list(iter_conditions(data)) if len(conditions) > 100: add(errors, f"{rel}: more than 100 condition nodes") for condition in conditions: if condition_depth(condition) > 12: add(errors, f"{rel}: condition nesting exceeds 12") if "fact" in condition: fact = condition["fact"] path = fact.get("path", "") if path.startswith("inputs."): key = path.split(".", 2)[1] if key not in input_map: add(errors, f"{rel}: condition references undeclared input {key}") op = fact.get("operator") if op in {"exists", "truthy", "falsy"} and "value" in fact: add(errors, f"{rel}: operator {op} must not provide value") if op not in {"exists", "truthy", "falsy"} and "value" not in fact: add(errors, f"{rel}: operator {op} requires value") # Exact file inventory. folder = manifest_path.parent declared = data.get("package", {}).get("files", []) declared_paths = [x.get("path") for x in declared if isinstance(x, dict)] if len(declared_paths) != len(set(declared_paths)): add(errors, f"{rel}: duplicate package file path") actual_paths = sorted(str(x.relative_to(folder)).replace("\\", "/") for x in folder.rglob("*") if x.is_file() and x.name != "playbook.yaml") if sorted(declared_paths) != actual_paths: missing = sorted(set(declared_paths) - set(actual_paths)) undeclared = sorted(set(actual_paths) - set(declared_paths)) add(errors, f"{rel}: package inventory mismatch; missing={missing}, undeclared={undeclared}") for file_entry in declared: path = folder / file_entry.get("path", "") if not path.is_file() or path.is_symlink(): add(errors, f"{rel}: declared file is missing or not regular: {file_entry.get('path')}") main = spec.get("template", {}).get("main") roles = {x.get("path"): x.get("role") for x in declared} if roles.get(main) != "template": add(errors, f"{rel}: main template must be declared with role template") for partial in spec.get("template", {}).get("partials", []): if roles.get(partial) != "partial": add(errors, f"{rel}: partial {partial} lacks partial role") # Template references. template_path = folder / str(main or "") if template_path.is_file(): template = template_path.read_text(encoding="utf-8") for ref_name in template_pattern.findall(template): if ref_name.startswith("inputs."): key = ref_name.split(".", 1)[1] if key not in input_map: add(errors, f"{template_path.relative_to(ROOT)}: unknown input reference {key}") elif ref_name.split(".", 1)[0] not in {"repository", "platform", "autonomy", "inputs", "composition"}: add(errors, f"{template_path.relative_to(ROOT)}: disallowed template root {ref_name}") # Examples and evaluations. evaluation_ids: set[str] = set() for ep in sorted(folder.glob("evaluations/*.yaml")): try: ev = load_yaml(ep) for issue in eval_validator.iter_errors(ev): pointer = ".".join(str(p) for p in issue.absolute_path) or "" add(errors, f"{ep.relative_to(ROOT)}:{pointer}: {issue.message}") eid = ev.get("metadata", {}).get("id") if eid: evaluation_ids.add(eid) input_path = (ep.parent / ev.get("spec", {}).get("inputFile", "")).resolve() if not input_path.is_file() or folder.resolve() not in input_path.parents: add(errors, f"{ep.relative_to(ROOT)}: inputFile missing or escapes package") if ev.get("spec", {}).get("playbookVersion") != meta.get("version"): add(errors, f"{ep.relative_to(ROOT)}: evaluation version does not match package") except Exception as exc: # noqa: BLE001 add(errors, f"Cannot validate {ep.relative_to(ROOT)}: {exc}") claimed = set(data.get("quality", {}).get("evaluationCaseIds", [])) if claimed != evaluation_ids: add(errors, f"{rel}: quality evaluation IDs {sorted(claimed)} do not match files {sorted(evaluation_ids)}") if meta.get("lifecycle") in {"validated", "battle-tested"} and data.get("quality", {}).get("reviewStatus") != "evaluation-backed": add(errors, f"{rel}: validated lifecycle requires evaluation-backed review status") # Minimal example type validation. for example_path in sorted(folder.glob("examples/*.yaml")): ex = load_yaml(example_path) if ex.get("playbook", {}).get("slug") != slug or ex.get("playbook", {}).get("version") != meta.get("version"): add(errors, f"{example_path.relative_to(ROOT)}: playbook identity mismatch") supplied = ex.get("inputs", {}) for key, item in input_map.items(): if item.get("required") and key not in supplied: add(errors, f"{example_path.relative_to(ROOT)}: missing required input {key}") if key in supplied and not type_matches(supplied[key], item.get("type")): add(errors, f"{example_path.relative_to(ROOT)}: input {key} has wrong type") return result def validate_catalog(schema: dict[str, Any], content: dict[str, dict[str, Any]], errors: list[str]) -> dict[str, Any]: path = ROOT / "catalog/seed-catalog.yaml" data = load_yaml(path) validator = Draft202012Validator(schema, format_checker=FormatChecker()) for issue in validator.iter_errors(data): pointer = ".".join(str(p) for p in issue.absolute_path) or "" add(errors, f"{path.relative_to(ROOT)}:{pointer}: {issue.message}") entries = data.get("playbooks", []) if isinstance(data, dict) else [] if data.get("metadata", {}).get("count") != len(entries) or len(entries) != CATALOG_COUNT: add(errors, f"Seed catalog must contain and declare {CATALOG_COUNT} entries") p0 = [x for x in entries if x.get("priority") == "P0"] if len(p0) != PUBLISHABLE_P0_COUNT or data.get("metadata", {}).get("publishableCount") != PUBLISHABLE_P0_COUNT: add(errors, f"Seed catalog must contain and declare {PUBLISHABLE_P0_COUNT} P0 publishable entries") for field in ("id", "slug", "title"): vals = [x.get(field) for x in entries] if len(vals) != len(set(vals)): add(errors, f"Seed catalog contains duplicate {field}") by_slug = {x["slug"]: x for x in entries} for entry in p0: if entry.get("deliveryStatus") != "publishable-package": add(errors, f"P0 {entry['slug']} is not marked publishable-package") pkg = content.get(entry["slug"]) if not pkg: add(errors, f"P0 {entry['slug']} has no content package") continue meta, spec = pkg["metadata"], pkg["spec"] checks = { "id": (meta.get("id"), entry.get("id")), "title": (meta.get("title"), entry.get("title")), "category": (meta.get("category"), entry.get("category")), "riskTier": (meta.get("riskTier"), entry.get("riskTier")), "type": (spec.get("type"), entry.get("type")), "defaultMode": (spec.get("defaultMode"), entry.get("defaultMode")), "defaultAutonomy": (spec.get("autonomy", {}).get("default"), entry.get("defaultAutonomy")), } for field, (actual, expected) in checks.items(): if actual != expected: add(errors, f"P0 {entry['slug']} {field} mismatch: {actual!r} != {expected!r}") extra = set(content) - {x["slug"] for x in p0} if extra: add(errors, f"Runtime content includes non-P0 or unknown slugs: {sorted(extra)}") for entry in entries: expected = "publishable-package" if entry["priority"] == "P0" else "authored-backlog" if entry.get("deliveryStatus") != expected: add(errors, f"Catalog {entry['slug']} deliveryStatus must be {expected}") return by_slug def validate_examples_match(content: dict[str, dict[str, Any]], examples: dict[str, dict[str, Any]], errors: list[str]) -> None: if set(examples) != NORMATIVE_EXAMPLES: add(errors, f"Normative examples must be exactly {sorted(NORMATIVE_EXAMPLES)}") for slug, ex in examples.items(): pkg = content.get(slug) if not pkg: continue for relative in ["playbook.yaml", "prompt.md", "README.md", "CHANGELOG.md", "examples/minimal.yaml", "evaluations/static-structure.yaml"]: a = ROOT / "examples/playbooks" / slug / relative b = ROOT / "content/playbooks" / slug / relative if a.read_bytes() != b.read_bytes(): add(errors, f"Normative example {slug}/{relative} differs from runtime package") def validate_fixture_schemas(schemas: dict[str, dict[str, Any]], errors: list[str]) -> None: profile_path = ROOT / "examples/repository-profiles/example-profile.yaml" profile = load_yaml(profile_path) for issue in Draft202012Validator(schemas["repository-profile"], format_checker=FormatChecker()).iter_errors(profile): add(errors, f"{profile_path.relative_to(ROOT)}:{'.'.join(map(str, issue.path))}: {issue.message}") expected = profile.get("metadata", {}).get("contentDigest") raw = json.loads(json.dumps(profile)) raw.get("metadata", {}).pop("contentDigest", None) actual = hashlib.sha256(canonical_json_bytes(raw)).hexdigest() if expected != actual: add(errors, f"Example Repository Profile digest mismatch: {expected} != {actual}") config_path = ROOT / "examples/instance-config/example-config.yaml" config = load_yaml(config_path) for issue in Draft202012Validator(schemas["instance-config"], format_checker=FormatChecker()).iter_errors(config): add(errors, f"{config_path.relative_to(ROOT)}:{'.'.join(map(str, issue.path))}: {issue.message}") pack = ROOT / "examples/run-packs/root-cause-example" manifest_path = pack / "manifest.json" manifest = load_json(manifest_path) for issue in Draft202012Validator(schemas["run-pack"], format_checker=FormatChecker()).iter_errors(manifest): add(errors, f"{manifest_path.relative_to(ROOT)}:{'.'.join(map(str, issue.path))}: {issue.message}") declared = {x["path"]: x for x in manifest.get("files", [])} actual_files = {str(x.relative_to(pack)).replace("\\", "/") for x in pack.rglob("*") if x.is_file() and x.name != "manifest.json"} if set(declared) != actual_files: add(errors, f"Example Run Pack file set mismatch: declared={sorted(declared)}, actual={sorted(actual_files)}") for path, item in declared.items(): b = (pack / path).read_bytes() if len(b) != item.get("sizeBytes") or hashlib.sha256(b).hexdigest() != item.get("sha256"): add(errors, f"Example Run Pack file digest/size mismatch: {path}") raw_manifest = dict(manifest) expected_manifest_digest = raw_manifest.pop("manifestDigest", None) actual_manifest_digest = hashlib.sha256(canonical_json_bytes(raw_manifest)).hexdigest() if expected_manifest_digest != actual_manifest_digest: add(errors, f"Example Run Pack manifest digest mismatch: {expected_manifest_digest} != {actual_manifest_digest}") def validate_release_evidence_template(schema: dict[str, Any], errors: list[str]) -> None: path = ROOT / "templates/release-evidence.template.json" try: data = load_json(path) except Exception as exc: # noqa: BLE001 add(errors, f"Invalid release evidence template: {exc}") return validator = Draft202012Validator(schema, format_checker=FormatChecker()) for issue in validator.iter_errors(data): pointer = ".".join(str(p) for p in issue.absolute_path) or "" add(errors, f"{path.relative_to(ROOT)}:{pointer}: {issue.message}") trace = (ROOT / "docs/33-requirements-traceability.md").read_text(encoding="utf-8") expected_ids = re.findall(r"`((?:FR|NFR)-[A-Z]+-[0-9]{3})`", trace) actual_ids = [item.get("requirementId") for item in data.get("requirements", [])] if len(expected_ids) != 68 or len(set(expected_ids)) != 68: add(errors, f"Traceability document must contain exactly 68 unique requirement IDs, found {len(set(expected_ids))}") if actual_ids != expected_ids: add(errors, "Release evidence template requirement IDs/order differ from document 33") if data.get("summary", {}).get("blocked") != len(expected_ids): add(errors, "Release evidence template blocked summary must equal requirement count") if any(item.get("status") != "blocked" for item in data.get("requirements", [])): add(errors, "Unreleased release evidence template requirements must start blocked") def validate_golden_prompts(schema: dict[str, Any], content: dict[str, dict[str, Any]], errors: list[str]) -> None: root = ROOT / "examples/rendered-prompts" manifest_path = root / "manifest.json" if not manifest_path.is_file(): add(errors, "Missing examples/rendered-prompts/manifest.json") return try: manifest = load_json(manifest_path) except Exception as exc: # noqa: BLE001 add(errors, f"Invalid rendered prompt manifest: {exc}") return validator = Draft202012Validator(schema, format_checker=FormatChecker()) for issue in validator.iter_errors(manifest): pointer = ".".join(str(p) for p in issue.absolute_path) or "" add(errors, f"{manifest_path.relative_to(ROOT)}:{pointer}: {issue.message}") fixtures = manifest.get("fixtures", []) if manifest.get("count") != len(fixtures) or len(fixtures) != GOLDEN_PROMPT_COUNT: add(errors, f"Rendered prompt manifest must contain and declare {GOLDEN_PROMPT_COUNT} fixtures") slugs = [item.get("slug") for item in fixtures] if len(slugs) != len(set(slugs)): add(errors, "Rendered prompt manifest contains duplicate slugs") if set(slugs) != set(content): add(errors, f"Rendered prompt slugs differ from P0 content: missing={sorted(set(content)-set(slugs))}, extra={sorted(set(slugs)-set(content))}") headings = manifest.get("canonicalHeadings", []) required_headings = [ "Mission", "Repository context", "Required reconnaissance", "Scope", "Constraints and guardrails", "Autonomy and decision policy", "Execution workflow", "Validation plan", "Failure and recovery behavior", "Completion contract", "Final reporting format", ] if headings != required_headings: add(errors, "Rendered prompt canonical heading order mismatch") for item in fixtures: slug = item.get("slug") if not isinstance(slug, str): continue path = root / f"{slug}.md" if not path.is_file(): add(errors, f"Missing golden rendered prompt: {path.relative_to(ROOT)}") continue raw = path.read_bytes() if len(raw) != item.get("sizeBytes"): add(errors, f"Golden rendered prompt size mismatch: {slug}") if hashlib.sha256(raw).hexdigest() != item.get("sha256"): add(errors, f"Golden rendered prompt digest mismatch: {slug}") try: text = raw.decode("utf-8") except UnicodeDecodeError: add(errors, f"Golden rendered prompt is not UTF-8: {slug}") continue if "\r" in text: add(errors, f"Golden rendered prompt contains CR newline: {slug}") if "{{" in text or "}}" in text: add(errors, f"Golden rendered prompt contains unresolved template marker: {slug}") for heading in required_headings: if f"## {heading}\n" not in text: add(errors, f"Golden rendered prompt {slug} missing heading {heading}") pkg = content.get(slug) if pkg and item.get("playbookVersion") != pkg.get("metadata", {}).get("version"): add(errors, f"Golden rendered prompt version mismatch: {slug}") for ref_key in ("exampleFile", "repositoryProfileFile"): ref = item.get(ref_key) if ref is not None and not (ROOT / ref).is_file(): add(errors, f"Golden rendered prompt {slug} references missing {ref_key}: {ref}") process = subprocess.run( [sys.executable, str(ROOT / "scripts/reference_compose.py"), "--check"], cwd=ROOT, capture_output=True, text=True, check=False, ) if process.returncode != 0: detail = (process.stdout + process.stderr).strip() add(errors, f"Reference composer check failed: {detail}") def resolve_ref(doc: dict[str, Any], ref: str) -> bool: if not ref.startswith("#/"): return True value: Any = doc try: for token in ref[2:].split("/"): token = token.replace("~1", "/").replace("~0", "~") value = value[token] return True except Exception: return False def walk_refs(value: Any) -> Iterable[str]: if isinstance(value, dict): if isinstance(value.get("$ref"), str): yield value["$ref"] for child in value.values(): yield from walk_refs(child) elif isinstance(value, list): for child in value: yield from walk_refs(child) def validate_openapi(errors: list[str]) -> None: path = ROOT / "api/openapi.yaml" doc = load_yaml(path) if doc.get("openapi") != "3.1.0": add(errors, "OpenAPI must use 3.1.0") missing = REQUIRED_OPENAPI_PATHS - set(doc.get("paths", {})) if missing: add(errors, f"OpenAPI missing required paths: {sorted(missing)}") ids: list[str] = [] for operations in doc.get("paths", {}).values(): for method, operation in operations.items(): if method.lower() in {"get", "post", "put", "patch", "delete"}: oid = operation.get("operationId") if not oid: add(errors, "OpenAPI operation missing operationId") ids.append(oid) if len(ids) != len(set(ids)): add(errors, "OpenAPI operationIds are not unique") for ref in walk_refs(doc): if not resolve_ref(doc, ref): add(errors, f"OpenAPI unresolved local reference: {ref}") def validate_sql(errors: list[str]) -> None: path = ROOT / "database/reference-schema.sql" sql = path.read_text(encoding="utf-8") tables = set(re.findall(r"CREATE TABLE\s+([a-z_]+)", sql, flags=re.I)) missing = REQUIRED_SQL_TABLES - {x.lower() for x in tables} if missing: add(errors, f"Reference SQL missing tables: {sorted(missing)}") def validate_build_metadata(errors: list[str]) -> None: path = ROOT / "BUILD_PACK.json" try: data = load_json(path) except Exception as exc: # noqa: BLE001 add(errors, f"Invalid BUILD_PACK.json: {exc}") return expected = { "schemaVersion": 1, "version": "1.2.0", "status": "implementation-contract", "publishablePlaybookPackages": PUBLISHABLE_P0_COUNT, "normativeExamplePackages": len(NORMATIVE_EXAMPLES), "roadmapCatalogEntries": CATALOG_COUNT, "jsonSchemas": 9, "goldenRenderedPrompts": GOLDEN_PROMPT_COUNT, "releaseEvidenceRequirements": 68, "codexNativeExecutionProtocol": True, "minimumPythonVersion": "3.11", "canonicalValidator": "scripts/validate_pack.py", "canonicalArchiveBuilder": "scripts/build_archive.py", "license": "MIT", } for key, value in expected.items(): if data.get(key) != value: add(errors, f"BUILD_PACK.json {key} mismatch: {data.get(key)!r} != {value!r}") if data.get("releaseDate") != "2026-07-27": add(errors, "BUILD_PACK.json releaseDate must be 2026-07-27 for v1.2.0") def validate_integrity_files(errors: list[str]) -> None: index_path = ROOT / "FILE_INDEX.txt" manifest_path = ROOT / "PACK_MANIFEST.sha256" if not index_path.exists() and not manifest_path.exists(): return if not index_path.is_file() or not manifest_path.is_file(): add(errors, "FILE_INDEX.txt and PACK_MANIFEST.sha256 must either both exist or both be absent") return if is_development_tree(): # The original archive digest was verified before repository initialization. # Mutable state/docs and implementation files intentionally diverge afterward. return actual_files: set[str] = set() for path in repository_files(): if path.is_symlink(): add(errors, f"Symlink included in build pack: {path.relative_to(ROOT)}") elif path.is_file(): actual_files.add(path.relative_to(ROOT).as_posix()) index = {line for line in index_path.read_text(encoding="utf-8").splitlines() if line} if index != actual_files: add(errors, f"FILE_INDEX mismatch; missing={sorted(index - actual_files)}, extra={sorted(actual_files - index)}") manifest: dict[str, str] = {} for number, line in enumerate(manifest_path.read_text(encoding="utf-8").splitlines(), 1): if not line.strip(): continue try: digest, relative = line.split(" ", 1) except ValueError: add(errors, f"PACK_MANIFEST invalid line {number}") continue if not re.fullmatch(r"[0-9a-f]{64}", digest): add(errors, f"PACK_MANIFEST invalid SHA-256 at line {number}") if relative in manifest: add(errors, f"PACK_MANIFEST duplicate path {relative}") manifest[relative] = digest expected = actual_files - {"PACK_MANIFEST.sha256"} if set(manifest) != expected: add(errors, f"PACK_MANIFEST path mismatch; missing={sorted(expected - set(manifest))}, extra={sorted(set(manifest) - expected)}") for relative, expected_digest in manifest.items(): path = ROOT / relative if path.is_file(): actual = hashlib.sha256(path.read_bytes()).hexdigest() if actual != expected_digest: add(errors, f"PACK_MANIFEST checksum mismatch for {relative}") def validate_docs_and_secrets(errors: list[str]) -> None: for relative in sorted(REQUIRED_SUPPORT_FILES): path = ROOT / relative if not path.is_file() or path.is_symlink() or not path.read_bytes(): add(errors, f"Missing, empty or unsafe required support file: {relative}") for relative in REQUIRED_DOCS: path = ROOT / relative if not path.is_file() or not path.read_text(encoding="utf-8").strip(): add(errors, f"Missing or empty required document: {relative}") # Backtick references to package-local paths and top-level contracts. ref_pattern = re.compile(r"`((?:docs|schemas|templates|api|catalog|database|config|examples|scripts|adr|content)/[^`\n]+|(?:README|START_HERE_CODEX|CODEX_EXECUTION_PROTOCOL|AGENTS|CURRENT_STATE|DECISIONS|IMPLEMENTATION_PLAN|CODEX_MASTER_PROMPT|PACK_REVIEW|CHANGELOG)\.md)`") for md in repository_files("*.md"): text = md.read_text(encoding="utf-8") for ref in ref_pattern.findall(text): if any(ch in ref for ch in "*{}<>"): continue if not (ROOT / ref).exists(): add(errors, f"{md.relative_to(ROOT)} references missing path {ref}") forbidden_names = {".env", "id_rsa", "id_ed25519", "secrets.json"} private_key = re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----") for path in repository_files(): if path.name in forbidden_names: add(errors, f"Forbidden secret-like file included: {path.relative_to(ROOT)}") if path.stat().st_size <= 5_000_000: try: if private_key.search(path.read_text(encoding="utf-8")): add(errors, f"Private-key material found: {path.relative_to(ROOT)}") except UnicodeDecodeError: pass def main() -> int: errors: list[str] = [] schema_files = { "playbook": "playbook.schema.json", "condition": "condition.schema.json", "repository-profile": "repository-profile.schema.json", "run-pack": "run-pack-manifest.schema.json", "evaluation-case": "evaluation-case.schema.json", "seed-catalog": "seed-catalog.schema.json", "instance-config": "instance-config.schema.json", "rendered-prompt-manifest": "rendered-prompt-manifest.schema.json", "release-evidence": "release-evidence.schema.json", } schemas: dict[str, dict[str, Any]] = {} for key, filename in schema_files.items(): schemas[key] = validate_schema(ROOT / "schemas" / filename, errors) content = validate_playbook_tree(ROOT / "content/playbooks", schemas["playbook"], schemas["evaluation-case"], errors, "content") examples = validate_playbook_tree(ROOT / "examples/playbooks", schemas["playbook"], schemas["evaluation-case"], errors, "example") validate_catalog(schemas["seed-catalog"], content, errors) validate_examples_match(content, examples, errors) validate_fixture_schemas(schemas, errors) validate_release_evidence_template(schemas["release-evidence"], errors) validate_golden_prompts(schemas["rendered-prompt-manifest"], content, errors) validate_openapi(errors) validate_sql(errors) validate_build_metadata(errors) validate_integrity_files(errors) validate_docs_and_secrets(errors) if errors: print("DevRunbook build-pack validation FAILED\n") for error in errors: print(f"- {error}") return 1 print("DevRunbook build-pack validation PASSED") print(f"- Publishable P0 packages: {len(content)}") print(f"- Normative example packages: {len(examples)}") print(f"- Roadmap catalog entries: {CATALOG_COUNT}") print(f"- JSON Schemas: {len(schemas)} valid") print("- Cross-file identity, package inventory and evaluation references: valid") print("- Example profile, Run Pack, 28 golden prompts and 68-item release evidence template: valid") print("- OpenAPI, reference SQL, documents, support files and secret guards: valid") if is_development_tree(): print("- Embedded archive checksums: pre-initialization evidence retained; development tree excluded") else: print("- Embedded file index and checksums: valid when present") return 0 if __name__ == "__main__": sys.exit(main())