|
|
|
@@ -0,0 +1,585 @@
|
|
|
|
|
import {
|
|
|
|
|
cp,
|
|
|
|
|
link,
|
|
|
|
|
mkdir,
|
|
|
|
|
mkdtemp,
|
|
|
|
|
readFile,
|
|
|
|
|
rm,
|
|
|
|
|
writeFile,
|
|
|
|
|
} from 'node:fs/promises'
|
|
|
|
|
import { tmpdir } from 'node:os'
|
|
|
|
|
import path from 'node:path'
|
|
|
|
|
import { afterEach, describe, expect, it } from 'vitest'
|
|
|
|
|
import {
|
|
|
|
|
builtInPlaybookCount,
|
|
|
|
|
ContentValidationError,
|
|
|
|
|
defaultBuiltInContentRoot,
|
|
|
|
|
defaultSeedCatalogPath,
|
|
|
|
|
loadBuiltInPlaybookRecords,
|
|
|
|
|
loadBuiltInPlaybooks,
|
|
|
|
|
loadPlaybookPackage,
|
|
|
|
|
playbookPackageValidationLimits,
|
|
|
|
|
type PlaybookPackageFileRecord,
|
|
|
|
|
validatePlaybookPackageFiles,
|
|
|
|
|
validatePlaybookPackageArchiveFiles,
|
|
|
|
|
} from './index'
|
|
|
|
|
|
|
|
|
|
const temporaryRoots: string[] = []
|
|
|
|
|
const exhaustiveCatalogTimeout = 60_000
|
|
|
|
|
|
|
|
|
|
async function temporaryDirectory(label: string): Promise<string> {
|
|
|
|
|
const directory = await mkdtemp(path.join(tmpdir(), `devrunbook-${label}-`))
|
|
|
|
|
temporaryRoots.push(directory)
|
|
|
|
|
return directory
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function copyPackage(slug = 'root-cause-bugfix'): Promise<string> {
|
|
|
|
|
const root = await temporaryDirectory(slug)
|
|
|
|
|
const target = path.join(root, slug)
|
|
|
|
|
await cp(path.join(defaultBuiltInContentRoot, slug), target, {
|
|
|
|
|
recursive: true,
|
|
|
|
|
})
|
|
|
|
|
return target
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function packageFileRecords(
|
|
|
|
|
packageRoot: string,
|
|
|
|
|
): Promise<PlaybookPackageFileRecord[]> {
|
|
|
|
|
const loaded = await loadPlaybookPackage(packageRoot)
|
|
|
|
|
const declaredFiles = await Promise.all(
|
|
|
|
|
loaded.files.map(async (file) => ({
|
|
|
|
|
path: file.path,
|
|
|
|
|
role: file.role,
|
|
|
|
|
content: await readFile(path.join(packageRoot, ...file.path.split('/'))),
|
|
|
|
|
})),
|
|
|
|
|
)
|
|
|
|
|
return [
|
|
|
|
|
{
|
|
|
|
|
path: 'playbook.yaml',
|
|
|
|
|
role: 'manifest',
|
|
|
|
|
content: await readFile(path.join(packageRoot, 'playbook.yaml')),
|
|
|
|
|
},
|
|
|
|
|
...declaredFiles,
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
afterEach(async () => {
|
|
|
|
|
await Promise.all(
|
|
|
|
|
temporaryRoots
|
|
|
|
|
.splice(0)
|
|
|
|
|
.map((root) => rm(root, { recursive: true, force: true })),
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
describe('built-in playbook persistence records', () => {
|
|
|
|
|
it('validates and materializes all 28 executable P0 packages deterministically', async () => {
|
|
|
|
|
const records = await loadBuiltInPlaybookRecords()
|
|
|
|
|
|
|
|
|
|
expect(records).toHaveLength(builtInPlaybookCount)
|
|
|
|
|
expect(records.map((record) => record.slug)).toEqual(
|
|
|
|
|
[...records.map((record) => record.slug)].sort(),
|
|
|
|
|
)
|
|
|
|
|
for (const record of records) {
|
|
|
|
|
expect(record.namespace).toBe('builtin')
|
|
|
|
|
expect(record.sourceType).toBe('built_in')
|
|
|
|
|
expect(record.packageApiVersion).toBe('devrunbook.io/v1alpha1')
|
|
|
|
|
expect(record.contentDigest).toMatch(/^[a-f0-9]{64}$/u)
|
|
|
|
|
expect(record.templateText.endsWith('\n')).toBe(true)
|
|
|
|
|
expect(record.files).toHaveLength(5)
|
|
|
|
|
expect(record.searchProjection.searchText).toContain(record.title)
|
|
|
|
|
expect(record.packageJson.metadata.slug).toBe(record.slug)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const secondLoad = await loadBuiltInPlaybookRecords()
|
|
|
|
|
expect(secondLoad.map((record) => record.contentDigest)).toEqual(
|
|
|
|
|
records.map((record) => record.contentDigest),
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('keeps the existing web summary API compatible', async () => {
|
|
|
|
|
const summaries = await loadBuiltInPlaybooks()
|
|
|
|
|
const rootCause = summaries.find(
|
|
|
|
|
(playbook) => playbook.slug === 'root-cause-bugfix',
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
expect(summaries).toHaveLength(builtInPlaybookCount)
|
|
|
|
|
expect(rootCause).toMatchObject({
|
|
|
|
|
title: 'Root-Cause Bug Fix',
|
|
|
|
|
version: '1.0.0',
|
|
|
|
|
lifecycle: 'reviewed',
|
|
|
|
|
})
|
|
|
|
|
expect(rootCause?.digest).toMatch(/^[a-f0-9]{64}$/u)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('normalizes BOM, line endings, Unicode and trailing whitespace before digesting text', async () => {
|
|
|
|
|
const original = await copyPackage()
|
|
|
|
|
const variant = await copyPackage()
|
|
|
|
|
const promptPath = path.join(variant, 'prompt.md')
|
|
|
|
|
const prompt = await readFile(promptPath, 'utf8')
|
|
|
|
|
const transformed = `\uFEFF${prompt
|
|
|
|
|
.normalize('NFD')
|
|
|
|
|
.split('\n')
|
|
|
|
|
.map((line) => `${line} \t`)
|
|
|
|
|
.join('\r\n')}`
|
|
|
|
|
await writeFile(promptPath, transformed, 'utf8')
|
|
|
|
|
|
|
|
|
|
const [left, right] = await Promise.all([
|
|
|
|
|
loadPlaybookPackage(original),
|
|
|
|
|
loadPlaybookPackage(variant),
|
|
|
|
|
])
|
|
|
|
|
expect(right.templateText).toBe(left.templateText)
|
|
|
|
|
expect(right.contentDigest).toBe(left.contentDigest)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('produces byte-identical records and digests from in-memory files for all 28 built-ins', async () => {
|
|
|
|
|
const filesystemRecords = await loadBuiltInPlaybookRecords()
|
|
|
|
|
|
|
|
|
|
for (const filesystemRecord of filesystemRecords) {
|
|
|
|
|
const packageRoot = path.join(
|
|
|
|
|
defaultBuiltInContentRoot,
|
|
|
|
|
filesystemRecord.slug,
|
|
|
|
|
)
|
|
|
|
|
const memoryRecord = await validatePlaybookPackageFiles(
|
|
|
|
|
await packageFileRecords(packageRoot),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
expect(memoryRecord).toEqual(filesystemRecord)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('normalizes in-memory text before computing its content digest', async () => {
|
|
|
|
|
const packageRoot = await copyPackage()
|
|
|
|
|
const baseline = await loadPlaybookPackage(packageRoot)
|
|
|
|
|
const records = await packageFileRecords(packageRoot)
|
|
|
|
|
const normalizedVariant = records.map((file) => {
|
|
|
|
|
if (file.path !== 'prompt.md') return file
|
|
|
|
|
const source = Buffer.from(file.content).toString('utf8')
|
|
|
|
|
return {
|
|
|
|
|
...file,
|
|
|
|
|
content: Buffer.from(
|
|
|
|
|
`\uFEFF${source
|
|
|
|
|
.normalize('NFD')
|
|
|
|
|
.split('\n')
|
|
|
|
|
.map((line) => `${line} \t`)
|
|
|
|
|
.join('\r\n')}`,
|
|
|
|
|
'utf8',
|
|
|
|
|
),
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
const imported = await validatePlaybookPackageFiles(normalizedVariant)
|
|
|
|
|
expect(imported.templateText).toBe(baseline.templateText)
|
|
|
|
|
expect(imported.contentDigest).toBe(baseline.contentDigest)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('derives archive-entry roles only from the canonical manifest', async () => {
|
|
|
|
|
const packageRoot = await copyPackage()
|
|
|
|
|
const baseline = await loadPlaybookPackage(packageRoot)
|
|
|
|
|
const archiveEntries = (await packageFileRecords(packageRoot)).map(
|
|
|
|
|
({ path: filePath, content }) => ({ path: filePath, content }),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
await expect(
|
|
|
|
|
validatePlaybookPackageArchiveFiles(archiveEntries),
|
|
|
|
|
).resolves.toEqual(baseline)
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
describe('in-memory package validation', () => {
|
|
|
|
|
it('rejects duplicate, colliding and unsafe paths with structured issues', async () => {
|
|
|
|
|
const records = await packageFileRecords(await copyPackage())
|
|
|
|
|
const prompt = records.find((file) => file.path === 'prompt.md')!
|
|
|
|
|
|
|
|
|
|
for (const [candidate, code] of [
|
|
|
|
|
[[...records, { ...prompt }], 'package_path_duplicate'],
|
|
|
|
|
[
|
|
|
|
|
[
|
|
|
|
|
...records,
|
|
|
|
|
{ path: 'PROMPT.md', role: 'template', content: prompt.content },
|
|
|
|
|
],
|
|
|
|
|
'package_path_collision',
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
records.map((file) =>
|
|
|
|
|
file.path === 'prompt.md' ? { ...file, path: '../prompt.md' } : file,
|
|
|
|
|
),
|
|
|
|
|
'package_path_unsafe',
|
|
|
|
|
],
|
|
|
|
|
] as const) {
|
|
|
|
|
const failure = await validatePlaybookPackageFiles(candidate).catch(
|
|
|
|
|
(error: unknown) => error,
|
|
|
|
|
)
|
|
|
|
|
expect(failure).toBeInstanceOf(ContentValidationError)
|
|
|
|
|
expect((failure as ContentValidationError).issues).toEqual(
|
|
|
|
|
expect.arrayContaining([expect.objectContaining({ code })]),
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('rejects oversized files before parsing their content', async () => {
|
|
|
|
|
const records = await packageFileRecords(await copyPackage())
|
|
|
|
|
const oversized = records.map((file) =>
|
|
|
|
|
file.path === 'prompt.md'
|
|
|
|
|
? {
|
|
|
|
|
...file,
|
|
|
|
|
content: new Uint8Array(
|
|
|
|
|
playbookPackageValidationLimits.maxFileBytes + 1,
|
|
|
|
|
),
|
|
|
|
|
}
|
|
|
|
|
: file,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
const failure = await validatePlaybookPackageFiles(oversized).catch(
|
|
|
|
|
(error: unknown) => error,
|
|
|
|
|
)
|
|
|
|
|
expect(failure).toBeInstanceOf(ContentValidationError)
|
|
|
|
|
expect((failure as ContentValidationError).issues).toEqual(
|
|
|
|
|
expect.arrayContaining([
|
|
|
|
|
expect.objectContaining({ code: 'package_file_size_exceeded' }),
|
|
|
|
|
]),
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('requires one correctly-role-labeled canonical manifest', async () => {
|
|
|
|
|
const records = await packageFileRecords(await copyPackage())
|
|
|
|
|
const missing = records.filter((file) => file.path !== 'playbook.yaml')
|
|
|
|
|
const missingFailure = await validatePlaybookPackageFiles(missing).catch(
|
|
|
|
|
(error: unknown) => error,
|
|
|
|
|
)
|
|
|
|
|
expect((missingFailure as ContentValidationError).issues).toEqual(
|
|
|
|
|
expect.arrayContaining([
|
|
|
|
|
expect.objectContaining({ code: 'package_manifest_missing' }),
|
|
|
|
|
]),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
const wrongRole = records.map((file) =>
|
|
|
|
|
file.path === 'playbook.yaml' ? { ...file, role: 'documentation' } : file,
|
|
|
|
|
)
|
|
|
|
|
const roleFailure = await validatePlaybookPackageFiles(wrongRole).catch(
|
|
|
|
|
(error: unknown) => error,
|
|
|
|
|
)
|
|
|
|
|
expect((roleFailure as ContentValidationError).issues).toEqual(
|
|
|
|
|
expect.arrayContaining([
|
|
|
|
|
expect.objectContaining({ code: 'package_file_role_mismatch' }),
|
|
|
|
|
]),
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('rejects invalid UTF-8 and file roles that disagree with the manifest', async () => {
|
|
|
|
|
const records = await packageFileRecords(await copyPackage())
|
|
|
|
|
const invalidText = records.map((file) =>
|
|
|
|
|
file.path === 'prompt.md'
|
|
|
|
|
? { ...file, content: new Uint8Array([0xff]) }
|
|
|
|
|
: file,
|
|
|
|
|
)
|
|
|
|
|
const utf8Failure = await validatePlaybookPackageFiles(invalidText).catch(
|
|
|
|
|
(error: unknown) => error,
|
|
|
|
|
)
|
|
|
|
|
expect((utf8Failure as ContentValidationError).issues).toEqual(
|
|
|
|
|
expect.arrayContaining([
|
|
|
|
|
expect.objectContaining({ path: 'prompt.md', code: 'invalid_utf8' }),
|
|
|
|
|
]),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
const wrongRole = records.map((file) =>
|
|
|
|
|
file.path === 'prompt.md' ? { ...file, role: 'documentation' } : file,
|
|
|
|
|
)
|
|
|
|
|
await expect(validatePlaybookPackageFiles(wrongRole)).rejects.toThrow(
|
|
|
|
|
'does not match manifest role',
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('applies template, evaluation and published changelog semantics in memory', async () => {
|
|
|
|
|
const records = await packageFileRecords(await copyPackage())
|
|
|
|
|
const invalidTemplate = records.map((file) =>
|
|
|
|
|
file.path === 'prompt.md'
|
|
|
|
|
? { ...file, content: '{{ inputs.notDeclared }}\n' }
|
|
|
|
|
: file,
|
|
|
|
|
)
|
|
|
|
|
await expect(validatePlaybookPackageFiles(invalidTemplate)).rejects.toThrow(
|
|
|
|
|
'template references undeclared input notDeclared',
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
const invalidEvaluation = records.map((file) =>
|
|
|
|
|
file.path === 'evaluations/static-structure.yaml'
|
|
|
|
|
? {
|
|
|
|
|
...file,
|
|
|
|
|
content: Buffer.from(file.content)
|
|
|
|
|
.toString('utf8')
|
|
|
|
|
.replace('playbookVersion: 1.0.0', 'playbookVersion: 9.9.9'),
|
|
|
|
|
}
|
|
|
|
|
: file,
|
|
|
|
|
)
|
|
|
|
|
await expect(
|
|
|
|
|
validatePlaybookPackageFiles(invalidEvaluation),
|
|
|
|
|
).rejects.toThrow('playbook version does not match package')
|
|
|
|
|
|
|
|
|
|
const malformedEvaluation = records.map((file) =>
|
|
|
|
|
file.path === 'evaluations/static-structure.yaml'
|
|
|
|
|
? { ...file, content: 'not: [valid\n' }
|
|
|
|
|
: file,
|
|
|
|
|
)
|
|
|
|
|
const evaluationFailure = await validatePlaybookPackageFiles(
|
|
|
|
|
malformedEvaluation,
|
|
|
|
|
).catch((error: unknown) => error)
|
|
|
|
|
expect((evaluationFailure as ContentValidationError).issues).toEqual(
|
|
|
|
|
expect.arrayContaining([
|
|
|
|
|
expect.objectContaining({
|
|
|
|
|
path: 'evaluations/static-structure.yaml',
|
|
|
|
|
code: 'yaml_parse_error',
|
|
|
|
|
}),
|
|
|
|
|
]),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
const invalidChangelog = records.map((file) => {
|
|
|
|
|
if (file.path === 'playbook.yaml') {
|
|
|
|
|
return {
|
|
|
|
|
...file,
|
|
|
|
|
content: Buffer.from(file.content)
|
|
|
|
|
.toString('utf8')
|
|
|
|
|
.replace('role: changelog', 'role: documentation'),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return file.path === 'CHANGELOG.md'
|
|
|
|
|
? { ...file, role: 'documentation' }
|
|
|
|
|
: file
|
|
|
|
|
})
|
|
|
|
|
await expect(
|
|
|
|
|
validatePlaybookPackageFiles(invalidChangelog),
|
|
|
|
|
).rejects.toThrow('published package must declare a changelog')
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
describe('safe package rejection', () => {
|
|
|
|
|
it('rejects undeclared files', async () => {
|
|
|
|
|
const packageRoot = await copyPackage()
|
|
|
|
|
await writeFile(path.join(packageRoot, 'surprise.md'), 'undeclared\n')
|
|
|
|
|
await expect(loadPlaybookPackage(packageRoot)).rejects.toThrow(
|
|
|
|
|
'package inventory mismatch',
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('rejects duplicate YAML keys and custom tags', async () => {
|
|
|
|
|
const duplicateRoot = await copyPackage()
|
|
|
|
|
const duplicateManifest = path.join(duplicateRoot, 'playbook.yaml')
|
|
|
|
|
await writeFile(
|
|
|
|
|
duplicateManifest,
|
|
|
|
|
`${await readFile(duplicateManifest, 'utf8')}\nkind: Playbook\n`,
|
|
|
|
|
)
|
|
|
|
|
await expect(loadPlaybookPackage(duplicateRoot)).rejects.toBeInstanceOf(
|
|
|
|
|
ContentValidationError,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
const taggedRoot = await copyPackage()
|
|
|
|
|
const taggedManifest = path.join(taggedRoot, 'playbook.yaml')
|
|
|
|
|
const tagged = (await readFile(taggedManifest, 'utf8')).replace(
|
|
|
|
|
'title: Root-Cause Bug Fix',
|
|
|
|
|
'title: !untrusted Root-Cause Bug Fix',
|
|
|
|
|
)
|
|
|
|
|
await writeFile(taggedManifest, tagged)
|
|
|
|
|
await expect(loadPlaybookPackage(taggedRoot)).rejects.toBeInstanceOf(
|
|
|
|
|
ContentValidationError,
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('rejects schema-invalid manifests before semantic import', async () => {
|
|
|
|
|
const packageRoot = await copyPackage()
|
|
|
|
|
const manifestPath = path.join(packageRoot, 'playbook.yaml')
|
|
|
|
|
const manifest = (await readFile(manifestPath, 'utf8')).replace(
|
|
|
|
|
'apiVersion: devrunbook.io/v1alpha1',
|
|
|
|
|
'apiVersion: unsafe/v9',
|
|
|
|
|
)
|
|
|
|
|
await writeFile(manifestPath, manifest)
|
|
|
|
|
const failure = await loadPlaybookPackage(packageRoot).catch(
|
|
|
|
|
(error: unknown) => error,
|
|
|
|
|
)
|
|
|
|
|
expect(failure).toBeInstanceOf(ContentValidationError)
|
|
|
|
|
expect(failure).toMatchObject({
|
|
|
|
|
issues: [
|
|
|
|
|
expect.objectContaining({
|
|
|
|
|
path: '/apiVersion',
|
|
|
|
|
code: expect.any(String),
|
|
|
|
|
message: expect.any(String),
|
|
|
|
|
remediation: expect.any(String),
|
|
|
|
|
}),
|
|
|
|
|
],
|
|
|
|
|
})
|
|
|
|
|
expect((failure as Error).message).toContain('JSON Schema validation')
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('rejects semantic secret exposure and unknown template variables', async () => {
|
|
|
|
|
const secretRoot = await copyPackage()
|
|
|
|
|
const secretManifest = path.join(secretRoot, 'playbook.yaml')
|
|
|
|
|
const secret = (await readFile(secretManifest, 'utf8')).replace(
|
|
|
|
|
' sensitive: false\n includeInOutput: true',
|
|
|
|
|
' sensitive: true\n includeInOutput: true',
|
|
|
|
|
)
|
|
|
|
|
await writeFile(secretManifest, secret)
|
|
|
|
|
const secretFailure = await loadPlaybookPackage(secretRoot).catch(
|
|
|
|
|
(error: unknown) => error,
|
|
|
|
|
)
|
|
|
|
|
expect(secretFailure).toBeInstanceOf(ContentValidationError)
|
|
|
|
|
expect(secretFailure).toMatchObject({
|
|
|
|
|
issues: [
|
|
|
|
|
expect.objectContaining({
|
|
|
|
|
path: '/spec/inputs/problemStatement',
|
|
|
|
|
remediation: expect.stringContaining('package contract'),
|
|
|
|
|
}),
|
|
|
|
|
],
|
|
|
|
|
})
|
|
|
|
|
expect((secretFailure as Error).message).toContain(
|
|
|
|
|
'sensitive input problemStatement cannot be included in output',
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
const templateRoot = await copyPackage()
|
|
|
|
|
await writeFile(
|
|
|
|
|
path.join(templateRoot, 'prompt.md'),
|
|
|
|
|
'{{ inputs.notDeclared }}\n',
|
|
|
|
|
)
|
|
|
|
|
await expect(loadPlaybookPackage(templateRoot)).rejects.toThrow(
|
|
|
|
|
'template references undeclared input notDeclared',
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('rejects executable package content even when declared', async () => {
|
|
|
|
|
const packageRoot = await copyPackage()
|
|
|
|
|
const manifestPath = path.join(packageRoot, 'playbook.yaml')
|
|
|
|
|
const manifest = await readFile(manifestPath, 'utf8')
|
|
|
|
|
const declaration = [
|
|
|
|
|
' - path: scripts/run.sh',
|
|
|
|
|
' role: resource',
|
|
|
|
|
' digest: true',
|
|
|
|
|
' exportByDefault: false',
|
|
|
|
|
].join('\n')
|
|
|
|
|
await writeFile(
|
|
|
|
|
manifestPath,
|
|
|
|
|
manifest.replace('spec:\n', `${declaration}\nspec:\n`),
|
|
|
|
|
)
|
|
|
|
|
await mkdir(path.join(packageRoot, 'scripts'))
|
|
|
|
|
await writeFile(path.join(packageRoot, 'scripts', 'run.sh'), '#!/bin/sh\n')
|
|
|
|
|
await expect(loadPlaybookPackage(packageRoot)).rejects.toThrow(
|
|
|
|
|
'executable package content',
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('reports invalid UTF-8 as a structured path-specific issue', async () => {
|
|
|
|
|
const packageRoot = await copyPackage()
|
|
|
|
|
await writeFile(path.join(packageRoot, 'prompt.md'), Buffer.from([0xff]))
|
|
|
|
|
|
|
|
|
|
const failure = await loadPlaybookPackage(packageRoot).catch(
|
|
|
|
|
(error: unknown) => error,
|
|
|
|
|
)
|
|
|
|
|
expect(failure).toBeInstanceOf(ContentValidationError)
|
|
|
|
|
expect(failure).toMatchObject({
|
|
|
|
|
issues: [
|
|
|
|
|
expect.objectContaining({
|
|
|
|
|
path: 'prompt.md',
|
|
|
|
|
code: 'invalid_utf8',
|
|
|
|
|
remediation: expect.stringContaining('UTF-8'),
|
|
|
|
|
}),
|
|
|
|
|
],
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('rejects hard-linked package files', async () => {
|
|
|
|
|
const packageRoot = await copyPackage()
|
|
|
|
|
await link(
|
|
|
|
|
path.join(packageRoot, 'prompt.md'),
|
|
|
|
|
path.join(packageRoot, 'prompt-link.md'),
|
|
|
|
|
)
|
|
|
|
|
const manifestPath = path.join(packageRoot, 'playbook.yaml')
|
|
|
|
|
const manifest = await readFile(manifestPath, 'utf8')
|
|
|
|
|
await writeFile(
|
|
|
|
|
manifestPath,
|
|
|
|
|
manifest.replace(
|
|
|
|
|
'spec:\n',
|
|
|
|
|
[
|
|
|
|
|
' - path: prompt-link.md',
|
|
|
|
|
' role: resource',
|
|
|
|
|
' digest: true',
|
|
|
|
|
' exportByDefault: false',
|
|
|
|
|
'spec:',
|
|
|
|
|
'',
|
|
|
|
|
].join('\n'),
|
|
|
|
|
),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
await expect(loadPlaybookPackage(packageRoot)).rejects.toThrow('hard link')
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it(
|
|
|
|
|
'aggregates validation issues from multiple built-in directories',
|
|
|
|
|
async () => {
|
|
|
|
|
const root = await temporaryDirectory('aggregate-catalog')
|
|
|
|
|
await cp(defaultBuiltInContentRoot, root, { recursive: true })
|
|
|
|
|
for (const slug of ['accessibility-audit', 'agents-instructions']) {
|
|
|
|
|
await writeFile(path.join(root, slug, 'playbook.yaml'), 'not: [valid\n')
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const failure = await loadBuiltInPlaybookRecords(root).catch(
|
|
|
|
|
(error: unknown) => error,
|
|
|
|
|
)
|
|
|
|
|
expect(failure).toBeInstanceOf(ContentValidationError)
|
|
|
|
|
const issues = (failure as ContentValidationError).issues
|
|
|
|
|
expect(
|
|
|
|
|
issues.some((issue) => issue.path.includes('accessibility-audit')),
|
|
|
|
|
).toBe(true)
|
|
|
|
|
expect(
|
|
|
|
|
issues.some((issue) => issue.path.includes('agents-instructions')),
|
|
|
|
|
).toBe(true)
|
|
|
|
|
},
|
|
|
|
|
exhaustiveCatalogTimeout,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
it(
|
|
|
|
|
'rejects a P0 package that differs from the governed seed catalog',
|
|
|
|
|
async () => {
|
|
|
|
|
const root = await temporaryDirectory('catalog-mismatch-content')
|
|
|
|
|
await cp(defaultBuiltInContentRoot, root, { recursive: true })
|
|
|
|
|
const catalogRoot = await temporaryDirectory('catalog-mismatch-seed')
|
|
|
|
|
const catalogPath = path.join(catalogRoot, 'seed-catalog.yaml')
|
|
|
|
|
const catalog = (await readFile(defaultSeedCatalogPath, 'utf8')).replace(
|
|
|
|
|
'title: Accessibility Audit',
|
|
|
|
|
'title: Accessibility Audit Mismatch',
|
|
|
|
|
)
|
|
|
|
|
await writeFile(catalogPath, catalog)
|
|
|
|
|
|
|
|
|
|
const failure = await loadBuiltInPlaybookRecords(root, catalogPath).catch(
|
|
|
|
|
(error: unknown) => error,
|
|
|
|
|
)
|
|
|
|
|
expect(failure).toBeInstanceOf(ContentValidationError)
|
|
|
|
|
expect((failure as ContentValidationError).issues).toEqual(
|
|
|
|
|
expect.arrayContaining([
|
|
|
|
|
expect.objectContaining({
|
|
|
|
|
code: 'p0_catalog_mismatch',
|
|
|
|
|
message: expect.stringContaining('title differs'),
|
|
|
|
|
}),
|
|
|
|
|
]),
|
|
|
|
|
)
|
|
|
|
|
},
|
|
|
|
|
exhaustiveCatalogTimeout,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
it(
|
|
|
|
|
'rejects duplicate identity and version across the built-in catalog',
|
|
|
|
|
async () => {
|
|
|
|
|
const root = await temporaryDirectory('duplicate-catalog')
|
|
|
|
|
await cp(defaultBuiltInContentRoot, root, { recursive: true })
|
|
|
|
|
const firstPath = path.join(root, 'accessibility-audit', 'playbook.yaml')
|
|
|
|
|
const secondPath = path.join(root, 'agents-instructions', 'playbook.yaml')
|
|
|
|
|
const first = await readFile(firstPath, 'utf8')
|
|
|
|
|
const firstId = /^ {2}id: (.+)$/mu.exec(first)?.[1]
|
|
|
|
|
expect(firstId).toBeTruthy()
|
|
|
|
|
const second = (await readFile(secondPath, 'utf8')).replace(
|
|
|
|
|
/^ {2}id: .+$/mu,
|
|
|
|
|
` id: ${firstId}`,
|
|
|
|
|
)
|
|
|
|
|
await writeFile(secondPath, second)
|
|
|
|
|
|
|
|
|
|
await expect(loadBuiltInPlaybookRecords(root)).rejects.toThrow(
|
|
|
|
|
'Duplicate built-in package',
|
|
|
|
|
)
|
|
|
|
|
},
|
|
|
|
|
exhaustiveCatalogTimeout,
|
|
|
|
|
)
|
|
|
|
|
})
|
DevRunbook release export