This commit is contained in:
DevRunbook release export
@@ -0,0 +1,292 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import {
|
||||
lintPlaybookPackage,
|
||||
type PlaybookPackageLintInput,
|
||||
} from './playbook-package-linter'
|
||||
|
||||
interface MutableManifest {
|
||||
metadata: { lifecycle: string }
|
||||
package: { files: unknown[] }
|
||||
spec: {
|
||||
intent: unknown
|
||||
modes: unknown[]
|
||||
autonomy: unknown
|
||||
inputs: unknown[]
|
||||
workflow: unknown[]
|
||||
validation: unknown
|
||||
completion: unknown
|
||||
reporting: unknown
|
||||
}
|
||||
quality: { evaluationCaseIds: string[] }
|
||||
}
|
||||
|
||||
function mutableManifest(input: PlaybookPackageLintInput): MutableManifest {
|
||||
return input.packageJson as unknown as MutableManifest
|
||||
}
|
||||
|
||||
function validInput(
|
||||
overrides: Partial<PlaybookPackageLintInput> = {},
|
||||
): PlaybookPackageLintInput {
|
||||
return {
|
||||
packageDigest: 'a'.repeat(64),
|
||||
packageJson: {
|
||||
metadata: { lifecycle: 'reviewed' },
|
||||
package: {
|
||||
files: [{ path: 'CHANGELOG.md', role: 'changelog' }],
|
||||
},
|
||||
spec: {
|
||||
intent: {
|
||||
problem: 'A bounded problem.',
|
||||
outcome: 'A verifiable outcome.',
|
||||
whenToUse: ['For a bounded change.'],
|
||||
whenNotToUse: ['When authority is missing.'],
|
||||
},
|
||||
modes: ['implement'],
|
||||
autonomy: { min: 'observe', max: 'verify', default: 'implement' },
|
||||
inputs: [{ key: 'target' }],
|
||||
workflow: [
|
||||
{
|
||||
id: 'implement',
|
||||
instruction: 'Implement the requested bounded behavior.',
|
||||
},
|
||||
],
|
||||
validation: {
|
||||
commandRoles: ['build'],
|
||||
checks: [
|
||||
{
|
||||
id: 'build',
|
||||
description: 'Run the production build.',
|
||||
evidence: 'Record command result and exit status.',
|
||||
},
|
||||
],
|
||||
},
|
||||
completion: { criteria: ['The requested behavior is verified.'] },
|
||||
reporting: {
|
||||
sections: [
|
||||
{
|
||||
id: 'evidence',
|
||||
description: 'Report inspected evidence sources.',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
quality: { evaluationCaseIds: [] },
|
||||
},
|
||||
template: {
|
||||
path: 'prompt.md',
|
||||
content: 'Work only on {{ inputs.target }}.',
|
||||
},
|
||||
representativeRenderedPrompts: [
|
||||
{
|
||||
path: 'examples/minimal.rendered.md',
|
||||
content: 'Work only on packages/application. Record evidence.',
|
||||
},
|
||||
],
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
describe('playbook package linter', () => {
|
||||
it('reports a clean validated package as export-ready only with exact evidence', () => {
|
||||
const input = validInput()
|
||||
const manifest = mutableManifest(input)
|
||||
manifest.metadata.lifecycle = 'validated'
|
||||
manifest.quality.evaluationCaseIds = ['bounded.static']
|
||||
|
||||
const result = lintPlaybookPackage({
|
||||
...input,
|
||||
lifecycleEvidence: {
|
||||
targetDigest: 'a'.repeat(64),
|
||||
passingEvaluationCaseIds: ['bounded.static'],
|
||||
fixtureVersionRecorded: true,
|
||||
environmentDigest: 'b'.repeat(64),
|
||||
unresolvedSafetyRegression: false,
|
||||
},
|
||||
})
|
||||
|
||||
expect(result).toMatchObject({
|
||||
exportReadiness: 'ready',
|
||||
findings: [],
|
||||
provenance: {
|
||||
kind: 'static-analysis',
|
||||
source: 'playbook-package-linter/v1',
|
||||
artifactDigest: 'a'.repeat(64),
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('finds structural, duplicate, autonomy, template and publication defects', () => {
|
||||
const input = validInput({ published: true })
|
||||
const manifest = mutableManifest(input)
|
||||
manifest.spec.intent = {}
|
||||
manifest.spec.inputs = [{ key: 'same' }, { key: 'same' }]
|
||||
manifest.spec.workflow = [{ id: 'same' }, { id: 'same' }]
|
||||
manifest.spec.autonomy = {
|
||||
min: 'repair',
|
||||
max: 'observe',
|
||||
default: 'invalid',
|
||||
}
|
||||
manifest.spec.completion = { criteria: [] }
|
||||
manifest.spec.reporting = { sections: [] }
|
||||
manifest.package.files = []
|
||||
|
||||
const result = lintPlaybookPackage({
|
||||
...input,
|
||||
template: { path: 'prompt.md', content: '{{ inputs.missing }}' },
|
||||
})
|
||||
|
||||
expect(result.exportReadiness).toBe('blocked')
|
||||
expect(result.findings.map((finding) => finding.ruleId)).toEqual([
|
||||
'PB001',
|
||||
'PB002',
|
||||
'PB003',
|
||||
'PB004',
|
||||
'PB005',
|
||||
'PB005',
|
||||
'PB006',
|
||||
'PB007',
|
||||
'PB008',
|
||||
])
|
||||
expect(
|
||||
result.findings.find((finding) => finding.ruleId === 'PB007')?.path,
|
||||
).toBe('prompt.md:1')
|
||||
})
|
||||
|
||||
it('blocks validated claims when evidence is absent, stale or incomplete', () => {
|
||||
const input = validInput()
|
||||
const manifest = mutableManifest(input)
|
||||
manifest.metadata.lifecycle = 'validated'
|
||||
manifest.quality.evaluationCaseIds = ['case-a', 'case-b']
|
||||
|
||||
const result = lintPlaybookPackage({
|
||||
...input,
|
||||
lifecycleEvidence: {
|
||||
targetDigest: 'stale',
|
||||
passingEvaluationCaseIds: ['case-a'],
|
||||
fixtureVersionRecorded: false,
|
||||
unresolvedSafetyRegression: true,
|
||||
},
|
||||
})
|
||||
|
||||
expect(result.findings).toEqual([
|
||||
expect.objectContaining({
|
||||
ruleId: 'PB009',
|
||||
path: '/metadata/lifecycle',
|
||||
provenance: expect.objectContaining({ artifactDigest: 'a'.repeat(64) }),
|
||||
}),
|
||||
])
|
||||
})
|
||||
|
||||
it('flags only literal prompt risks and never exposes a matched secret', () => {
|
||||
const secret = `ghp_${'x'.repeat(32)}`
|
||||
const result = lintPlaybookPackage({
|
||||
...validInput(),
|
||||
representativeRenderedPrompts: [
|
||||
{
|
||||
path: 'rendered/risky.md',
|
||||
content: [
|
||||
'Improve everything using best practices.',
|
||||
'Do not modify the repository. Modify the repository and claim success.',
|
||||
'Drop table users.',
|
||||
'Run git push and create a release.',
|
||||
`Use ${secret}`,
|
||||
].join('\n'),
|
||||
},
|
||||
],
|
||||
repository: {},
|
||||
})
|
||||
|
||||
expect(result.findings.map((finding) => finding.ruleId)).toEqual([
|
||||
'PR001',
|
||||
'PR002',
|
||||
'PR003',
|
||||
'PR004',
|
||||
'SA001',
|
||||
'SA003',
|
||||
'SA004',
|
||||
'SA004',
|
||||
])
|
||||
expect(JSON.stringify(result)).not.toContain(secret)
|
||||
})
|
||||
|
||||
it('uses explicit repository, trust-boundary and task context for safety and validation rules', () => {
|
||||
const input = validInput({
|
||||
taskKind: 'dependency-change',
|
||||
repository: {
|
||||
protectedPaths: ['infra/production'],
|
||||
changeScopePaths: ['infra'],
|
||||
},
|
||||
importedContentPlacements: [
|
||||
{
|
||||
sourcePath: 'README.md',
|
||||
destinationSection: 'Authoritative policy',
|
||||
},
|
||||
],
|
||||
})
|
||||
const result = lintPlaybookPackage(input)
|
||||
|
||||
expect(result.findings.map((finding) => finding.ruleId)).toEqual([
|
||||
'SA002',
|
||||
'SA005',
|
||||
'VA003',
|
||||
])
|
||||
expect(result.findings.every((finding) => finding.path.length > 0)).toBe(
|
||||
true,
|
||||
)
|
||||
expect(
|
||||
result.findings.every((finding) => finding.rationale.length > 0),
|
||||
).toBe(true)
|
||||
expect(
|
||||
result.findings.every((finding) => finding.remediation.length > 0),
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('covers contextual bugfix, implementation, frontend and inspection validation rules', () => {
|
||||
const base = validInput()
|
||||
const manifest = mutableManifest(base)
|
||||
manifest.spec.validation = { commandRoles: [], checks: [] }
|
||||
manifest.spec.workflow = [{ id: 'work', instruction: 'Perform work.' }]
|
||||
manifest.spec.reporting = {
|
||||
sections: [{ id: 'outcome', description: 'Outcome.' }],
|
||||
}
|
||||
manifest.spec.modes = []
|
||||
|
||||
expect(
|
||||
lintPlaybookPackage({ ...base, taskKind: 'bugfix' }).findings,
|
||||
).toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ ruleId: 'VA001' })]),
|
||||
)
|
||||
expect(
|
||||
lintPlaybookPackage({
|
||||
...base,
|
||||
taskKind: 'implementation',
|
||||
repository: { availableCommandRoles: ['build'] },
|
||||
}).findings,
|
||||
).toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ ruleId: 'VA002' })]),
|
||||
)
|
||||
expect(
|
||||
lintPlaybookPackage({ ...base, taskKind: 'frontend-flow' }).findings,
|
||||
).toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ ruleId: 'VA004' })]),
|
||||
)
|
||||
expect(
|
||||
lintPlaybookPackage({ ...base, taskKind: 'inspection' }).findings,
|
||||
).toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ ruleId: 'VA005' })]),
|
||||
)
|
||||
})
|
||||
|
||||
it('is byte-for-byte deterministic for inert pattern-like package text', () => {
|
||||
const input = validInput({
|
||||
template: {
|
||||
path: 'prompt.md',
|
||||
content: '(a+)+$ {{ inputs.target }} ${notExecuted} <%= inert %>',
|
||||
},
|
||||
})
|
||||
expect(JSON.stringify(lintPlaybookPackage(input))).toBe(
|
||||
JSON.stringify(lintPlaybookPackage(input)),
|
||||
)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user