Publish DevRunbook source

packages/application/src/quality/static-quality-evaluation.test.ts

@@ -0,0 +1,316 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+  assessLifecycleEvidence,
+  compareVersionEvaluations,
+  createQualityFinding,
+  createQualityMatrix,
+  evaluateStaticCase,
+  evaluationFreshness,
+  QUALITY_DIMENSIONS,
+  type LifecycleEvidence,
+  type StaticEvaluationCase,
+  type StaticEvaluationObservation,
+  type StaticEvaluationResult,
+} from './static-quality-evaluation'
+
+const target = { id: 'playbook', version: '1.0.0', digest: 'a'.repeat(64) }
+const fixture = {
+  id: 'fixture',
+  version: '2.0.0',
+  digest: 'b'.repeat(64),
+  environmentDigest: 'c'.repeat(64),
+}
+const evaluationCase: StaticEvaluationCase = {
+  id: 'safe-change.static',
+  version: '1.0.0',
+  target,
+  fixture,
+  expectedHeadings: ['# Mission'],
+  requiredText: ['Do not modify protected paths.'],
+  prohibitedText: ['Authorization: Bearer'],
+  deterministic: true,
+  expectedLintStatus: 'ready',
+}
+const observation: StaticEvaluationObservation = {
+  target,
+  fixture,
+  renderedPrompt: '# Mission\n\nDo not modify protected paths.',
+  renderedPromptDigest: 'd'.repeat(64),
+  repeatedRenderDigest: 'd'.repeat(64),
+  lintStatus: 'ready',
+  evaluatedAt: '2026-07-27T10:00:00.000Z',
+}
+
+function result(
+  overrides: Partial<StaticEvaluationResult> = {},
+): StaticEvaluationResult {
+  return { ...evaluateStaticCase(evaluationCase, observation), ...overrides }
+}
+
+function evidence(
+  overrides: Partial<LifecycleEvidence> = {},
+): LifecycleEvidence {
+  return {
+    schemaAndSemanticValidationPassed: true,
+    blockingLintFindingCount: 0,
+    humanEditorialReviewCompleted: true,
+    limitationsDocumented: true,
+    evaluationResults: [result()],
+    currentEvaluationContext: { target, fixture },
+    unresolvedSafetyRegression: false,
+    realWorldRunCount: 20,
+    realWorldFailureCount: 1,
+    unaddressedSevereIncidentCount: 0,
+    latestRealWorldEvidenceAt: '2026-07-20T10:00:00.000Z',
+    ...overrides,
+  }
+}
+
+const policy = {
+  requiredEvaluationCaseIds: [evaluationCase.id],
+  minimumRealWorldRuns: 10,
+  maximumFailureRate: 0.1,
+  maximumEvidenceAgeDays: 30,
+}
+
+describe('quality findings and dimensions', () => {
+  it('keeps every dimension visible without inventing an aggregate score', () => {
+    const matrix = createQualityMatrix([
+      {
+        dimension: 'safety',
+        rating: 'strong',
+        rationale: 'Protected paths are explicit.',
+        provenance: [{ kind: 'static-evaluation', source: 'case-1' }],
+      },
+    ])
+
+    expect(Object.keys(matrix)).toEqual(QUALITY_DIMENSIONS)
+    expect(matrix.safety.rating).toBe('strong')
+    expect(matrix.reporting.rating).toBe('not-assessed')
+    expect(matrix).not.toHaveProperty('score')
+  })
+
+  it('creates structured PB/PR/SA/VA findings and rejects unknown families', () => {
+    const finding = createQualityFinding({
+      ruleId: 'SA002',
+      severity: 'error',
+      path: 'spec.scope.paths[0]',
+      message: 'Protected path is mutable.',
+      rationale: 'The scope conflicts with repository policy.',
+      remediation: 'Exclude the protected path.',
+      provenance: { kind: 'static-analysis', source: 'prompt-linter' },
+    })
+    expect(finding.family).toBe('SA')
+    expect(() => createQualityFinding({ ...finding, ruleId: 'XX001' })).toThrow(
+      /PB, PR, SA or VA/,
+    )
+  })
+})
+
+describe('static evaluation', () => {
+  it('passes literal expectations tied to exact identities', () => {
+    const assessed = evaluateStaticCase(evaluationCase, observation)
+    expect(assessed.status).toBe('passed')
+    expect(assessed.checks).toHaveLength(7)
+    expect(assessed.target).toEqual(target)
+    expect(assessed.fixture).toEqual(fixture)
+  })
+
+  it('treats supplied pattern syntax as literal text, never as a regex', () => {
+    const assessed = evaluateStaticCase(
+      {
+        ...evaluationCase,
+        requiredText: ['(a+)+$'],
+        prohibitedText: ['.*secret.*'],
+      },
+      { ...observation, renderedPrompt: '# Mission\n(a+)+$' },
+    )
+    expect(
+      assessed.checks.find((check) => check.kind === 'required-text')?.passed,
+    ).toBe(true)
+    expect(
+      assessed.checks.find((check) => check.kind === 'prohibited-text')?.passed,
+    ).toBe(true)
+  })
+
+  it('fails changed playbook identity and a mismatched repeated digest', () => {
+    const assessed = evaluateStaticCase(evaluationCase, {
+      ...observation,
+      target: { ...target, version: '1.0.1' },
+      repeatedRenderDigest: 'e'.repeat(64),
+    })
+    expect(assessed.status).toBe('failed')
+    expect(
+      assessed.checks
+        .filter((check) => !check.passed)
+        .map((check) => check.kind),
+    ).toEqual(['identity', 'determinism'])
+  })
+
+  it('detects stale playbook, fixture and environment evidence independently', () => {
+    expect(
+      evaluationFreshness(result(), {
+        target: { ...target, digest: 'x'.repeat(64) },
+        fixture: {
+          ...fixture,
+          version: '2.1.0',
+          environmentDigest: 'y'.repeat(64),
+        },
+      }),
+    ).toEqual({
+      stale: true,
+      reasons: [
+        'playbook-digest-changed',
+        'fixture-version-changed',
+        'environment-changed',
+      ],
+    })
+  })
+})
+
+describe('lifecycle evidence policy', () => {
+  it('allows draft without presenting it as evidence-backed', () => {
+    expect(
+      assessLifecycleEvidence(
+        'draft',
+        evidence({ evaluationResults: [] }),
+        policy,
+        new Date(),
+      ),
+    ).toMatchObject({ eligible: true, requirements: [], findings: [] })
+  })
+
+  it('blocks reviewed when editorial requirements are missing', () => {
+    const assessed = assessLifecycleEvidence(
+      'reviewed',
+      evidence({ humanEditorialReviewCompleted: false }),
+      policy,
+      new Date('2026-07-27T10:00:00.000Z'),
+    )
+    expect(assessed.eligible).toBe(false)
+    expect(assessed.findings[0]).toMatchObject({
+      ruleId: 'PB009',
+      family: 'PB',
+    })
+  })
+
+  it('blocks validated when required evidence is stale', () => {
+    const assessed = assessLifecycleEvidence(
+      'validated',
+      evidence({
+        currentEvaluationContext: {
+          target: { ...target, version: '1.1.0' },
+          fixture,
+        },
+      }),
+      policy,
+      new Date('2026-07-27T10:00:00.000Z'),
+    )
+    expect(assessed.eligible).toBe(false)
+    expect(
+      assessed.requirements.find((item) => item.id.startsWith('evaluation:')),
+    ).toMatchObject({
+      satisfied: false,
+    })
+  })
+
+  it('requires real-world volume, failure rate, incident and recency evidence for battle-tested', () => {
+    const accepted = assessLifecycleEvidence(
+      'battle-tested',
+      evidence(),
+      policy,
+      new Date('2026-07-27T10:00:00.000Z'),
+    )
+    expect(accepted.eligible).toBe(true)
+
+    const rejected = assessLifecycleEvidence(
+      'battle-tested',
+      evidence({
+        realWorldRunCount: 2,
+        realWorldFailureCount: 1,
+        unaddressedSevereIncidentCount: 1,
+        latestRealWorldEvidenceAt: '2025-01-01T00:00:00.000Z',
+      }),
+      policy,
+      new Date('2026-07-27T10:00:00.000Z'),
+    )
+    expect(
+      rejected.requirements
+        .filter((item) => !item.satisfied)
+        .map((item) => item.id),
+    ).toEqual([
+      'real-world-runs',
+      'failure-rate',
+      'severe-incidents',
+      'evidence-recency',
+    ])
+  })
+
+  it('requires a rationale or replacement for deprecated lifecycle', () => {
+    expect(
+      assessLifecycleEvidence('deprecated', evidence(), policy, new Date())
+        .eligible,
+    ).toBe(false)
+    expect(
+      assessLifecycleEvidence(
+        'deprecated',
+        evidence({ replacementPlaybookId: 'safe-change-v2' }),
+        policy,
+        new Date(),
+      ).eligible,
+    ).toBe(true)
+  })
+})
+
+describe('version comparison', () => {
+  it('compares common exact cases and reports dimension changes separately', () => {
+    const previous = result({
+      status: 'failed',
+      dimensions: createQualityMatrix([
+        {
+          dimension: 'verification',
+          rating: 'weak',
+          rationale: 'Missing evidence.',
+          provenance: [],
+        },
+      ]),
+    })
+    const current = result({
+      target: { ...target, version: '1.1.0', digest: 'f'.repeat(64) },
+      dimensions: createQualityMatrix([
+        {
+          dimension: 'verification',
+          rating: 'strong',
+          rationale: 'Evidence is explicit.',
+          provenance: [
+            { kind: 'static-evaluation', source: evaluationCase.id },
+          ],
+        },
+      ]),
+    })
+    expect(compareVersionEvaluations([previous], [current])).toEqual({
+      newlyPassingCaseIds: [evaluationCase.id],
+      newlyFailingCaseIds: [],
+      unchangedCaseIds: [],
+      dimensionChanges: [
+        {
+          caseId: evaluationCase.id,
+          dimension: 'verification',
+          previous: 'weak',
+          current: 'strong',
+        },
+      ],
+    })
+  })
+
+  it('does not compare results from a changed fixture', () => {
+    const changedFixture = result({ fixture: { ...fixture, version: '3.0.0' } })
+    expect(compareVersionEvaluations([result()], [changedFixture])).toEqual({
+      newlyPassingCaseIds: [],
+      newlyFailingCaseIds: [],
+      unchangedCaseIds: [],
+      dimensionChanges: [],
+    })
+  })
+})