DevRunbook-Public/packages/application/src/quality/static-quality-evaluation.test.ts

import { describe, expect, it } from 'vitest'

import {
  assessLifecycleEvidence,
  compareVersionEvaluations,
  createQualityFinding,
  createQualityMatrix,
  evaluateStaticCase,
  evaluationFreshness,
  QUALITY_DIMENSIONS,
  type LifecycleEvidence,
  type StaticEvaluationCase,
  type StaticEvaluationObservation,
  type StaticEvaluationResult,
} from './static-quality-evaluation'

const target = { id: 'playbook', version: '1.0.0', digest: 'a'.repeat(64) }
const fixture = {
  id: 'fixture',
  version: '2.0.0',
  digest: 'b'.repeat(64),
  environmentDigest: 'c'.repeat(64),
}
const evaluationCase: StaticEvaluationCase = {
  id: 'safe-change.static',
  version: '1.0.0',
  target,
  fixture,
  expectedHeadings: ['# Mission'],
  requiredText: ['Do not modify protected paths.'],
  prohibitedText: ['Authorization: Bearer'],
  deterministic: true,
  expectedLintStatus: 'ready',
}
const observation: StaticEvaluationObservation = {
  target,
  fixture,
  renderedPrompt: '# Mission\n\nDo not modify protected paths.',
  renderedPromptDigest: 'd'.repeat(64),
  repeatedRenderDigest: 'd'.repeat(64),
  lintStatus: 'ready',
  evaluatedAt: '2026-07-27T10:00:00.000Z',
}

function result(
  overrides: Partial<StaticEvaluationResult> = {},
): StaticEvaluationResult {
  return { ...evaluateStaticCase(evaluationCase, observation), ...overrides }
}

function evidence(
  overrides: Partial<LifecycleEvidence> = {},
): LifecycleEvidence {
  return {
    schemaAndSemanticValidationPassed: true,
    blockingLintFindingCount: 0,
    humanEditorialReviewCompleted: true,
    limitationsDocumented: true,
    evaluationResults: [result()],
    currentEvaluationContext: { target, fixture },
    unresolvedSafetyRegression: false,
    realWorldRunCount: 20,
    realWorldFailureCount: 1,
    unaddressedSevereIncidentCount: 0,
    latestRealWorldEvidenceAt: '2026-07-20T10:00:00.000Z',
    ...overrides,
  }
}

const policy = {
  requiredEvaluationCaseIds: [evaluationCase.id],
  minimumRealWorldRuns: 10,
  maximumFailureRate: 0.1,
  maximumEvidenceAgeDays: 30,
}

describe('quality findings and dimensions', () => {
  it('keeps every dimension visible without inventing an aggregate score', () => {
    const matrix = createQualityMatrix([
      {
        dimension: 'safety',
        rating: 'strong',
        rationale: 'Protected paths are explicit.',
        provenance: [{ kind: 'static-evaluation', source: 'case-1' }],
      },
    ])

    expect(Object.keys(matrix)).toEqual(QUALITY_DIMENSIONS)
    expect(matrix.safety.rating).toBe('strong')
    expect(matrix.reporting.rating).toBe('not-assessed')
    expect(matrix).not.toHaveProperty('score')
  })

  it('creates structured PB/PR/SA/VA findings and rejects unknown families', () => {
    const finding = createQualityFinding({
      ruleId: 'SA002',
      severity: 'error',
      path: 'spec.scope.paths[0]',
      message: 'Protected path is mutable.',
      rationale: 'The scope conflicts with repository policy.',
      remediation: 'Exclude the protected path.',
      provenance: { kind: 'static-analysis', source: 'prompt-linter' },
    })
    expect(finding.family).toBe('SA')
    expect(() => createQualityFinding({ ...finding, ruleId: 'XX001' })).toThrow(
      /PB, PR, SA or VA/,
    )
  })
})

describe('static evaluation', () => {
  it('passes literal expectations tied to exact identities', () => {
    const assessed = evaluateStaticCase(evaluationCase, observation)
    expect(assessed.status).toBe('passed')
    expect(assessed.checks).toHaveLength(7)
    expect(assessed.target).toEqual(target)
    expect(assessed.fixture).toEqual(fixture)
  })

  it('treats supplied pattern syntax as literal text, never as a regex', () => {
    const assessed = evaluateStaticCase(
      {
        ...evaluationCase,
        requiredText: ['(a+)+$'],
        prohibitedText: ['.*secret.*'],
      },
      { ...observation, renderedPrompt: '# Mission\n(a+)+$' },
    )
    expect(
      assessed.checks.find((check) => check.kind === 'required-text')?.passed,
    ).toBe(true)
    expect(
      assessed.checks.find((check) => check.kind === 'prohibited-text')?.passed,
    ).toBe(true)
  })

  it('fails changed playbook identity and a mismatched repeated digest', () => {
    const assessed = evaluateStaticCase(evaluationCase, {
      ...observation,
      target: { ...target, version: '1.0.1' },
      repeatedRenderDigest: 'e'.repeat(64),
    })
    expect(assessed.status).toBe('failed')
    expect(
      assessed.checks
        .filter((check) => !check.passed)
        .map((check) => check.kind),
    ).toEqual(['identity', 'determinism'])
  })

  it('detects stale playbook, fixture and environment evidence independently', () => {
    expect(
      evaluationFreshness(result(), {
        target: { ...target, digest: 'x'.repeat(64) },
        fixture: {
          ...fixture,
          version: '2.1.0',
          environmentDigest: 'y'.repeat(64),
        },
      }),
    ).toEqual({
      stale: true,
      reasons: [
        'playbook-digest-changed',
        'fixture-version-changed',
        'environment-changed',
      ],
    })
  })
})

describe('lifecycle evidence policy', () => {
  it('allows draft without presenting it as evidence-backed', () => {
    expect(
      assessLifecycleEvidence(
        'draft',
        evidence({ evaluationResults: [] }),
        policy,
        new Date(),
      ),
    ).toMatchObject({ eligible: true, requirements: [], findings: [] })
  })

  it('blocks reviewed when editorial requirements are missing', () => {
    const assessed = assessLifecycleEvidence(
      'reviewed',
      evidence({ humanEditorialReviewCompleted: false }),
      policy,
      new Date('2026-07-27T10:00:00.000Z'),
    )
    expect(assessed.eligible).toBe(false)
    expect(assessed.findings[0]).toMatchObject({
      ruleId: 'PB009',
      family: 'PB',
    })
  })

  it('blocks validated when required evidence is stale', () => {
    const assessed = assessLifecycleEvidence(
      'validated',
      evidence({
        currentEvaluationContext: {
          target: { ...target, version: '1.1.0' },
          fixture,
        },
      }),
      policy,
      new Date('2026-07-27T10:00:00.000Z'),
    )
    expect(assessed.eligible).toBe(false)
    expect(
      assessed.requirements.find((item) => item.id.startsWith('evaluation:')),
    ).toMatchObject({
      satisfied: false,
    })
  })

  it('requires real-world volume, failure rate, incident and recency evidence for battle-tested', () => {
    const accepted = assessLifecycleEvidence(
      'battle-tested',
      evidence(),
      policy,
      new Date('2026-07-27T10:00:00.000Z'),
    )
    expect(accepted.eligible).toBe(true)

    const rejected = assessLifecycleEvidence(
      'battle-tested',
      evidence({
        realWorldRunCount: 2,
        realWorldFailureCount: 1,
        unaddressedSevereIncidentCount: 1,
        latestRealWorldEvidenceAt: '2025-01-01T00:00:00.000Z',
      }),
      policy,
      new Date('2026-07-27T10:00:00.000Z'),
    )
    expect(
      rejected.requirements
        .filter((item) => !item.satisfied)
        .map((item) => item.id),
    ).toEqual([
      'real-world-runs',
      'failure-rate',
      'severe-incidents',
      'evidence-recency',
    ])
  })

  it('requires a rationale or replacement for deprecated lifecycle', () => {
    expect(
      assessLifecycleEvidence('deprecated', evidence(), policy, new Date())
        .eligible,
    ).toBe(false)
    expect(
      assessLifecycleEvidence(
        'deprecated',
        evidence({ replacementPlaybookId: 'safe-change-v2' }),
        policy,
        new Date(),
      ).eligible,
    ).toBe(true)
  })
})

describe('version comparison', () => {
  it('compares common exact cases and reports dimension changes separately', () => {
    const previous = result({
      status: 'failed',
      dimensions: createQualityMatrix([
        {
          dimension: 'verification',
          rating: 'weak',
          rationale: 'Missing evidence.',
          provenance: [],
        },
      ]),
    })
    const current = result({
      target: { ...target, version: '1.1.0', digest: 'f'.repeat(64) },
      dimensions: createQualityMatrix([
        {
          dimension: 'verification',
          rating: 'strong',
          rationale: 'Evidence is explicit.',
          provenance: [
            { kind: 'static-evaluation', source: evaluationCase.id },
          ],
        },
      ]),
    })
    expect(compareVersionEvaluations([previous], [current])).toEqual({
      newlyPassingCaseIds: [evaluationCase.id],
      newlyFailingCaseIds: [],
      unchangedCaseIds: [],
      dimensionChanges: [
        {
          caseId: evaluationCase.id,
          dimension: 'verification',
          previous: 'weak',
          current: 'strong',
        },
      ],
    })
  })

  it('does not compare results from a changed fixture', () => {
    const changedFixture = result({ fixture: { ...fixture, version: '3.0.0' } })
    expect(compareVersionEvaluations([result()], [changedFixture])).toEqual({
      newlyPassingCaseIds: [],
      newlyFailingCaseIds: [],
      unchangedCaseIds: [],
      dimensionChanges: [],
    })
  })
})